From 979837f134d83c9517ab7d358084b6ea1e6cf185 Mon Sep 17 00:00:00 2001 From: Derek Bekoe Date: Tue, 14 Mar 2017 10:50:16 -0700 Subject: [PATCH 1/2] Script that creates packaged releases package archive Saves a couple of hours during releasing for Docker/Debian --- packaged_releases/README.md | 65 ++++---------- scripts/automation/release/packaged.py | 112 +++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 50 deletions(-) create mode 100644 scripts/automation/release/packaged.py diff --git a/packaged_releases/README.md b/packaged_releases/README.md index e6ff26e2bd5..6c0bd03d11d 100644 --- a/packaged_releases/README.md +++ b/packaged_releases/README.md @@ -3,67 +3,32 @@ Creating a Packaged Release This document provides instructions on creating a packaged release. -Throughout this document, `{VERSION}` refers to a semantic version for this packaged release (e.g. `0.1.5`). +Throughout this document, `{VERSION}` refers to a semantic version for this packaged release (e.g. `0.2.3`). -1 - Assemble module source code -------------------------------- - -To assemble the source code, visit [Releases](https://github.com/Azure/azure-cli/releases) and download the source code for each component (and version) that should be part of the packaged release. +1 - Create release archive +-------------------------- -Use the downloaded source code to create a directory with the structure outlined below. +We create a `.tar.gz` containing the source code for the packaged release. +This archive will be used as the basis for the Docker, Debian and Homebrew builds as they build from this source. -Make a directory with name `azure-cli_packaged_{VERSION}`: +Clone the repo afresh: ``` -$ mkdir azure-cli_packaged_{VERSION} +$ git clone https://github.com/azure/azure-cli ``` -**IMPORTANT** - The builds expect a certain folder structure. -Expected folder structure inside of `azure-cli_packaged_{VERSION}`: +Run the script to create the release archive from the 'scripts' folder at the repo root: ``` -. -|-- az.completion -|-- src -| |-- azure-cli -| |-- setup.py -| `-- etc... -| |-- azure-cli-core -| |-- setup.py -| `-- etc... -| |-- azure-cli-nspkg -| |-- setup.py -| `-- etc... -| |-- command_modules -| |-- -| |-- setup.py -| `-- etc... +$ cd scripts +$ python -m automation.release.packaged --version {VERSION} --components azure-cli=VERSION ... ``` -(A script may be available in the future to make this step more straightforward.) - -Notes: -- Only the packages that will be in the CLI should be included here; leave out 'optional' components unless there's a specific reason to include any extra components. -- Make sure the versions of components don't include the `+dev` suffix. Remove these if this is the case. - -APPLY ANY PATCHES: -Modify the file in question in the directory created from (You can use the `patch_*` files in `patches` subdirectory for this). - - -2 - Create release archive --------------------------- - -We create a `.tar.gz` containing the source code for the packaged release. -This archive will be used as the basis for the Docker, Debian and Homebrew builds as they build from this source. - -The archive should have the following name `azure-cli_packaged_{VERSION}.tar.gz`. - -Archive the assembled source code from above: +A full example: ``` -$ tar -cvzf azure-cli_packaged_{VERSION}.tar.gz azure-cli_packaged_{VERSION}/ +$ python -m automation.release.packaged --version 0.2.3 --components azure-cli=2.0.1 acs=2.0.1 appservice=0.1.1b6 batch=0.1.1b5 cloud=2.0.0 component=2.0.0 configure=2.0.1 container=0.1.1b4 core=2.0.1 documentdb=0.1.1b2 feedback=2.0.0 find=0.0.1b1 iot=0.1.1b3 keyvault=0.1.1b6 network=2.0.1 nspkg=2.0.0 profile=2.0.1 redis=0.1.1b3 resource=2.0.1 role=2.0.0 sql=0.1.1b6 storage=2.0.1 vm=2.0.1 ``` - -3 - Upload release archive +2 - Upload release archive -------------------------- The release archive should be uploaded to a storage account. @@ -86,13 +51,13 @@ Get the SHA256 checksum: $ shasum -a 256 azure-cli_packaged_{VERSION}.tar.gz ``` -4 - Build/Release for Debian, Docker, Homebrew +3 - Build/Release for Debian, Docker, Homebrew ---------------------------------------------- Follow the instructions in the `debian`, `docker` and `homebrew` subdirectories to create these releases. -5 - Modify HISTORY.md +4 - Modify HISTORY.md --------------------- Modify the packaged release history with release notes on this release and create a PR for this change. diff --git a/scripts/automation/release/packaged.py b/scripts/automation/release/packaged.py new file mode 100644 index 00000000000..ec9da1c1b63 --- /dev/null +++ b/scripts/automation/release/packaged.py @@ -0,0 +1,112 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +# pylint: disable=line-too-long + +from __future__ import print_function + +import argparse +import os +import sys +import tempfile +import tarfile +import shutil +from subprocess import check_call, check_output + +from .version_patcher import VersionPatcher +from ..utilities.path import get_all_module_paths, get_repo_root + +REPO_ROOT_DIR = get_repo_root() +COMPLETION_FILE = os.path.join(REPO_ROOT_DIR, 'packaged_releases', 'az.completion') +ARCHIVE_FILE_TMPL = 'azure-cli_packaged_{}' + +class Patch(object): + def __init__(self, src_of_patch, path_to_patch): + """ + - src: Relative path from the repo root + - dest: Relative path to file to patch in the packaged release + """ + self.src_of_patch = src_of_patch + self.path_to_patch = path_to_patch + + def apply(self, base_dir): + src = os.path.join(REPO_ROOT_DIR, self.src_of_patch) + dest = os.path.join(base_dir, self.path_to_patch) + shutil.copy(src, dest) + +PATCHES = [ + Patch(os.path.join('packaged_releases', 'patches', 'patch_pkg_util.py'), + os.path.join('src', 'azure-cli-core', 'azure', 'cli', 'core', '_pkg_util.py')), + Patch(os.path.join('packaged_releases', 'patches', 'patch_component_custom.py'), + os.path.join('src', 'command_modules', 'azure-cli-component', 'azure', 'cli', 'command_modules', 'component', 'custom.py')) + ] + +def error_exit(msg): + print('ERROR: '+msg, file=sys.stderr) + sys.exit(1) + +def _gen_tag(c_name, c_version): + return '{}-{}'.format(c_name, c_version) + +def _verified_tags(components): + available_tags = check_output(['git', 'tag'], cwd=REPO_ROOT_DIR).split() + for c_name, c_version in components: + t = _gen_tag(c_name, c_version) + if t not in available_tags: + print('Tag {} not found.'.format(t)) + return False + return True + +def create_packaged_archive(version, components, archive_dest=None, use_version_patch=True): + # Verify the components and versions by checking git tags + if not _verified_tags(components): + error_exit('Some components or versions are not valid.') + working_dir = tempfile.mkdtemp() + print('Using tmp directory {}'.format(working_dir)) + modules = {n: p for n, p in get_all_module_paths()} + cur_git_commitish = check_output(['git', 'rev-parse', 'HEAD'], cwd=REPO_ROOT_DIR).strip() + for c_name, c_version in components: + c_path = modules[c_name] + git_tag = _gen_tag(c_name, c_version) + check_call(['git', 'checkout', git_tag], cwd=REPO_ROOT_DIR) + patcher = VersionPatcher(use_version_patch, c_name, c_path) + patcher.patch() + sub_dir = 'command_modules' if 'command_modules' in c_path else '' + shutil.copytree(c_path, os.path.join(working_dir, 'src', sub_dir, c_name)) + patcher.unpatch() + check_call(['git', 'checkout', cur_git_commitish], cwd=REPO_ROOT_DIR) + # Add completion file + completion_dest = os.path.join(working_dir, 'az.completion') + shutil.copy(COMPLETION_FILE, completion_dest) + # Apply patches + for patch in PATCHES: + patch.apply(working_dir) + # Build archive + archive_filename = ARCHIVE_FILE_TMPL.format(version) + archive_dest = os.path.expanduser(archive_dest) or os.getcwd() + archive_path = os.path.join(archive_dest, archive_filename+'.tar.gz') + with tarfile.open(archive_path, 'w:gz') as tar: + tar.add(working_dir, arcname=archive_filename) + print("Archive saved to {}".format(archive_path)) + print("Done.") + +def _type_components_list(value): + c_name, c_version = value.split('=', 1) + if not c_name.startswith('azure-cli'): + c_name = 'azure-cli-' + c_name + return (c_name, c_version) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="Automated generation of the packaged release archive.") + parser.add_argument('--components', '-c', required=True, nargs='+', + help="Space separated list in 'component=version' format. (e.g. azure-cli=2.0.0 vm=2.0.0)", + type=_type_components_list) + parser.add_argument('--version', '-v', required=True, + help="The version to name the packaged release.") + parser.add_argument('--dest', '-d', + help="The destination directory to place the archive. Defaults to current directory.") + args = parser.parse_args() + create_packaged_archive(args.version, args.components, args.dest) From 188a78a9857ef96c798f41d94eb06d1a934d2e7e Mon Sep 17 00:00:00 2001 From: Derek Bekoe Date: Thu, 16 Mar 2017 15:55:02 -0700 Subject: [PATCH 2/2] Support JSON file & address style comments with pylint and pep8 --- packaged_releases/README.md | 13 +++++++ scripts/automation/release/packaged.py | 47 ++++++++++++++++++++------ 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/packaged_releases/README.md b/packaged_releases/README.md index 6c0bd03d11d..81d6e456658 100644 --- a/packaged_releases/README.md +++ b/packaged_releases/README.md @@ -24,6 +24,19 @@ $ python -m automation.release.packaged --version {VERSION} --components azure-c ``` A full example: +``` +$ cat ~/cli-components.json +{ + "azure-cli": "2.0.1", + "azure-cli-core": "2.0.1", + "azure-cli-component": "2.0.0", + "azure-cli-acs": "2.0.0" +} +$ python -m automation.release.packaged --version 0.2.3 -f ~/cli-components.json +``` + +OR + ``` $ python -m automation.release.packaged --version 0.2.3 --components azure-cli=2.0.1 acs=2.0.1 appservice=0.1.1b6 batch=0.1.1b5 cloud=2.0.0 component=2.0.0 configure=2.0.1 container=0.1.1b4 core=2.0.1 documentdb=0.1.1b2 feedback=2.0.0 find=0.0.1b1 iot=0.1.1b3 keyvault=0.1.1b6 network=2.0.1 nspkg=2.0.0 profile=2.0.1 redis=0.1.1b3 resource=2.0.1 role=2.0.0 sql=0.1.1b6 storage=2.0.1 vm=2.0.1 ``` diff --git a/scripts/automation/release/packaged.py b/scripts/automation/release/packaged.py index ec9da1c1b63..efd26883ef8 100644 --- a/scripts/automation/release/packaged.py +++ b/scripts/automation/release/packaged.py @@ -3,8 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -# pylint: disable=line-too-long - from __future__ import print_function import argparse @@ -13,6 +11,7 @@ import tempfile import tarfile import shutil +import json from subprocess import check_call, check_output from .version_patcher import VersionPatcher @@ -22,7 +21,8 @@ COMPLETION_FILE = os.path.join(REPO_ROOT_DIR, 'packaged_releases', 'az.completion') ARCHIVE_FILE_TMPL = 'azure-cli_packaged_{}' -class Patch(object): + +class Patch(object): # pylint: disable=too-few-public-methods def __init__(self, src_of_patch, path_to_patch): """ - src: Relative path from the repo root @@ -31,25 +31,30 @@ def __init__(self, src_of_patch, path_to_patch): self.src_of_patch = src_of_patch self.path_to_patch = path_to_patch - def apply(self, base_dir): + def apply(self, working_dir): src = os.path.join(REPO_ROOT_DIR, self.src_of_patch) - dest = os.path.join(base_dir, self.path_to_patch) + dest = os.path.join(working_dir, self.path_to_patch) shutil.copy(src, dest) + PATCHES = [ Patch(os.path.join('packaged_releases', 'patches', 'patch_pkg_util.py'), os.path.join('src', 'azure-cli-core', 'azure', 'cli', 'core', '_pkg_util.py')), Patch(os.path.join('packaged_releases', 'patches', 'patch_component_custom.py'), - os.path.join('src', 'command_modules', 'azure-cli-component', 'azure', 'cli', 'command_modules', 'component', 'custom.py')) + os.path.join('src', 'command_modules', 'azure-cli-component', 'azure', 'cli', + 'command_modules', 'component', 'custom.py')) ] + def error_exit(msg): print('ERROR: '+msg, file=sys.stderr) sys.exit(1) + def _gen_tag(c_name, c_version): return '{}-{}'.format(c_name, c_version) + def _verified_tags(components): available_tags = check_output(['git', 'tag'], cwd=REPO_ROOT_DIR).split() for c_name, c_version in components: @@ -59,6 +64,7 @@ def _verified_tags(components): return False return True + def create_packaged_archive(version, components, archive_dest=None, use_version_patch=True): # Verify the components and versions by checking git tags if not _verified_tags(components): @@ -92,21 +98,40 @@ def create_packaged_archive(version, components, archive_dest=None, use_version_ print("Archive saved to {}".format(archive_path)) print("Done.") + def _type_components_list(value): c_name, c_version = value.split('=', 1) if not c_name.startswith('azure-cli'): c_name = 'azure-cli-' + c_name return (c_name, c_version) + +def _type_json_file(value): + with open(os.path.expanduser(value)) as open_file: + data = json.load(open_file) + return [(k, data[k]) for k in data] + + if __name__ == '__main__': parser = argparse.ArgumentParser( description="Automated generation of the packaged release archive.") - parser.add_argument('--components', '-c', required=True, nargs='+', - help="Space separated list in 'component=version' format. (e.g. azure-cli=2.0.0 vm=2.0.0)", - type=_type_components_list) + + group = parser.add_mutually_exclusive_group(required=True) + + group.add_argument('--components', '-c', nargs='+', + help="Space separated list in 'component=version' format. " + "(e.g. azure-cli=2.0.0 vm=2.0.0)", + type=_type_components_list) + group.add_argument('--file-data', '-f', + help='Path to JSON file with commands in key/value format. ' + '(e.g. {"azure-cli":"2.0.0", ...})', + type=_type_json_file) parser.add_argument('--version', '-v', required=True, help="The version to name the packaged release.") parser.add_argument('--dest', '-d', - help="The destination directory to place the archive. Defaults to current directory.") + help="The destination directory to place the archive. " + "Defaults to current directory.") args = parser.parse_args() - create_packaged_archive(args.version, args.components, args.dest) + + components_list = args.components or args.file_data + create_packaged_archive(args.version, components_list, args.dest)