From ec0d2725db2b9691c971e33a76fb31d8f4dec392 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Wed, 30 Mar 2022 13:25:34 +0200 Subject: [PATCH 1/4] Add GH wokflow to build and release on PyPI (WIP) Add workflow with two jobs to build and publish on PyPI. The release job waits for the build job and uses a custom release environment, which can be configured to require review. To share the build artifacts between the jobs and to make them available for intermedieate review, they are stored using 'actions/upload-artifact' and 'actions/download-artifact'. https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts To upload the build artifacts to PyPI, the PyPA recommended 'pypa/gh-action-pypi-publish' is used. https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ **Caveat** The URL to grab the artifacts, e.g. for review, requires knowledge of action ID and artifact ID, and a login token (no special permissions). This makes it a bit cumbersome to fetch the artifacts with a script and compare them to a local build. https://docs.github.com/en/actions/managing-workflow-runs/downloading-workflow-artifacts Signed-off-by: Lukas Puehringer --- .github/workflows/cd.yml | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/cd.yml diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000000..b101b6ac8e --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,53 @@ +name: CD +concurrency: cd + +# Trigger workflow on release tag push +on: + push: + # TODO: Should we restrict to vX.Y.Z tags? + tags: v* + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout release tag + uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 + + - name: Set up Python + uses: actions/setup-python@0ebf233433c08fb9061af664d501c3f3ff0e9e20 + with: + python-version: '3.x' + + - name: Install build dependency + run: python3 -m pip install --upgrade pip build + + - name: Build binary wheel and source tarball + run: python3 -m build --sdist --wheel --outdir dist/ . + + - name: Store build artifacts for review and release + uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 + with: + name: build-artifacts + path: dist + + release-on-pypi: + name: Release on PyPI + runs-on: ubuntu-latest + needs: build + environment: release + steps: + - name: Fetch build artifacts + uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741 + with: + name: build-artifacts + path: dist + + - name: Publish binary wheel and source tarball on PyPI + uses: pypa/gh-action-pypi-publish@717ba43cfbb0387f6ce311b169a825772f54d295 + with: + user: __token__ + # TODO: Change to PyPI and update token + repository_url: https://test.pypi.org/legacy/ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} From 3a58abf42bae55cb433ec985d573520fd18af0ed Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 31 Mar 2022 15:16:27 +0200 Subject: [PATCH 2/4] Update auto release workflow (WIP) - Publish GitHub release candiate as part of the build process -> makes integration with local verify_release before review easier - Add --skip-pypi flag to verify_release script for pre-release check between build and release job (before review approval) - Tweak verify_release to work with my fork - Comment out release on (test) pypi step in release job, because verify_release currently only works with vX.Y.Z releases, and I don't want to push them to test pypi for ongoing prototyping, because we can't reuse a version later when we want to test an an actual release. - Add finalize GitHub release step to release job (does not work yet, error: release already exists) --- .github/workflows/cd.yml | 38 +++++++++++++++++++++++++++++--------- tuf/__init__.py | 2 +- verify_release | 36 +++++++++++++++++++++++------------- 3 files changed, 53 insertions(+), 23 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index b101b6ac8e..4b6f43ea11 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -11,6 +11,7 @@ jobs: build: name: Build runs-on: ubuntu-latest + if: ${{ github.ref_type == 'tag' }} steps: - name: Checkout release tag uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 @@ -26,14 +27,26 @@ jobs: - name: Build binary wheel and source tarball run: python3 -m build --sdist --wheel --outdir dist/ . - - name: Store build artifacts for review and release + - name: Publish GitHub release candiate + uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 + with: + name: ${{ github.ref_name }}-rc + tag_name: ${{ github.ref }} + # prerelease: true # <- verify_release script 'get_github_version' ignores pre-releases (and drafts) + body: "Release waiting for review..." + files: dist/* + + - name: Store build artifacts + # NOTE: The release job could download the assets from the GitHub release page, + # published in the previous step. But using the GitHub upload/download actions + # seems more robust as there is no need to compute download URLs. uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 with: name: build-artifacts path: dist - release-on-pypi: - name: Release on PyPI + release: + name: Release runs-on: ubuntu-latest needs: build environment: release @@ -43,11 +56,18 @@ jobs: with: name: build-artifacts path: dist + + # - name: Publish binary wheel and source tarball on PyPI + # uses: pypa/gh-action-pypi-publish@717ba43cfbb0387f6ce311b169a825772f54d295 + # with: + # user: __token__ + # # TODO: Change to PyPI and update token + # repository_url: https://test.pypi.org/legacy/ + # password: ${{ secrets.TEST_PYPI_API_TOKEN }} - - name: Publish binary wheel and source tarball on PyPI - uses: pypa/gh-action-pypi-publish@717ba43cfbb0387f6ce311b169a825772f54d295 + - name: Finalize GitHub release + uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 with: - user: __token__ - # TODO: Change to PyPI and update token - repository_url: https://test.pypi.org/legacy/ - password: ${{ secrets.TEST_PYPI_API_TOKEN }} + name: ${{ github.ref_name }} + tag_name: ${{ github.ref }} + body: "See CHANGELOG.md for details." diff --git a/tuf/__init__.py b/tuf/__init__.py index 322d316543..46c3293aa9 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -5,4 +5,4 @@ """ # This value is used in the requests user agent. -__version__ = "1.0.0" +__version__ = "1.0.1" diff --git a/verify_release b/verify_release index 6479720184..3942c20892 100755 --- a/verify_release +++ b/verify_release @@ -9,6 +9,7 @@ Builds a release from current commit and verifies that the release artifacts on GitHub and PyPI match the built release artifacts. """ +import argparse import json import os import subprocess @@ -27,8 +28,8 @@ except ImportError: # Project variables # Note that only these project artifacts are supported: # [f"{PYPI_PROJECT}-{VER}-none-any.whl", f"{PYPI_PROJECT}-{VER}.tar.gz"] -GITHUB_ORG = "theupdateframework" -GITHUB_PROJECT = "python-tuf" +GITHUB_ORG = "lukpueh" +GITHUB_PROJECT = "tuf" PYPI_PROJECT = "tuf" @@ -126,9 +127,17 @@ def progress(s: str) -> None: def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--skip-pypi", + action="store_true", + dest="skip_pypi", + help="Skip comparison with PyPI release.", + ) + args = parser.parse_args() + success = True with TemporaryDirectory() as build_dir: - progress("Building release") build_version = build(build_dir) finished(f"Built release {build_version}") @@ -143,16 +152,17 @@ def main() -> int: if github_version != build_version: finished(f"WARNING: GitHub latest version is {github_version}") - progress("Checking PyPI latest version") - pypi_version = get_pypi_pip_version() - if pypi_version != build_version: - finished(f"WARNING: PyPI latest version is {pypi_version}") - - progress("Downloading release from PyPI") - if not verify_pypi_release(build_version, build_dir): - # This is expected while build is not reproducible - finished("ERROR: PyPI artifacts do not match built release") - success = False + if not args.skip_pypi: + progress("Checking PyPI latest version") + pypi_version = get_pypi_pip_version() + if pypi_version != build_version: + finished(f"WARNING: PyPI latest version is {pypi_version}") + + progress("Downloading release from PyPI") + if not verify_pypi_release(build_version, build_dir): + # This is expected while build is not reproducible + finished("ERROR: PyPI artifacts do not match built release") + success = False progress("Downloading release from GitHub") if not verify_github_release(build_version, build_dir): From 76c0aa99a503cb7c297387427878f3bebfa938f1 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Fri, 1 Apr 2022 12:22:02 +0200 Subject: [PATCH 3/4] debug cd.yml --- .github/workflows/cd.yml | 77 ++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 4b6f43ea11..5f47ee1ff5 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -5,45 +5,48 @@ concurrency: cd on: push: # TODO: Should we restrict to vX.Y.Z tags? - tags: v* + # tags: v* jobs: build: name: Build runs-on: ubuntu-latest - if: ${{ github.ref_type == 'tag' }} + # if: ${{ github.ref_type == 'tag' }} + outputs: + release_id: ${{ steps.gh-release.outputs.id }} steps: - name: Checkout release tag uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 - - name: Set up Python - uses: actions/setup-python@0ebf233433c08fb9061af664d501c3f3ff0e9e20 - with: - python-version: '3.x' + # - name: Set up Python + # uses: actions/setup-python@0ebf233433c08fb9061af664d501c3f3ff0e9e20 + # with: + # python-version: '3.x' - - name: Install build dependency - run: python3 -m pip install --upgrade pip build + # - name: Install build dependency + # run: python3 -m pip install --upgrade pip build - - name: Build binary wheel and source tarball - run: python3 -m build --sdist --wheel --outdir dist/ . + # - name: Build binary wheel and source tarball + # run: python3 -m build --sdist --wheel --outdir dist/ . - - name: Publish GitHub release candiate + - id: gh-release + name: Publish GitHub release candiate uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 with: name: ${{ github.ref_name }}-rc tag_name: ${{ github.ref }} # prerelease: true # <- verify_release script 'get_github_version' ignores pre-releases (and drafts) body: "Release waiting for review..." - files: dist/* + # files: dist/* - - name: Store build artifacts - # NOTE: The release job could download the assets from the GitHub release page, - # published in the previous step. But using the GitHub upload/download actions - # seems more robust as there is no need to compute download URLs. - uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 - with: - name: build-artifacts - path: dist + # - name: Store build artifacts + # # NOTE: The release job could download the assets from the GitHub release page, + # # published in the previous step. But using the GitHub upload/download actions + # # seems more robust as there is no need to compute download URLs. + # uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 + # with: + # name: build-artifacts + # path: dist release: name: Release @@ -51,12 +54,12 @@ jobs: needs: build environment: release steps: - - name: Fetch build artifacts - uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741 - with: - name: build-artifacts - path: dist - + # - name: Fetch build artifacts + # uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741 + # with: + # name: build-artifacts + # path: dist + # - name: Publish binary wheel and source tarball on PyPI # uses: pypa/gh-action-pypi-publish@717ba43cfbb0387f6ce311b169a825772f54d295 # with: @@ -66,8 +69,22 @@ jobs: # password: ${{ secrets.TEST_PYPI_API_TOKEN }} - name: Finalize GitHub release - uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 + uses: actions/github-script@9ac08808f993958e9de277fe43a64532a609130e with: - name: ${{ github.ref_name }} - tag_name: ${{ github.ref }} - body: "See CHANGELOG.md for details." + script: | + console.log(${{needs.build.outputs.release_id}}) +# await github.rest.repos.updateRelease({ + + +# }) + +# octokit.rest.repos.createRelease({ +# owner, +# repo, +# tag_name, +# }); + + +# name: ${{ github.ref_name }} +# tag_name: ${{ github.ref }} +# body: "See CHANGELOG.md for details." From e61839e6711eab6cf106da9f4ab2f1b4965c24c0 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Fri, 1 Apr 2022 12:26:01 +0200 Subject: [PATCH 4/4] debug cd.yml --- .github/workflows/cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 5f47ee1ff5..ef31f6018a 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -71,8 +71,8 @@ jobs: - name: Finalize GitHub release uses: actions/github-script@9ac08808f993958e9de277fe43a64532a609130e with: - script: | - console.log(${{needs.build.outputs.release_id}}) + script: | + console.log(${{needs.build.outputs.release_id}}) # await github.rest.repos.updateRelease({