From 24ef2f9502babf737304753ecfdefd5538fa58c2 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Tue, 13 Sep 2022 11:58:57 -0500 Subject: [PATCH 1/5] Migrate Python Sickbay PostCommit job from Jenkins to GA --- .../job-postcommit-python-sickbay.yml | 65 +++++++++++++++++++ .../job_PostCommit_Python_Sickbay.groovy | 48 -------------- CI.md | 7 ++ 3 files changed, 72 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/job-postcommit-python-sickbay.yml delete mode 100644 .test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy diff --git a/.github/workflows/job-postcommit-python-sickbay.yml b/.github/workflows/job-postcommit-python-sickbay.yml new file mode 100644 index 000000000000..5c0ac29ecf85 --- /dev/null +++ b/.github/workflows/job-postcommit-python-sickbay.yml @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Python PostCommit sickbay tests. + +name: Python Sickbay + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + +jobs: + set-properties: + runs-on: self-hosted + outputs: + properties: ${{ steps.test-properties.outputs.properties }} + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - id: test-properties + uses: ./.github/actions/setup-default-test-properties + + sickbay: + needs: set-properties + name: Python Sickbay + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Set python version + run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV + - name: Run :pythonSickbayPostCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :python${{ env.PYTHON_VERSION }}SickbayPostCommit diff --git a/.test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy b/.test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy deleted file mode 100644 index 9789121ee77f..000000000000 --- a/.test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import CommonJobProperties as commonJobProperties -import PostcommitJobBuilder - -import static PythonTestProperties.ALL_SUPPORTED_VERSIONS - -// This job defines the Python postcommit sickbay tests. -ALL_SUPPORTED_VERSIONS.each { pythonVersion -> - def versionSuffix = pythonVersion.replace('.', '') - PostcommitJobBuilder.postCommitJob("beam_PostCommit_Sickbay_Python${versionSuffix}", - "Run Python ${pythonVersion} PostCommit Sickbay", - "Python${versionSuffix}_PC_Sickbay(\"Run Python ${pythonVersion} PostCommit Sickbay\")", this) { - description('Runs Python postcommit sickbay tests using Python ${pythonVersion}.') - - // Set common parameters. - commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180) - - publishers { - archiveJunit('**/pytest*.xml') - } - - // Execute shell command to test Python SDK. - steps { - gradle { - rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(":python${versionSuffix}SickbayPostCommit") - commonJobProperties.setGradleSwitches(delegate) - } - } - } -} \ No newline at end of file diff --git a/CI.md b/CI.md index ed75a4dd4433..224dfd0a8b01 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,13 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PostCommit Workflows + +#### Python Sickbay - [job-postcommit-python-sickbay.yml](.github/workflows/job-postcommit-python-sickbay.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|----------------|---------------------------|------------------|-----------------------|---------------|--------------------------| +| Python Sickbay | Runs Python Sickbay Tests | - | Yes | Yes | - | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. From 5716ca8c31a1e0046ca91472b0aafad220f57567 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Tue, 13 Sep 2022 12:03:27 -0500 Subject: [PATCH 2/5] Remove Job reference from README Jenkins file --- .test-infra/jenkins/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 2c252a4d55aa..2aa25d255eeb 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -126,9 +126,6 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_Python_VR_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/) | `Run Python Spark ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark) | | beam_PostCommit_Python37 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python37), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python37_PR/) | `Run Python 3.7 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python37/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python37) | | beam_PostCommit_Python38 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python38), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python38_PR/) | `Run Python 3.8 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python38/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python38) | -| beam_PostCommit_Sickbay_Python36 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python36_PR/) | `Run Python 3.6 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36) | -| beam_PostCommit_Sickbay_Python37 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python37_PR/) | `Run Python 3.7 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37) | -| beam_PostCommit_Sickbay_Python38 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python38_PR/) | `Run Python 3.8 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38) | | beam_PostCommit_SQL | [cron](https://ci-beam.apache.org/job/beam_PostCommit_SQL/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SQL_PR/) | `Run SQL PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_SQL/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_SQL) | | beam_PostCommit_Website_Publish | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish/) | N/A | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish) | | beam_PostCommit_Website_Test | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test/) | `Run Full Website Test` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test) | From cfb124a6bfaba8e8a1b081cc7f7af320e338f73a Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Tue, 20 Sep 2022 11:23:04 -0500 Subject: [PATCH 3/5] Add permissions: read-all and minor updates to workflows --- .github/workflows/job-postcommit-python-sickbay.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/job-postcommit-python-sickbay.yml b/.github/workflows/job-postcommit-python-sickbay.yml index 5c0ac29ecf85..a584f6dda16a 100644 --- a/.github/workflows/job-postcommit-python-sickbay.yml +++ b/.github/workflows/job-postcommit-python-sickbay.yml @@ -25,7 +25,8 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] +permissions: read-all jobs: set-properties: @@ -34,7 +35,7 @@ jobs: properties: ${{ steps.test-properties.outputs.properties }} steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive @@ -51,12 +52,14 @@ jobs: version: ${{fromJson(needs.set-properties.outputs.properties).PythonTestProperties.ALL_SUPPORTED_VERSIONS}} steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false - name: Set python version run: echo "PYTHON_VERSION=$(echo ${{ matrix.version }} | sed -e 's/\.//g')" >> $GITHUB_ENV - name: Run :pythonSickbayPostCommit From ed183fb7eb09156000c83f727f88d2f4ef4322d7 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Fri, 23 Sep 2022 13:29:42 -0500 Subject: [PATCH 4/5] Change from self-hosted to [self-hosted, ubuntu-20.04] --- .github/workflows/job-postcommit-python-sickbay.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/job-postcommit-python-sickbay.yml b/.github/workflows/job-postcommit-python-sickbay.yml index a584f6dda16a..0b97f181e594 100644 --- a/.github/workflows/job-postcommit-python-sickbay.yml +++ b/.github/workflows/job-postcommit-python-sickbay.yml @@ -30,7 +30,7 @@ permissions: read-all jobs: set-properties: - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] outputs: properties: ${{ steps.test-properties.outputs.properties }} steps: @@ -45,7 +45,7 @@ jobs: sickbay: needs: set-properties name: Python Sickbay - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] strategy: fail-fast: false matrix: From f406b6eca745655e8b31e11ee815ee15b2076427 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Thu, 29 Sep 2022 16:56:45 -0500 Subject: [PATCH 5/5] Remove submodules:recursive --- .github/workflows/job-postcommit-python-sickbay.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/job-postcommit-python-sickbay.yml b/.github/workflows/job-postcommit-python-sickbay.yml index 0b97f181e594..3bc3f5e4db51 100644 --- a/.github/workflows/job-postcommit-python-sickbay.yml +++ b/.github/workflows/job-postcommit-python-sickbay.yml @@ -38,7 +38,6 @@ jobs: uses: actions/checkout@v3 with: persist-credentials: false - submodules: recursive - id: test-properties uses: ./.github/actions/setup-default-test-properties @@ -55,7 +54,6 @@ jobs: uses: actions/checkout@v3 with: persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: