From 03953d8d02666df1f3dfe71c17876097ddc7a118 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Wed, 28 Sep 2022 13:42:13 -0500 Subject: [PATCH 01/10] Migrate precommit website and typescript workflows from Jenkins to GA --- .../workflows/job-precommit-typescript.yml | 53 +++++++++++++++++ .../job-precommit-website-stage-gcs.yml | 59 +++++++++++++++++++ .github/workflows/job-precommit-website.yml | 59 +++++++++++++++++++ .test-infra/jenkins/README.md | 3 - .../jenkins/job_PreCommit_Typescript.groovy | 29 --------- .../jenkins/job_PreCommit_Website.groovy | 27 --------- .../job_PreCommit_Website_Stage_GCS.groovy | 31 ---------- CI.md | 8 ++- website/Dockerfile | 22 +++++++ website/build.gradle | 53 ++++++++++++++--- 10 files changed, 242 insertions(+), 102 deletions(-) create mode 100644 .github/workflows/job-precommit-typescript.yml create mode 100644 .github/workflows/job-precommit-website-stage-gcs.yml create mode 100644 .github/workflows/job-precommit-website.yml delete mode 100644 .test-infra/jenkins/job_PreCommit_Typescript.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Website.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy diff --git a/.github/workflows/job-precommit-typescript.yml b/.github/workflows/job-precommit-typescript.yml new file mode 100644 index 000000000000..d0e87c866816 --- /dev/null +++ b/.github/workflows/job-precommit-typescript.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Typescript PreCommit. + +name: Typescript + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/python/apache_beam/runners/interactive/extensions/**'] +permissions: read-all + +jobs: + typescript: + name: Typescript + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run :typescriptPreCommit + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :typescriptPreCommit diff --git a/.github/workflows/job-precommit-website-stage-gcs.yml b/.github/workflows/job-precommit-website-stage-gcs.yml new file mode 100644 index 000000000000..e700d2e3c870 --- /dev/null +++ b/.github/workflows/job-precommit-website-stage-gcs.yml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run the Website in Stage GCS. + +name: Website Stage GCS + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['website/**'] +permissions: read-all + +jobs: + website-stage-gcs: + name: Run Website Stage in GCS + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run :website:stageWebsite + uses: ./.github/actions/gradle-command-self-hosted-action + env: + ghprbPullId: ${{github.event.pull_request.number}} + ghprbPullAuthorLogin: ${{github.actor}} + ghprbSourceBranch: ${{github.ref_name}} + ghBranch: ${{github.head_ref}} + repositoryUrl: ${{github.repository}} + with: + gradle-command: :website:stageWebsite diff --git a/.github/workflows/job-precommit-website.yml b/.github/workflows/job-precommit-website.yml new file mode 100644 index 000000000000..7628ba8ac618 --- /dev/null +++ b/.github/workflows/job-precommit-website.yml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Website PreCommit. + +name: Website + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request_target: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['website/**'] +permissions: read-all + +jobs: + website: + name: website + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + with: + requires-go-18: false + - name: Run :websitePreCommit + uses: ./.github/actions/gradle-command-self-hosted-action + env: + ghprbPullId: ${{github.event.pull_request.number}} + ghprbPullAuthorLogin: ${{github.actor}} + ghprbSourceBranch: ${{github.ref_name}} + ghBranch: ${{github.head_ref}} + repositoryUrl: ${{github.repository}} + with: + gradle-command: :websitePreCommit diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 08b9665a92ba..6274c765755e 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -52,10 +52,7 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PreCommit_Spotless | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Phrase/) | `Run Spotless PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Spotless_Cron) | | beam_PreCommit_SQL | [commit](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Phrase/) | `Run SQL PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Cron/) | | beam_PreCommit_SQL_Java11 | [commit](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Phrase/) | `Run SQL_Java11 PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_SQL_Java11_Cron/) | -| beam_PreCommit_Website | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Website_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Website_Phrase/) | `Run Website PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron) | -| beam_PreCommit_Website_Stage_GCS | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Phrase/) | `Run Website_Stage_GCS PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Website_Stage_GCS_Cron) | | beam_PreCommit_Whitespace | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Phrase/) | `Run Whitespace PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron) | -| beam_PreCommit_Typescript | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Phrase/) | `Run Typescript PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron) | ### PostCommit Jobs diff --git a/.test-infra/jenkins/job_PreCommit_Typescript.groovy b/.test-infra/jenkins/job_PreCommit_Typescript.groovy deleted file mode 100644 index 29cb93ede8b6..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Typescript.groovy +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Typescript', - gradleTask: ':typescriptPreCommit', - triggerPathPatterns: [ - '^sdks/python/apache_beam/runners/interactive/extensions/.*$', - ] - ) -builder.build() diff --git a/.test-infra/jenkins/job_PreCommit_Website.groovy b/.test-infra/jenkins/job_PreCommit_Website.groovy deleted file mode 100644 index 73014819ed00..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Website.groovy +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Website', - gradleTask: ':websitePreCommit', - triggerPathPatterns: ['^website/.*$']) -builder.build() - diff --git a/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy b/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy deleted file mode 100644 index e2f7202d14eb..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Website_Stage_GCS.groovy +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Website_Stage_GCS', - gradleTask: ':website:stageWebsite', - triggerPathPatterns: ['^website/.*$']) -builder.build { - publishers { - buildDescription(/Website published to (http:\/\/.+\/index.html)/) - } -} - diff --git a/CI.md b/CI.md index ee911584f7f0..7b002b51d7ea 100644 --- a/CI.md +++ b/CI.md @@ -133,9 +133,11 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | PreCommit | Yes | Yes | Yes | Yes | ### PreCommit Workflows -| Workflow | Description | Requires GCP Credentials | -|----------------------------------------------------------------------------------|-------------------------|---------------------------| -| [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No | +| Workflow | Description | Requires GCP Credentials | +|----------------------------------------------------------------------------------------------|---------------------------|--------------------------| +| [job-precommit-typescript.yml](.github/workflows/job-precommit-typescript.yml) | Runs Typescript PreCommit | No | +| [job-precommit-website.yml](.github/workflows/job-precommit-website.yml) | Runs Website PreCommit | No | +| [job-precommit-website-stage-gcs.yml](.github/workflows/job-precommit-website-stage-gcs.yml) | Runs Website in Stage GCS | Yes | ### PostCommit Workflows | Workflow | Description | Requires GCP Credentials | diff --git a/website/Dockerfile b/website/Dockerfile index 4c4847481045..8b709474cbe2 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -69,4 +69,26 @@ RUN HUGOHOME="$(mktemp -d)" \ && chmod +x /usr/local/bin/hugo \ && rm -r "${HUGOHOME}" + +# Install gcloud +# Make sure that your operating system meets the requirements + +# Add the gcloud CLI distribution URI as a package source +RUN apt-get install apt-transport-https ca-certificates gnupg +RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list +# Import the Google Cloud public key +RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - +# Update and install the gcloud CLI +RUN apt-get update && apt-get install google-cloud-cli +# #Installing and replacing default python version +RUN apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev \ + && curl -O https://www.python.org/ftp/python/3.8.2/Python-3.8.2.tar.xz \ + && tar -xf Python-3.8.2.tar.xz \ + && cd Python-3.8.2 && ./configure --enable-optimizations \ + && make -j 4 && make altinstall + +RUN update-alternatives --install /usr/bin/python python /usr/local/bin/python3.8 1\ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.5 2 \ + && update-alternatives --set python $(update-alternatives --list python | grep python3.8) + WORKDIR /opt/ diff --git a/website/build.gradle b/website/build.gradle index 6aa88a2c7feb..0c6a9a8f490a 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -80,10 +80,10 @@ task createDockerContainer(type: Exec) { } else { // Otherwise: run as current user, so as to not require sudo to clean up // build/ directory. - extraOptions += " -u \$(id -u):\$(id -g)" + extraOptions += " -u 0:0" } commandLine '/bin/bash', '-c', - "docker create -v $project.rootDir:$dockerWorkDir $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'" + "docker create $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'" } } @@ -115,9 +115,35 @@ task buildCodeSamples(type: Exec) { "${->startDockerContainer.containerId()}", 'yarn', 'build_code_samples' } +task cloneRepository(type: Exec) { + commandLine 'docker', 'exec', + "${->startDockerContainer.containerId()}", "git", "clone", "-b", System.getenv('ghBranch'), + "https://github.com/${System.getenv('repositoryUrl')}", "." +} + +task setupVirtualenvWithDependenciesInDocker { + doLast { + exec { + commandLine 'docker', 'exec', + "${->startDockerContainer.containerId()}", "python${project.ext.pythonVersion}", + '-m', 'venv', "${envdir}" + } + exec { + commandLine 'docker', 'exec', + "${->startDockerContainer.containerId()}", '/bin/bash', '-c', + """. ${envdir}/bin/activate && pip install beautifulsoup4""" + } + } +} + +initGitSubmodules.mustRunAfter cloneRepository +installDependencies.mustRunAfter cloneRepository +buildCodeSamples.mustRunAfter cloneRepository + task setupDockerContainer(type: Exec) { dependsOn startDockerContainer - finalizedBy initGitSubmodules, installDependencies, buildCodeSamples + finalizedBy cloneRepository, initGitSubmodules, installDependencies, buildCodeSamples, + setupVirtualenvWithDependenciesInDocker ext.containerId = { return startDockerContainer.containerId() } @@ -147,7 +173,6 @@ def createBuildTask = { BuildTaskConfiguration config = it as BuildTaskConfiguration task "build${config.name}Website" (type:Exec) { dependsOn setupDockerContainer - finalizedBy stopAndRemoveDockerContainer def configs = "$dockerSourceDir/site/config.toml" def baseUrlFlag = config.baseUrl ? "--baseURL /${config.baseUrl}" : "" @@ -230,13 +255,13 @@ def getBranchRepo() { return "apache/beam/blob/master" } -def buildContentDir(name) { - "${project.rootDir}/build/website/generated-${name.toLowerCase()}-content" +def buildContentDir(name, dir = project.rootDir) { + "${dir}/build/website/generated-${name.toLowerCase()}-content" } task serveWebsite(type: Exec) { dependsOn setupDockerContainer - finalizedBy stopAndRemoveDockerContainer +// finalizedBy stopAndRemoveDockerContainer commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', """cd $dockerSourceDir && \ @@ -263,6 +288,7 @@ task preCommit { // Creates a new commit on asf-site branch task commitWebsite { + finalizedBy stopAndRemoveDockerContainer doLast { assert grgit : "Cannot commit website outside of git repository" assert file("${buildContentDir('apache')}/index.html").exists() @@ -346,6 +372,7 @@ publishWebsite.dependsOn commitWebsite * ./gradlew :website:stageWebsite -PwebsiteBucket=foo */ task stageWebsite { + finalizedBy stopAndRemoveDockerContainer doLast { def baseUrl = getBaseUrl() assert baseUrl : 'Website staging requires a valid baseUrl' @@ -353,10 +380,18 @@ task stageWebsite { def gcs_path = "gs://${gcs_bucket}/${baseUrl}" // Fixup the links to index.html files - shell ". ${envdir}/bin/activate && python append_index_html_to_internal_links.py ${buildContentDir('gcs')}" + exec { + commandLine 'docker', 'exec', '--workdir', "$dockerWorkDir/website", + "${->startDockerContainer.containerId()}", '/bin/bash', '-c', + """. ${envdir}/bin/activate && python append_index_html_to_internal_links.py ${buildContentDir('gcs', ".")} """ + } // Copy the build website to GCS - shell "gsutil -m rsync -r -d ${buildContentDir('gcs')} ${gcs_path}" + exec { + commandLine 'docker', 'exec', '--workdir', "$dockerWorkDir", + "${->startDockerContainer.containerId()}", '/bin/bash', '-c', + """gsutil -m rsync -r -d ${buildContentDir('gcs', ".")} ${gcs_path}""" + } println "Website published to http://${gcs_bucket}." + "storage.googleapis.com/${baseUrl}/index.html" From d0d9bc462779b90445bef6633d9e6f05185b02cd Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Wed, 28 Sep 2022 16:30:13 -0500 Subject: [PATCH 02/10] Set env var defaults and non-required envs in workflow --- .github/workflows/job-precommit-typescript.yml | 3 +++ .github/workflows/job-precommit-website-stage-gcs.yml | 3 +++ .github/workflows/job-precommit-website.yml | 3 +++ website/build.gradle | 8 ++++++-- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/job-precommit-typescript.yml b/.github/workflows/job-precommit-typescript.yml index d0e87c866816..d004274065d3 100644 --- a/.github/workflows/job-precommit-typescript.yml +++ b/.github/workflows/job-precommit-typescript.yml @@ -47,6 +47,9 @@ jobs: uses: ./.github/actions/setup-self-hosted-action with: requires-go-18: false + requires-py-37: false + requires-py-38: false + requires-py-39: false - name: Run :typescriptPreCommit uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-precommit-website-stage-gcs.yml b/.github/workflows/job-precommit-website-stage-gcs.yml index e700d2e3c870..0ce70c9440d0 100644 --- a/.github/workflows/job-precommit-website-stage-gcs.yml +++ b/.github/workflows/job-precommit-website-stage-gcs.yml @@ -47,6 +47,9 @@ jobs: uses: ./.github/actions/setup-self-hosted-action with: requires-go-18: false + requires-py-37: false + requires-py-38: false + requires-py-39: false - name: Run :website:stageWebsite uses: ./.github/actions/gradle-command-self-hosted-action env: diff --git a/.github/workflows/job-precommit-website.yml b/.github/workflows/job-precommit-website.yml index 7628ba8ac618..8862b9dfbd70 100644 --- a/.github/workflows/job-precommit-website.yml +++ b/.github/workflows/job-precommit-website.yml @@ -47,6 +47,9 @@ jobs: uses: ./.github/actions/setup-self-hosted-action with: requires-go-18: false + requires-py-37: false + requires-py-38: false + requires-py-39: false - name: Run :websitePreCommit uses: ./.github/actions/gradle-command-self-hosted-action env: diff --git a/website/build.gradle b/website/build.gradle index 0c6a9a8f490a..24fcf1b2525a 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -116,9 +116,13 @@ task buildCodeSamples(type: Exec) { } task cloneRepository(type: Exec) { + def branch = System.getenv('ghBranch') ? System.getenv('ghBranch'): + grgit ? grgit.branch.current().getName(): 'master' + def repositoryUrl = System.getenv('repositoryUrl') ? "https://github.com/${System.getenv('repositoryUrl')}" : + "https://github.com/apache/beam" + commandLine 'docker', 'exec', - "${->startDockerContainer.containerId()}", "git", "clone", "-b", System.getenv('ghBranch'), - "https://github.com/${System.getenv('repositoryUrl')}", "." + "${->startDockerContainer.containerId()}", "git", "clone", "-b", branch, repositoryUrl, "." } task setupVirtualenvWithDependenciesInDocker { From 2863a91fbf1d2c3f9acc836b9b5d19c29589edd4 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Wed, 28 Sep 2022 17:08:37 -0500 Subject: [PATCH 03/10] Remove optional python environment for typescript workflow --- .github/workflows/job-precommit-typescript.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/job-precommit-typescript.yml b/.github/workflows/job-precommit-typescript.yml index d004274065d3..d0e87c866816 100644 --- a/.github/workflows/job-precommit-typescript.yml +++ b/.github/workflows/job-precommit-typescript.yml @@ -47,9 +47,6 @@ jobs: uses: ./.github/actions/setup-self-hosted-action with: requires-go-18: false - requires-py-37: false - requires-py-38: false - requires-py-39: false - name: Run :typescriptPreCommit uses: ./.github/actions/gradle-command-self-hosted-action with: From ffc52fdb58e0e4207265c1c34e48e3dc1a66eecf Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Wed, 28 Sep 2022 17:52:30 -0500 Subject: [PATCH 04/10] Remove optional python environment for website gcs stage workflow --- .github/workflows/job-precommit-website-stage-gcs.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/job-precommit-website-stage-gcs.yml b/.github/workflows/job-precommit-website-stage-gcs.yml index 0ce70c9440d0..e700d2e3c870 100644 --- a/.github/workflows/job-precommit-website-stage-gcs.yml +++ b/.github/workflows/job-precommit-website-stage-gcs.yml @@ -47,9 +47,6 @@ jobs: uses: ./.github/actions/setup-self-hosted-action with: requires-go-18: false - requires-py-37: false - requires-py-38: false - requires-py-39: false - name: Run :website:stageWebsite uses: ./.github/actions/gradle-command-self-hosted-action env: From 967857318e9a9d54ca14e59bf611d4bdcb10c65a Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Thu, 29 Sep 2022 16:25:39 -0500 Subject: [PATCH 05/10] Remove submodules:recursive --- .github/workflows/job-precommit-typescript.yml | 1 - .github/workflows/job-precommit-website-stage-gcs.yml | 1 - .github/workflows/job-precommit-website.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/job-precommit-typescript.yml b/.github/workflows/job-precommit-typescript.yml index d0e87c866816..578515825b5b 100644 --- a/.github/workflows/job-precommit-typescript.yml +++ b/.github/workflows/job-precommit-typescript.yml @@ -42,7 +42,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-precommit-website-stage-gcs.yml b/.github/workflows/job-precommit-website-stage-gcs.yml index e700d2e3c870..9b15b3afe292 100644 --- a/.github/workflows/job-precommit-website-stage-gcs.yml +++ b/.github/workflows/job-precommit-website-stage-gcs.yml @@ -42,7 +42,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-precommit-website.yml b/.github/workflows/job-precommit-website.yml index 8862b9dfbd70..6c0dc139703f 100644 --- a/.github/workflows/job-precommit-website.yml +++ b/.github/workflows/job-precommit-website.yml @@ -42,7 +42,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: From ee2c35fdde875d7e802d77f725b06a5db72b37c8 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Tue, 11 Oct 2022 14:38:38 -0500 Subject: [PATCH 06/10] Add TODO for renaming env vars --- .github/workflows/job-precommit-website-stage-gcs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/job-precommit-website-stage-gcs.yml b/.github/workflows/job-precommit-website-stage-gcs.yml index 9b15b3afe292..10d3d134a31b 100644 --- a/.github/workflows/job-precommit-website-stage-gcs.yml +++ b/.github/workflows/job-precommit-website-stage-gcs.yml @@ -48,7 +48,7 @@ jobs: requires-go-18: false - name: Run :website:stageWebsite uses: ./.github/actions/gradle-command-self-hosted-action - env: + env: # TODO(https://github.com/apache/beam/issues/23580) Rename ghprb env vars to a more GHA concept. ghprbPullId: ${{github.event.pull_request.number}} ghprbPullAuthorLogin: ${{github.actor}} ghprbSourceBranch: ${{github.ref_name}} From 774a18879ad10f7f81f2df1cfe967ddd5a0b5a76 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Tue, 11 Oct 2022 14:43:59 -0500 Subject: [PATCH 07/10] Add TODO for renaming env vars --- .github/workflows/job-precommit-website.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/job-precommit-website.yml b/.github/workflows/job-precommit-website.yml index 6c0dc139703f..bdcfffe98e95 100644 --- a/.github/workflows/job-precommit-website.yml +++ b/.github/workflows/job-precommit-website.yml @@ -51,7 +51,7 @@ jobs: requires-py-39: false - name: Run :websitePreCommit uses: ./.github/actions/gradle-command-self-hosted-action - env: + env: # TODO(https://github.com/apache/beam/issues/23580) Rename ghprb env vars to a more GHA concept. ghprbPullId: ${{github.event.pull_request.number}} ghprbPullAuthorLogin: ${{github.actor}} ghprbSourceBranch: ${{github.ref_name}} From ba6f16b5dca69d8ca9ce30025be8ed0489db4cb2 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Fri, 14 Oct 2022 14:20:47 -0500 Subject: [PATCH 08/10] Add dedicated user to have permissions on folder opt --- website/Dockerfile | 3 +++ website/build.gradle | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/website/Dockerfile b/website/Dockerfile index 8b709474cbe2..fb9f8f25887a 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -91,4 +91,7 @@ RUN update-alternatives --install /usr/bin/python python /usr/local/bin/python3. && update-alternatives --install /usr/bin/python python /usr/bin/python3.5 2 \ && update-alternatives --set python $(update-alternatives --list python | grep python3.8) +# Add dedicated user to have permissions on opt dir +RUN useradd -m actions && usermod -aG sudo actions && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +RUN chown -R actions:actions /opt/ WORKDIR /opt/ diff --git a/website/build.gradle b/website/build.gradle index 24fcf1b2525a..a5c147214b95 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -80,7 +80,7 @@ task createDockerContainer(type: Exec) { } else { // Otherwise: run as current user, so as to not require sudo to clean up // build/ directory. - extraOptions += " -u 0:0" + extraOptions += " -u actions" } commandLine '/bin/bash', '-c', "docker create $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'" From da806ac93547f3be0f1f071f2cb54ecd49a319bb Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Mon, 17 Oct 2022 12:34:04 -0500 Subject: [PATCH 09/10] Remove commented line for stopAndRemoveDockerContainer --- website/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/website/build.gradle b/website/build.gradle index a5c147214b95..8eab3a05d031 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -265,7 +265,6 @@ def buildContentDir(name, dir = project.rootDir) { task serveWebsite(type: Exec) { dependsOn setupDockerContainer -// finalizedBy stopAndRemoveDockerContainer commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', """cd $dockerSourceDir && \ From 763c9d0dff5e5484f3646b7c10c95ed08333eb94 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Fri, 9 Dec 2022 17:04:58 -0600 Subject: [PATCH 10/10] Setup service account with action --- .github/workflows/job-precommit-website-stage-gcs.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/job-precommit-website-stage-gcs.yml b/.github/workflows/job-precommit-website-stage-gcs.yml index 10d3d134a31b..64035a36ea4a 100644 --- a/.github/workflows/job-precommit-website-stage-gcs.yml +++ b/.github/workflows/job-precommit-website-stage-gcs.yml @@ -37,6 +37,14 @@ jobs: name: Run Website Stage in GCS runs-on: [self-hosted, ubuntu-20.04] steps: + - name: Checking out composite to setup service account + uses: actions/checkout@v3 + with: + ref: master + - name: Setup service account + uses: ./.github/actions/activate-service-account + with: + GCP_SERVICE_ACCOUNT: ${{secrets.GCP_SELF_HOSTED_SA_KEY}} - name: Checkout code uses: actions/checkout@v3 with: