From 947282d98c87364c4adeb7276e0e0726b89aa4a2 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Wed, 16 Jul 2025 10:02:05 -0400 Subject: [PATCH 1/4] Fix Java8 tests to have gradle run on Java11 --- ..._ValidatesRunner_Dataflow_JavaVersions.yml | 11 ++-------- ...va_ValidatesRunner_Direct_JavaVersions.yml | 11 ++-------- ...ommit_Java_ValidatesRunner_Flink_Java8.yml | 14 ++---------- ...ommit_Java_ValidatesRunner_Spark_Java8.yml | 14 ++---------- sdks/go/test/build.gradle | 16 +++++++------- sdks/go/test/run_validatesrunner_tests.sh | 22 ++++--------------- 6 files changed, 20 insertions(+), 68 deletions(-) diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml index a84485c717fb..eeb462fa29ff 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml @@ -89,15 +89,8 @@ jobs: with: gradle-command: :runners:google-cloud-dataflow-java:validatesRunner arguments: | - -x shadowJar \ - -x shadowTestJar \ - -x compileJava \ - -x compileTestJava \ - -x jar \ - -x testJar \ - -x classes \ - -x testClasses \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PtestJavaVersion=${{ matrix.java_version }} + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml index 11b18548249e..d0499b65c5a7 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml @@ -80,20 +80,13 @@ jobs: java-version: | ${{ matrix.java_version }} 11 - - name: run jar Java${{ matrix.java_version }} script - run: | - ./gradlew :runners:direct-java:shadowJar :runners:direct-java:shadowTestJar \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 - name: run validatesRunner Java${{ matrix.java_version }} script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:direct-java:validatesRunner arguments: | - -x shadowJar \ - -x shadowTestJar \ - -x compileJava \ - -x compileTestJava \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PtestJavaVersion=${{ matrix.java_version }} + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml index 7f7b9c9270d2..9b061028cbce 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml @@ -78,23 +78,13 @@ jobs: java-version: | 8 11 - - name: run jar Java8 script - run: | - ./gradlew :runners:flink:1.19:jar :runners:flink:1.19:testJar - name: run validatesRunner Java8 script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:flink:1.19:validatesRunner arguments: | - -x shadowJar \ - -x shadowTestJar \ - -x compileJava \ - -x compileTestJava \ - -x jar \ - -x testJar \ - -x classes \ - -x testClasses \ - -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ + -PtestJavaVersion=8 \ + -Pjava8Home=$JAVA_HOME_8_X64 \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml index 0852fcdf4afa..dae408e4346f 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml @@ -78,23 +78,13 @@ jobs: java-version: | 8 11 - - name: run jar Java8 script - run: | - ./gradlew :runners:spark:3:jar :runners:spark:3:testJar - name: run validatesRunner Java8 script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:spark:3:validatesRunner arguments: | - -x shadowJar \ - -x shadowTestJar \ - -x compileJava \ - -x compileTestJava \ - -x jar \ - -x testJar \ - -x classes \ - -x testClasses \ - -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ + -PtestJavaVersion=8 \ + -Pjava8Home=$JAVA_HOME_8_X64 \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/sdks/go/test/build.gradle b/sdks/go/test/build.gradle index 74b6a10cad4d..5576c40c0aab 100644 --- a/sdks/go/test/build.gradle +++ b/sdks/go/test/build.gradle @@ -41,7 +41,7 @@ task dataflowValidatesRunner() { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -66,7 +66,7 @@ task dataflowValidatesRunnerARM64() { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -95,7 +95,7 @@ task flinkValidatesRunner { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -123,7 +123,7 @@ task samzaValidatesRunner { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -151,7 +151,7 @@ task sparkValidatesRunner { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -190,7 +190,7 @@ tasks.register("ulrValidatesRunner") { } exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", ". ${envdir}/bin/activate && ./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -217,7 +217,7 @@ task prismValidatesRunner { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -262,7 +262,7 @@ ext.goIoValidatesRunnerTask = { proj, name, scriptOpts, pipelineOpts -> logger.info("Running the command: sh -c ./run_validatesrunner_tests.sh ${options.join(' ')}") exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" diff --git a/sdks/go/test/run_validatesrunner_tests.sh b/sdks/go/test/run_validatesrunner_tests.sh index 4278c817d0a4..2e1b5fa3f396 100755 --- a/sdks/go/test/run_validatesrunner_tests.sh +++ b/sdks/go/test/run_validatesrunner_tests.sh @@ -259,8 +259,8 @@ s.close() TMPDIR=$(mktemp -d) -if [[ -n "$JAVA_HOME" ]]; then - JAVA_CMD="$JAVA_HOME/bin/java" +if [[ -n "$JAVA_HOME_JOB_SERVER" ]]; then + JAVA_CMD="$JAVA_HOME_JOB_SERVER/bin/java" else JAVA_CMD="java" fi @@ -374,18 +374,7 @@ if [[ "$RUNNER" == "dataflow" ]]; then CONTAINER=us.gcr.io/$PROJECT/$USER/beam_go_sdk echo "Using container $CONTAINER" - # TODO(https://github.com/apache/beam/issues/27674): remove this branch once the jenkins VM can build multiarch, or jenkins is deprecated. - if [[ "$USER" == "jenkins" ]]; then - ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG - - # Verify it exists - docker images | grep $TAG - - # Push the container - gcloud docker -- push $CONTAINER:$TAG - else - ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers - fi + ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then ARGS="$ARGS --experiments=use_portable_job_submission" @@ -395,7 +384,7 @@ if [[ "$RUNNER" == "dataflow" ]]; then JAVA_TAG=$(date +%Y%m%d-%H%M%S) JAVA_CONTAINER=us.gcr.io/$PROJECT/$USER/beam_java11_sdk echo "Using container $JAVA_CONTAINER for cross-language java transforms" - ./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$JAVA_TAG -Pjava11Home=$JAVA11_HOME + ./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$JAVA_TAG # Verify it exists docker images | grep $JAVA_TAG @@ -457,9 +446,6 @@ if [[ "$RUNNER" == "dataflow" ]]; then # Note: we don't delete the multi-arch containers here because this command only deletes the manifest list with the tag, # the associated container images can't be deleted because they are not tagged. However, multi-arch containers that are # older than 6 weeks old are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs daily. - if [[ "$USER" == "jenkins" ]]; then - gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container" - fi if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then # Delete the java cross-language container locally and remotely docker rmi $JAVA_CONTAINER:$JAVA_TAG || echo "Failed to remove container" From 8629453473ad79b4498db4a7506343dae1ab97b7 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Wed, 16 Jul 2025 10:14:28 -0400 Subject: [PATCH 2/4] remove extra build jar --- ..._PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml index eeb462fa29ff..de31ef05dc82 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml @@ -80,10 +80,6 @@ jobs: java-version: | ${{ matrix.java_version }} 11 - - name: run jar Java${{ matrix.java_version }} script - run: | - ./gradlew runners:google-cloud-dataflow-java:testJar :runners:google-cloud-dataflow-java:worker:shadowJar \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 - name: run validatesRunner Java${{ matrix.java_version }} script uses: ./.github/actions/gradle-command-self-hosted-action with: From db14bfdf2f15b4dc13c2b0d5011ca267fb8f03e5 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Wed, 16 Jul 2025 10:17:55 -0400 Subject: [PATCH 3/4] Fix Direct JavaVersion typo, clean up --- ...am_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml | 2 +- buildSrc/build.gradle.kts | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml index d0499b65c5a7..365b50e9e350 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml @@ -85,7 +85,7 @@ jobs: with: gradle-command: :runners:direct-java:validatesRunner arguments: | - -PtestJavaVersion=${{ matrix.java_version }} + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index d053713cab91..9cb318c7f27f 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -92,9 +92,5 @@ gradlePlugin { id = "org.apache.beam.vendor-java" implementationClass = "org.apache.beam.gradle.VendorJavaPlugin" } - create("beamJenkins") { - id = "org.apache.beam.jenkins" - implementationClass = "org.apache.beam.gradle.BeamJenkinsPlugin" - } } } From e25f412958298f63e5a9b0cac6656c96924f9694 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Wed, 16 Jul 2025 10:24:37 -0400 Subject: [PATCH 4/4] Fix typo --- ...am_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml index de31ef05dc82..c03e2435a83b 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml @@ -85,7 +85,7 @@ jobs: with: gradle-command: :runners:google-cloud-dataflow-java:validatesRunner arguments: | - -PtestJavaVersion=${{ matrix.java_version }} + -PtestJavaVersion=${{ matrix.java_version }} \ -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ max-workers: 12 - name: Archive JUnit Test Results