Running tests on GitLab on arm64 Linux.

.gitlab-ci.yml

@@ -35,6 +35,7 @@ stages:
  - shared-pipeline
  - benchmarks
  - tests
+ - tests-arm64
  - test-summary
  - exploration-tests
  - ci-visibility-tests
@@ -164,6 +165,24 @@ default:
  echo -e "${TEXT_BOLD}${TEXT_YELLOW} Containers:${TEXT_CLEAR} https://app.datadoghq.com/containers?${TIME_PARAMS}query=image_name%3A%2A%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false"
  echo -e "${TEXT_BOLD}${TEXT_YELLOW} Processes:${TEXT_CLEAR} https://app.datadoghq.com/process?${TIME_PARAMS}query=image_name%3A%2A%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false"
 
+.tier_m:
+ variables: &tier_m_variables
+ GRADLE_WORKERS: 4
+ GRADLE_MEMORY_MIN: 1G
+ GRADLE_MEMORY_MAX: 5G
+ KUBERNETES_CPU_REQUEST: 6
+ KUBERNETES_MEMORY_REQUEST: 16Gi
+ KUBERNETES_MEMORY_LIMIT: 16Gi
+
+.tier_l:
+ variables: &tier_l_variables
+ GRADLE_WORKERS: 6
+ GRADLE_MEMORY_MIN: 1G
+ GRADLE_MEMORY_MAX: 6G
+ KUBERNETES_CPU_REQUEST: 10
+ KUBERNETES_MEMORY_REQUEST: 20Gi
+ KUBERNETES_MEMORY_LIMIT: 20Gi
+
 .gitlab_base_ref_params: &gitlab_base_ref_params
  - |
  export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh)
@@ -177,13 +196,8 @@ default:
  image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base
  stage: build
  variables:
+ <<: *tier_m_variables
  MAVEN_OPTS: "-Xms256M -Xmx1024M"
- GRADLE_WORKERS: 6
- GRADLE_MEMORY_MIN: 1G
- GRADLE_MEMORY_MAX: 4G
- KUBERNETES_CPU_REQUEST: 10
- KUBERNETES_MEMORY_REQUEST: 20Gi
- KUBERNETES_MEMORY_LIMIT: 20Gi
  CACHE_TYPE: "lib" #default
  FF_USE_FASTZIP: "true"
  CACHE_COMPRESSION_LEVEL: "slowest"
@@ -642,6 +656,7 @@ muzzle-dep-report:
  needs: [ build_tests ]
  stage: tests
  variables:
+ <<: *tier_m_variables
  GRADLE_PARAMS: "-PskipFlakyTests"
  CONTINUE_ON_FAILURE: "false"
  TESTCONTAINERS_CHECKS_DISABLE: "true"
@@ -716,6 +731,122 @@ muzzle-dep-report:
  - scheduler_failure
  - data_integrity_failure
 
+.test_job_arm64:
+ image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm
+ tags: [ "docker-in-docker:arm64" ]
+ stage: tests-arm64
+ needs: []
+ variables:
+ <<: *tier_m_variables
+ # arm64 does not include JDK 7 or ibm8 (upstream images not published for arm64).
+ DEFAULT_TEST_JVMS: /^(8|11|17|21|25|tip)$/
+ GRADLE_PARAMS: "-PskipFlakyTests"
+ TESTCONTAINERS_CHECKS_DISABLE: "true"
+ TESTCONTAINERS_RYUK_DISABLED: "true"
+ TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: "registry.ddbuild.io/images/mirror/"
+ JETTY_AVAILABLE_PROCESSORS: 4
+ GIT_SUBMODULE_STRATEGY: normal
+ GIT_SUBMODULE_DEPTH: 1
+ rules:
+ - if: $testJvm =~ $DEFAULT_TEST_JVMS
+ when: manual
+ allow_failure: true
+ - if: '$NON_DEFAULT_JVMS == "true" && $testJvm != "ibm8" && $testJvm != "oracle8"'
+ when: manual
+ allow_failure: true
+ cache:
+ - key: dependency-$CACHE_TYPE
+ paths:
+ - .gradle/wrapper
+ - .gradle/caches
+ - .gradle/notifications
+ - .mvn/caches
+ policy: pull
+ fallback_keys:
+ - dependency-base
+ - dependency-lib
+ unprotect: true
+ before_script:
+ - git config --global --add safe.directory "$CI_PROJECT_DIR"
+ # Akka token added to SSM from https://account.akka.io/token
+ - export ORG_GRADLE_PROJECT_akkaRepositoryToken=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.akka_repo_token --with-decryption --query "Parameter.Value" --out text)
+ - export ORG_GRADLE_PROJECT_mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY
+ - export ORG_GRADLE_PROJECT_gradlePluginProxy=$GRADLE_PLUGIN_PROXY
+ - |
+ JAVA_HOMES=$(env | grep -E '^JAVA_[A-Z0-9_]+_HOME=' | sed 's/=.*//' | paste -sd,)
+ cat >> gradle.properties <<EOF
+ org.gradle.java.installations.auto-detect=false
+ org.gradle.java.installations.auto-download=false
+ org.gradle.java.installations.fromEnv=$JAVA_HOMES
+ EOF
+ - mkdir -p .gradle .mvn/caches
+ # GitLab's cache helper restores .gradle as root, but we run as non-root-user (uid 1001),
+ # and Gradle does `chmod 700 .gradle` on startup which requires user ownership.
+ - sudo chown -R 1001:1001 .gradle
+ - export GRADLE_USER_HOME=$(pwd)/.gradle
+ - sed -i "s|https://repo.maven.apache.org/maven2/|$MAVEN_REPOSITORY_PROXY|g" .mvn/wrapper/maven-wrapper.properties
+ - *normalize_node_index
+ - *prepare_test_env
+ # Disable CDS in forked JVMs to avoid SIGSEGVs on Linux arm64.
+ - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xshare:off -Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djava.util.prefs.userRoot=/tmp/.java/.userPrefs-${CI_JOB_ID}' -Ddatadog.forkedMinHeapSize=128M -Ddatadog.forkedMaxHeapSize=1024M"
+ - export GRADLE_ARGS="--build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
+ - ./gradlew --version
+ script:
+ - *gitlab_base_ref_params
+ - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS --continue
+ after_script:
+ - *restore_pretest_env
+ - *set_datadog_api_keys
+ - *container_info
+ - *cgroup_info
+ - source .gitlab/gitlab-utils.sh
+ - gitlab_section_start "collect-reports" "Collecting reports"
+ - .gitlab/collect_reports.sh
+ - .gitlab/collect_results.sh
+ - .gitlab/upload_ciapp.sh $CACHE_TYPE $testJvm
+ - gitlab_section_end "collect-reports"
+ - .gitlab/count_tests.sh "$GRADLE_TARGET" "$testJvm" "./results" "./test_counts_${CI_JOB_ID}.json"
+ - URL_ENCODED_JOB_NAME=$(jq -rn --arg x "$CI_JOB_NAME" '$x|@uri')
+ - echo -e "${TEXT_BOLD}${TEXT_YELLOW}See test results in Datadog:${TEXT_CLEAR} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}%20%40ci.job.name%3A%22${URL_ENCODED_JOB_NAME}%22"
+ artifacts:
+ when: always
+ paths:
+ - ./reports.tar
+ - ./profiles.tar
+ - ./results
+ - './test_counts_*.json'
+ - '.gradle/daemon/*/*.out.log'
+ reports:
+ junit: results/*.xml
+ retry:
+ max: 2
+ when:
+ - unknown_failure
+ - stuck_or_timeout_failure
+ - runner_system_failure
+ - unmet_prerequisites
+ - scheduler_failure
+ - data_integrity_failure
+
+.test_job_arm64_with_test_agent:
+ extends: .test_job_arm64
+ variables:
+ CI_USE_TEST_AGENT: "true"
+ CI_AGENT_HOST: local-agent
+ services:
+ - name: registry.ddbuild.io/images/mirror/dd-apm-test-agent/ddapm-test-agent:v1.44.0
+ alias: local-agent
+ variables:
+ LOG_LEVEL: "DEBUG"
+ TRACE_LANGUAGE: "java"
+ DD_SUPPRESS_TRACE_PARSE_ERRORS: "true"
+ DD_POOL_TRACE_CHECK_FAILURES: "true"
+ DD_DISABLE_ERROR_RESPONSES: "true"
+ ENABLED_CHECKS: "trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service"
+ script:
+ - !reference [.test_job_arm64, script]
+ - .gitlab/check_test_agent_results.sh
+
 .test_job_with_test_agent:
  extends: .test_job
  variables:
@@ -772,10 +903,28 @@ test_base:
  - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi
  - !reference [.test_job, script]
 
+test_base_arm64:
+ extends: .test_job_arm64
+ variables:
+ GRADLE_TARGET: ":baseTest"
+ CACHE_TYPE: "base"
+ parallel:
+ matrix: *test_matrix_4
+
 test_inst:
  extends: .test_job_with_test_agent
  needs: *needs_build_tests_inst
  variables:
+ <<: *tier_l_variables
+ GRADLE_TARGET: ":instrumentationTest"
+ CACHE_TYPE: "inst"
+ parallel:
+ matrix: *test_matrix_8
+
+test_inst_arm64:
+ extends: .test_job_arm64_with_test_agent
+ variables:
+ <<: *tier_l_variables
  GRADLE_TARGET: ":instrumentationTest"
  CACHE_TYPE: "inst"
  parallel:
@@ -793,6 +942,20 @@ test_inst_latest:
  - GRADLE_TARGET: ":instrumentationLatestDepTest"
  CACHE_TYPE: "latestdep"
  variables:
+ <<: *tier_l_variables
+ GRADLE_TARGET: ":instrumentationLatestDepTest"
+ CACHE_TYPE: "latestdep"
+ parallel:
+ matrix:
+ - testJvm: ["8", "17", "21", "25"] # the latest "tip" version is LTS v25
+ # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time
+ # This emulates "parallel" by including it in the matrix
+ CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"]
+
+test_inst_latest_arm64:
+ extends: .test_job_arm64_with_test_agent
+ variables:
+ <<: *tier_l_variables
  GRADLE_TARGET: ":instrumentationLatestDepTest"
  CACHE_TYPE: "latestdep"
  parallel:
@@ -852,6 +1015,14 @@ test_profiling:
  parallel:
  matrix: *test_matrix
 
+test_profiling_arm64:
+ extends: .test_job_arm64
+ variables:
+ GRADLE_TARGET: ":profilingTest"
+ CACHE_TYPE: "profiling"
+ parallel:
+ matrix: *test_matrix
+
 # specific jvms list for debugger project because J9-based JVMs have issues with local vars
 # so need to test at least against one J9-based JVM
 test_debugger:
@@ -863,6 +1034,14 @@ test_debugger:
  parallel:
  matrix: *test_matrix
 
+test_debugger_arm64:
+ extends: .test_job_arm64
+ variables:
+ GRADLE_TARGET: ":debuggerTest"
+ CACHE_TYPE: "base"
+ parallel:
+ matrix: *test_matrix
+
 test_smoke:
  extends: .test_job
  # needs:parallel:matrix limits this job to a specific build_tests combination.
@@ -876,6 +1055,17 @@ test_smoke:
  CACHE_TYPE: "smoke"
  MAVEN_OPTS: "-Xms256M -Xmx1024M"
  variables:
+ <<: *tier_l_variables
+ GRADLE_TARGET: "stageMainDist :smokeTest"
+ GRADLE_PARAMS: "-PskipFlakyTests"
+ CACHE_TYPE: "smoke"
+ parallel:
+ matrix: *test_matrix_8
+
+test_smoke_arm64:
+ extends: .test_job_arm64
+ variables:
+ <<: *tier_l_variables
  GRADLE_TARGET: "stageMainDist :smokeTest"
  GRADLE_PARAMS: "-PskipFlakyTests"
  CACHE_TYPE: "smoke"
@@ -893,6 +1083,18 @@ test_ssi_smoke:
  - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/'
  when: on_success
  variables:
+ <<: *tier_l_variables
+ GRADLE_TARGET: "stageMainDist :smokeTest"
+ CACHE_TYPE: "smoke"
+ DD_INJECT_FORCE: "true"
+ DD_INJECTION_ENABLED: "tracer"
+ parallel:
+ matrix: *test_matrix_8
+
+test_ssi_smoke_arm64:
+ extends: .test_job_arm64
+ variables:
+ <<: *tier_l_variables
  GRADLE_TARGET: "stageMainDist :smokeTest"
  CACHE_TYPE: "smoke"
  DD_INJECT_FORCE: "true"
@@ -913,6 +1115,18 @@ test_smoke_graalvm:
  matrix:
  - testJvm: ["graalvm17", "graalvm21", "graalvm25"]
 
+test_smoke_graalvm_arm64:
+ extends: .test_job_arm64
+ tags: [ "arch:arm64" ]
+ variables:
+ GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test"
+ CACHE_TYPE: "smoke"
+ CI_NO_SPLIT: "true"
+ NON_DEFAULT_JVMS: "true"
+ parallel:
+ matrix:
+ - testJvm: ["graalvm17", "graalvm21", "graalvm25"]
+
 test_smoke_semeru8_debugger:
  extends: .test_job
  needs: *needs_build_tests_smoke

build.gradle.kts

@@ -1,4 +1,5 @@
 import com.diffplug.gradle.spotless.SpotlessExtension
+import datadog.gradle.plugin.HostPlatform
 import datadog.gradle.plugin.ci.testAggregate
 
 plugins {
@@ -82,6 +83,8 @@ allprojects {
  dependsOn(tasks.withType<AbstractCompile>())
  }
 
+ val isLinuxArm64 = HostPlatform.isLinuxArm64()
+
  tasks.configureEach {
  if (this is JavaForkOptions) {
  maxHeapSize = System.getProperty("datadog.forkedMaxHeapSize")
@@ -91,6 +94,20 @@ allprojects {
  "-XX:+HeapDumpOnOutOfMemoryError",
  "-XX:HeapDumpPath=/tmp"
  )
+ if (isLinuxArm64) {
+ // Disable CDS to avoid SIGSEGVs on Linux arm64.
+ jvmArgs("-Xshare:off")
+ }
+ }
+ }
+
+ // Disable CDS to avoid SIGSEGVs on Linux arm64.
+ if (isLinuxArm64) {
+ tasks.withType<JavaCompile>().configureEach {
+ options.forkOptions.jvmArgs?.add("-Xshare:off")
+ }
+ tasks.withType<GroovyCompile>().configureEach {
+ groovyOptions.forkOptions.jvmArgs?.add("-Xshare:off")
  }
  }
 }

buildSrc/src/main/kotlin/datadog/gradle/plugin/HostPlatform.kt

@@ -0,0 +1,21 @@
+package datadog.gradle.plugin
+
+import java.util.Locale
+
+object HostPlatform {
+ @JvmStatic
+ fun isLinuxArm64(): Boolean = isExpectedOs("linux") && isArm64()
+
+ @JvmStatic
+ fun isMacArm64(): Boolean = isExpectedOs("mac") && isArm64()
+
+ private fun isExpectedOs(expectedOs: String): Boolean {
+ val osName = System.getProperty("os.name", "").lowercase(Locale.ROOT)
+ return osName.contains(expectedOs)
+ }
+
+ private fun isArm64(): Boolean {
+ val osArch = System.getProperty("os.arch", "").lowercase(Locale.ROOT)
+ return osArch.contains("aarch64") || osArch.contains("arm64")
+ }
+}

buildSrc/src/main/kotlin/datadog/gradle/plugin/instrument/InstrumentPostProcessingAction.kt

@@ -1,5 +1,6 @@
 package datadog.gradle.plugin.instrument
 
+import datadog.gradle.plugin.HostPlatform
 import datadog.gradle.plugin.instrument.BuildTimeInstrumentationPlugin.Companion.BUILD_TIME_INSTRUMENTATION_PLUGIN_CONFIGURATION
 import org.gradle.api.Action
 import org.gradle.api.Project
@@ -73,6 +74,10 @@ abstract class InstrumentPostProcessingAction @Inject constructor(
  return workerExecutor.processIsolation {
  forkOptions {
  setExecutable(javaLauncher.executablePath.asFile.absolutePath)
+ if (HostPlatform.isLinuxArm64()) {
+ // Disable CDS to avoid SIGSEGVs on Linux arm64.
+ jvmArgs("-Xshare:off")
+ }
  }
  }
  }