From cb78deeb7b865ef5d3688317774eadbfa4de6626 Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Fri, 9 Feb 2024 16:13:49 +0100 Subject: [PATCH] Use libddwaf builds --- .github/workflows/actions.yml | 574 ++++----------------- .gitmodules | 3 - CMakeLists.txt | 4 +- README.md | 85 ++- build.gradle | 80 ++- ci/alpine-abuild-libc++/.gitignore | 1 - ci/alpine-abuild-libc++/APKBUILD | 66 --- ci/alpine-abuild-libc++/Dockerfile | 26 - ci/alpine-abuild-libc++/abuilder | 18 - ci/alpine-abuild-libc++/build_aarch64.sh | 3 - ci/alpine-abuild-libc++/build_x86_64.sh | 3 - ci/alpine-libc++-static/aarch64/Dockerfile | 39 -- ci/alpine-libc++-static/x86_64/Dockerfile | 39 -- ci/build | 152 ++++++ deps/CMakeLists.txt | 44 +- libddwaf | 1 - 16 files changed, 364 insertions(+), 774 deletions(-) delete mode 100644 ci/alpine-abuild-libc++/.gitignore delete mode 100644 ci/alpine-abuild-libc++/APKBUILD delete mode 100644 ci/alpine-abuild-libc++/Dockerfile delete mode 100644 ci/alpine-abuild-libc++/abuilder delete mode 100755 ci/alpine-abuild-libc++/build_aarch64.sh delete mode 100755 ci/alpine-abuild-libc++/build_x86_64.sh delete mode 100644 ci/alpine-libc++-static/aarch64/Dockerfile delete mode 100644 ci/alpine-libc++-static/x86_64/Dockerfile create mode 100755 ci/build delete mode 160000 libddwaf diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index a7d003a1..1224fa6e 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -17,9 +17,6 @@ jobs: steps: - uses: actions/checkout@v2 name: Checkout - with: - submodules: recursive - clean: true - name: Cache Gradle artifacts uses: actions/cache@v2 with: @@ -29,14 +26,6 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: | ${{ runner.os }}-gradle- - - name: Build libddwdaf - run: | - mkdir libddwaf/Debug - cd libddwaf/Debug - cmake .. -DCMAKE_BUILD_TYPE=Debug - VERBOSE=1 make -j - DESTDIR=out make install - cd ../.. - name: Build and test run: | ./gradlew check jacocoTestReport @@ -50,545 +39,174 @@ jobs: flags: helper verbose: true files: build/coverage.xml,build/reports/jacoco/test/jacocoTestReport.xml + Native_binaries_Stage_macos_x86_64: name: MacOS x86_64 runs-on: macOS-12 env: - generator: Unix Makefiles - shlib_prefix: lib - shlib_ext: dylib - debug_ext: dylib.dwarf + os: macos + arch: x86_64 + libc: "" artifactsuff: macos-x86_64 - libdir: macos/x86_64 steps: - - uses: actions/checkout@v2 - name: Checkout - with: - submodules: recursive - clean: true - - name: Create Build Directory for libddwaf - run: cmake -E make_directory "${{ env.tempdir }}/buildPW" - - name: Create Build Directory for JNI binding - run: cmake -E make_directory "${{ env.tempdir }}/buildAG" - - name: Create Packages Directory - run: cmake -E make_directory ${{ env.tempdir }}/packages - - name: Generate Build Scripts for libddwaf - run: | - cmake -DCMAKE_BUILD_TYPE=${{ env.buildType }} \ - -DLIBDDWAF_BUILD_STATIC=0 \ - -DCMAKE_C_FLAGS="-mmacosx-version-min=11.6 -fno-omit-frame-pointer" \ - -DCMAKE_CXX_FLAGS="-mmacosx-version-min=11.6 -fno-omit-frame-pointer" \ - -DCMAKE_INSTALL_PREFIX='${{ env.tempdir }}/out' \ - -G '${{ env.generator }}' '${{ github.workspace }}/libddwaf' - working-directory: ${{ env.tempdir }}/buildPW - - name: Build Binaries for libddwaf - run: cmake --build . --verbose - working-directory: ${{ env.tempdir }}/buildPW - - name: Install Binaries for libddwaf - run: cmake --build . --target install - working-directory: ${{ env.tempdir }}/buildPW - - name: Generate Build Scripts for JNI binding - run: | - cmake -DCMAKE_BUILD_TYPE=${{ env.buildType }} \ - -DCMAKE_PREFIX_PATH='${{ env.tempdir }}/out/share/cmake/libddwaf/' \ - -DCMAKE_C_FLAGS="-mmacosx-version-min=11.6 -fno-omit-frame-pointer" \ - -G "${{ env.generator }}" ${{ github.workspace }} - working-directory: ${{ env.tempdir }}/buildAG - - name: Build Binaries for JNI Binding - run: cmake --build . --verbose - working-directory: ${{ env.tempdir }}/buildAG - - name: Copy libddwaf binaries to packages - run: | - cmake -E copy ${{ env.shlib_prefix }}ddwaf.${{ env.shlib_ext }} \ - ${{ env.shlib_prefix }}ddwaf.${{ env.debug_ext }} \ - ${{ env.tempdir }}/packages - working-directory: ${{ env.tempdir }}/buildPW - - name: Copy libddwaf binding binaries to native_libs (testing) - run: | - cmake -E copy ${{ env.shlib_prefix }}ddwaf.${{ env.shlib_ext }} \ - ${{ env.shlib_prefix }}ddwaf.${{ env.debug_ext }} \ - ${{ github.workspace }}/native_libs/${{ env.libdir }} - working-directory: ${{ env.tempdir }}/buildPW - - name: Copy JNI binding binaries to packages - run: | - cmake -E copy ${{ env.shlib_prefix }}sqreen_jni.${{ env.shlib_ext }} \ - ${{ env.shlib_prefix }}sqreen_jni.${{ env.debug_ext }} \ - ${{ env.tempdir }}/packages - working-directory: ${{ env.tempdir }}/buildAG - - name: Copy JNI binding binaries to native_libs (testing) - run: | - cmake -E copy ${{ env.shlib_prefix }}sqreen_jni.${{ env.shlib_ext }} \ - ${{ env.shlib_prefix }}sqreen_jni.${{ env.debug_ext }} \ - ${{ github.workspace }}/native_libs/${{ env.libdir }} - working-directory: ${{ env.tempdir }}/buildAG - - name: Cache Gradle artifacts - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Run tests on binding - run: | - set -ex - cd '${{ github.workspace }}' - export LIBDDWAF_INSTALL_PREFIX='${{ env.tempdir }}/out' - ./gradlew --build-cache check --info -PuseReleaseBinaries - ./gradlew --stop - shell: bash - - name: Save Artifacts - uses: actions/upload-artifact@v2 - with: - path: ${{ env.tempdir }}/packages - name: libsqreen_jni_${{ env.artifactsuff }} + - name: Checkout repository + uses: actions/checkout@v2 + - name: Build JNI bindings + run: ci/build build-bindings --os '${{ env.os }}' --arch '${{ env.arch }}' --libc '${{ env.libc }}' + - name: Run tests + run: ci/build run-tests --os '${{ env.os }}' --arch '${{ env.arch }}' --libc '${{ env.libc }}' + - name: Save Artifacts + uses: actions/upload-artifact@v2 + with: + path: build/native_libs + name: jni_${{ env.artifactsuff }} + Native_binaries_Stage_macos_aarch64: name: MacOS aarch64 runs-on: macOS-12 env: - generator: Unix Makefiles - shlib_prefix: lib - shlib_ext: dylib - debug_ext: dylib.dwarf + os: macos + arch: aarch64 + libc: "" artifactsuff: macos-aarch64 - libdir: macos/aarch64 steps: - - uses: actions/checkout@v2 - name: Checkout - with: - submodules: recursive - clean: true - - name: Create Build Directory for libddwaf - run: cmake -E make_directory "${{ env.tempdir }}/buildPW" - - name: Create Build Directory for JNI binding - run: cmake -E make_directory "${{ env.tempdir }}/buildAG" - - name: Create Packages Directory - run: cmake -E make_directory ${{ env.tempdir }}/packages - - name: Generate Build Scripts for libddwaf - run: | - cmake -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DCMAKE_BUILD_TYPE=${{ env.buildType }} \ - -DLIBDDWAF_BUILD_STATIC=0 \ - -DCMAKE_C_FLAGS="-mmacosx-version-min=11.6 -fno-omit-frame-pointer" \ - -DCMAKE_CXX_FLAGS="-mmacosx-version-min=11.6 -fno-omit-frame-pointer" \ - -DCMAKE_INSTALL_PREFIX='${{ env.tempdir }}/out' \ - -G '${{ env.generator }}' '${{ github.workspace }}/libddwaf' - working-directory: ${{ env.tempdir }}/buildPW - - name: Build Binaries for libddwaf - run: cmake --build . - working-directory: ${{ env.tempdir }}/buildPW - - name: Install Binaries for libddwaf - run: cmake --build . --target install - working-directory: ${{ env.tempdir }}/buildPW - - name: Generate Build Scripts for JNI binding - run: | - cmake -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DCMAKE_BUILD_TYPE=${{ env.buildType }} \ - -DCMAKE_PREFIX_PATH='${{ env.tempdir }}/out/share/cmake/libddwaf/' \ - -DCMAKE_C_FLAGS="-mmacosx-version-min=11.6 -fno-omit-frame-pointer" \ - -G "${{ env.generator }}" ${{ github.workspace }} - working-directory: ${{ env.tempdir }}/buildAG - - name: Build Binaries for JNI Binding - run: cmake --build . - working-directory: ${{ env.tempdir }}/buildAG - - name: Copy libddwaf binaries to packages - run: | - cmake -E copy ${{ env.shlib_prefix }}ddwaf.${{ env.shlib_ext }} \ - ${{ env.shlib_prefix }}ddwaf.${{ env.debug_ext }} \ - ${{ env.tempdir }}/packages - working-directory: ${{ env.tempdir }}/buildPW - - name: Copy libddwaf binding binaries to native_libs (testing) - run: | - cmake -E copy ${{ env.shlib_prefix }}ddwaf.${{ env.shlib_ext }} \ - ${{ env.shlib_prefix }}ddwaf.${{ env.debug_ext }} \ - ${{ github.workspace }}/native_libs/${{ env.libdir }} - working-directory: ${{ env.tempdir }}/buildPW - - name: Copy JNI binding binaries to packages - run: | - cmake -E copy ${{ env.shlib_prefix }}sqreen_jni.${{ env.shlib_ext }} \ - ${{ env.shlib_prefix }}sqreen_jni.${{ env.debug_ext }} \ - ${{ env.tempdir }}/packages - working-directory: ${{ env.tempdir }}/buildAG - - name: Copy JNI binding binaries to native_libs (testing) - run: | - cmake -E copy ${{ env.shlib_prefix }}sqreen_jni.${{ env.shlib_ext }} \ - ${{ env.shlib_prefix }}sqreen_jni.${{ env.debug_ext }} \ - ${{ github.workspace }}/native_libs/${{ env.libdir }} - working-directory: ${{ env.tempdir }}/buildAG - - name: Cache Gradle artifacts - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- -# - name: Run tests on binding -# run: | -# set -ex -# cd '${{ github.workspace }}' -# export LIBDDWAF_INSTALL_PREFIX='${{ env.tempdir }}/out' -# ./gradlew --build-cache check --info -PuseReleaseBinaries -# ./gradlew --stop -# shell: bash + - name: Checkout repository + uses: actions/checkout@v2 + - name: Build JNI bindings + run: ci/build build-bindings --os '${{ env.os }}' --arch '${{ env.arch }}' --libc '${{ env.libc }}' + # No arm64 tests for now (will do with macos-13) + #- name: Run tests + # run: ci/build run-tests --os '${{ env.os }}' --arch '${{ env.arch }}' --libc '${{ env.libc }}' - name: Save Artifacts uses: actions/upload-artifact@v2 with: - path: ${{ env.tempdir }}/packages - name: libsqreen_jni_${{ env.artifactsuff }} + path: build/native_libs + name: jni_${{ env.artifactsuff }} Native_binaries_Stage_windows_x86_64: name: Windows x86_64 runs-on: windows-2019 defaults: run: - shell: cmd + shell: bash env: - generator: NMake Makefiles - shlib_prefix: - shlib_ext: dll - debug_ext: pdb + os: windows + arch: x86_64 + libc: "" artifactsuff: win-x86_64 - libdir: windows/x86_64 steps: - - uses: actions/checkout@v2 - name: Checkout - with: - submodules: recursive - clean: true + - name: Checkout repository + uses: actions/checkout@v2 - uses: ilammy/msvc-dev-cmd@v1 name: Setup x86_64 build with: toolset: 14.29 arch: amd64 - - name: Create Build Directory for libddwaf - run: cmake -E make_directory "${{ env.tempdir }}/buildPW" - - name: Create Build Directory for JNI binding - run: cmake -E make_directory "${{ env.tempdir }}/buildAG" - - name: Create Packages Directory - run: cmake -E make_directory ${{ env.tempdir }}/packages - - name: Generate Build Scripts for libddwaf - run: | - cmake -DCMAKE_BUILD_TYPE=${{ env.buildType }} -DLIBDDWAF_BUILD_SHARED=0 -DCMAKE_INSTALL_PREFIX="${{ env.tempdir }}/out" -G "${{ env.generator }}" "${{ github.workspace }}/libddwaf" - working-directory: ${{ env.tempdir }}/buildPW - - name: Build Binaries for libddwaf - run: cmake --build . --target libddwaf_static - working-directory: ${{ env.tempdir }}/buildPW - - name: Install Binaries for libddwaf - run: cmake --build . --target install - working-directory: ${{ env.tempdir }}/buildPW - - name: Generate Build Scripts for JNI binding - run: | - cmake -DCMAKE_BUILD_TYPE=${{ env.buildType }} -DCMAKE_PREFIX_PATH="${{ env.tempdir }}/out/share/cmake/libddwaf/" -G "${{ env.generator }}" "${{ github.workspace }}" - working-directory: ${{ env.tempdir }}/buildAG - - name: Build Binaries for JNI Binding - run: cmake --build . - working-directory: ${{ env.tempdir }}/buildAG - - name: Copy JNI binding binaries to packages - run: cmake -E copy ${{ env.shlib_prefix }}sqreen_jni.${{ env.shlib_ext }} ${{ env.shlib_prefix }}sqreen_jni.${{ env.debug_ext }} ${{ env.tempdir }}\packages - shell: cmd - working-directory: ${{ env.tempdir }}/buildAG - - name: Copy JNI binding binaries to native_libs (testing) - run: cmake -E copy ${{ env.shlib_prefix }}sqreen_jni.${{ env.shlib_ext }} ${{ env.shlib_prefix }}sqreen_jni.${{ env.debug_ext }} ${{ github.workspace }}\native_libs\${{ env.libdir }} - shell: cmd - working-directory: ${{ env.tempdir }}/buildAG - - name: Cache Gradle artifacts - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Run tests on binding - run: | - set -ex - cd '${{ github.workspace }}' - export LIBDDWAF_INSTALL_PREFIX='${{ env.tempdir }}/out' - ./gradlew --build-cache check --info -PuseReleaseBinaries - ./gradlew --stop - shell: bash + - name: Build JNI bindings + run: ci/build build-bindings --os '${{ env.os }}' --arch '${{ env.arch }}' --libc '${{ env.libc }}' + - name: Run tests + run: ci/build run-tests --os '${{ env.os }}' --arch '${{ env.arch }}' --libc '${{ env.libc }}' - name: Save Artifacts uses: actions/upload-artifact@v2 with: - path: ${{ env.tempdir }}/packages - name: libsqreen_jni_${{ env.artifactsuff }} - - Native_binaries_Stage_libddwaf_linux_x86_64: - name: Linux x86_64 (semi-static libddwaf.so) - runs-on: ubuntu-20.04 - env: - dockerfile: ci/alpine-libc++-static/x86_64 - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - submodules: recursive - clean: true - - uses: docker/setup-buildx-action@v1 - id: buildx - with: - install: true - - name: Create packages directory - run: mkdir -p ${{ env.tempdir }}/packages - - name: Build semi-statically compiled dynamic library - run: docker buildx build -f ${{ env.dockerfile }}/Dockerfile --progress=plain -o ${{ env.tempdir }}/packages . - - name: Save Artifacts - uses: actions/upload-artifact@v2 - with: - path: ${{ env.tempdir }}/packages - name: libddwaf_linux-x86_64 - - Native_binaries_Stage_libddwaf_linux_aarch64: - name: Linux aarch64 (semi-static libddwaf.so) - runs-on: ubuntu-22.04 - env: - dockerfile: ci/alpine-libc++-static/aarch64 - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - submodules: recursive - clean: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v1 - id: buildx - with: - install: true - - name: Create packages directory - run: mkdir -p ${{ env.tempdir }}/packages - - name: Build semi-statically compiled dynamic library - run: docker buildx build -f ${{ env.dockerfile }}/Dockerfile --platform linux/arm64 --progress=plain -o ${{ env.tempdir }}/packages . - - name: Save Artifacts - uses: actions/upload-artifact@v2 - with: - path: ${{ env.tempdir }}/packages - name: libddwaf_linux-aarch64 + path: build/native_libs + name: jni_${{ env.artifactsuff }} Native_binaries_Stage_linux_x86_64_glibc: name: Linux x86_64 (glibc) runs-on: ubuntu-20.04 - needs: - - Native_binaries_Stage_libddwaf_linux_x86_64 env: - dockerfile: ci/manylinux/x86_64 + os: linux + arch: x86_64 + libc: glibc artifactsuff: linux_glibc-x86_64 - libdir: linux/x86_64/glibc - artifactsDirectory: ${{ github.workspace }}/artifacts steps: - name: Checkout repository uses: actions/checkout@v2 - with: - submodules: recursive - clean: true - - name: Create artifacts directory - run: mkdir -p ${{ env.artifactsDirectory }} - - name: Download libddwaf artifact - uses: actions/download-artifact@v2 - with: - name: libddwaf_linux-x86_64 - path: ${{ env.artifactsDirectory }} - - name: Build docker linux image - run: docker build ${{ env.dockerfile }} -t linux_cmake - - name: Clean previous docker container - run: docker rm -f pwaf_java_build || true - - name: Build and test JNI binding - run: | - docker run --name pwaf_java_build -v $(pwd):${{ github.workspace }} linux_cmake bash -e -c 'export VERBOSE=1; - export LIBDDWAF_INSTALL_PREFIX=/; - tar --strip-components=1 -xf ${{ env.artifactsDirectory }}/libddwaf-x86_64.tar.gz -C /usr/local/ && - mkdir buildAG && cd buildAG && - cmake ${{ github.workspace }} -DCMAKE_BUILD_TYPE=${{ env.buildType }} -DCMAKE_C_FLAGS="-fno-omit-frame-pointer" && - make -j && - cp -v /usr/local/lib/libddwaf.so ${{ github.workspace }}/native_libs/linux/x86_64 && - cp -v *.so ${{ github.workspace }}/native_libs/${{ env.libdir }} && - cd ${{ github.workspace }} && - ./gradlew check --info -PuseReleaseBinaries' - shell: bash - - name: Copy binaries and debug symbols - run: | - sudo chown -R $(whoami) "${{ env.tempdir }}" - mkdir ${{ env.tempdir }}/packages - cd ${{ env.tempdir }}/packages - docker cp pwaf_java_build:/buildAG/libsqreen_jni.so . - docker cp pwaf_java_build:/buildAG/libsqreen_jni.so.debug . + - name: Build docker image + run: ci/build build-docker-image --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} + - name: Build JNI bindings + run: ci/build build-bindings-in-docker --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} + - name: Run tests + run: ci/build run-tests-in-docker --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} - name: Save Artifacts uses: actions/upload-artifact@v2 with: - path: ${{ env.tempdir }}/packages + path: build/native_libs name: jni_${{ env.artifactsuff }} Native_binaries_Stage_linux_x86_64_musl: name: Linux x86_64 (musl) runs-on: ubuntu-20.04 - needs: - - Native_binaries_Stage_libddwaf_linux_x86_64 env: - dockerfile: ci/alpine + os: linux + arch: x86_64 + libc: musl artifactsuff: linux_musl-x86_64 - libdir: linux/x86_64/musl - artifactsDirectory: ${{ github.workspace }}/artifacts - steps: # identical to glibc + steps: - name: Checkout repository uses: actions/checkout@v2 - with: - submodules: recursive - clean: true - - name: Create artifacts directory - run: mkdir -p ${{ env.artifactsDirectory }} - - name: Download libddwaf artifact - uses: actions/download-artifact@v2 - with: - name: libddwaf_linux-x86_64 - path: ${{ env.artifactsDirectory }} - - name: Build docker linux image - run: docker build ${{ env.dockerfile }} -t linux_cmake - - name: Clean previous docker container - run: docker rm -f pwaf_java_build || true - - name: Build and test with release binaries - run: | - docker run --name pwaf_java_build -v $(pwd):${{ github.workspace }} linux_cmake bash -e -c 'export VERBOSE=1; - export LIBDDWAF_INSTALL_PREFIX=/; - tar --strip-components=1 -xvf ${{ env.artifactsDirectory }}/libddwaf-x86_64.tar.gz -C /usr/local/ && - mkdir buildAG && cd buildAG && - cmake ${{ github.workspace }} -DCMAKE_BUILD_TYPE=${{ env.buildType }} -DCMAKE_C_FLAGS="-fno-omit-frame-pointer" && - make -j && - cp -v /usr/local/lib/libddwaf.so ${{ github.workspace }}/native_libs/linux/x86_64 && - cp -v *.so ${{ github.workspace }}/native_libs/${{ env.libdir }} && - cd ${{ github.workspace }} && - ./gradlew check --info -PuseReleaseBinaries' - shell: bash - - name: Copy binaries and debug symbols - run: | - sudo chown -R $(whoami) ${{ env.tempdir }} - mkdir ${{ env.tempdir }}/packages - cd ${{ env.tempdir }}/packages - docker cp pwaf_java_build:/buildAG/libsqreen_jni.so . - docker cp pwaf_java_build:/buildAG/libsqreen_jni.so.debug . + - name: Build docker image + run: ci/build build-docker-image --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} + - name: Build JNI bindings + run: ci/build build-bindings-in-docker --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} + - name: Run tests + run: ci/build run-tests-in-docker --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} - name: Save Artifacts uses: actions/upload-artifact@v2 with: - path: ${{ env.tempdir }}/packages + path: build/native_libs name: jni_${{ env.artifactsuff }} Native_binaries_Stage_linux_aarch64_glibc: name: Linux aarch64 (glibc) runs-on: ubuntu-22.04 - needs: - - Native_binaries_Stage_libddwaf_linux_aarch64 env: - dockerfile: ci/manylinux/aarch64 - artifactsuff: linux_glibc-aarch64 - libdir: linux/aarch64/glibc - artifactsDirectory: ${{ github.workspace }}/artifacts + os: linux + arch: aarch64 + libc: glibc + artifactsuff: linux_glibc-aarch64s steps: - name: Checkout repository uses: actions/checkout@v2 - with: - submodules: recursive - clean: true - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - name: Create artifacts directory - run: mkdir -p ${{ env.artifactsDirectory }} - - name: Download libddwaf artifact - uses: actions/download-artifact@v2 - with: - name: libddwaf_linux-aarch64 - path: ${{ env.artifactsDirectory }} - - name: Build docker linux image - run: docker build --platform linux/arm64 ${{ env.dockerfile }} -t linux_cmake_aarch64 - - name: Clean previous docker container - run: docker rm -f pwaf_java_build || true - - name: Build and test JNI binding - run: | - docker run --platform arm64 --name pwaf_java_build -v $(pwd):${{ github.workspace }} linux_cmake_aarch64 bash -e -c 'export VERBOSE=1; - export LIBDDWAF_INSTALL_PREFIX=/; - git config --global --add safe.directory /home/runner/work/AgentJavaNative/AgentJavaNative && - tar --strip-components=1 -xf ${{ env.artifactsDirectory }}/libddwaf-aarch64.tar.gz -C /usr/local/ && - mkdir buildAG && cd buildAG && - cmake ${{ github.workspace }} -DCMAKE_BUILD_TYPE=${{ env.buildType }} -DCMAKE_C_FLAGS="-fno-omit-frame-pointer" && - make -j && - cp -v /usr/local/lib/libddwaf.so ${{ github.workspace }}/native_libs/linux/aarch64 && - cp -v *.so /usr/local/lib/libddwaf.so ${{ github.workspace }}/native_libs/${{ env.libdir }} && - cd ${{ github.workspace }} && - ./gradlew check --info -PuseReleaseBinaries' - shell: bash - - name: Copy binaries and debug symbols - run: | - sudo chown -R $(whoami) "${{ env.tempdir }}" - mkdir ${{ env.tempdir }}/packages - cd ${{ env.tempdir }}/packages - docker cp pwaf_java_build:/buildAG/libsqreen_jni.so . - docker cp pwaf_java_build:/buildAG/libsqreen_jni.so.debug . + - name: Build docker image + run: ci/build build-docker-image --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} + - name: Build JNI bindings + run: ci/build build-bindings-in-docker --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} + - name: Run tests + run: ci/build run-tests-in-docker --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} - name: Save Artifacts uses: actions/upload-artifact@v2 with: - path: ${{ env.tempdir }}/packages + path: build/native_libs name: jni_${{ env.artifactsuff }} - - name: Cleanup - run: sudo git clean -ffdx Native_binaries_Stage_linux_aarch64_musl: name: Linux aarch64 (musl) runs-on: ubuntu-22.04 - needs: - - Native_binaries_Stage_libddwaf_linux_aarch64 env: - dockerfile: ci/alpine - artifactsuff: linux_musl-aarch64 - libdir: linux/aarch64/musl - artifactsDirectory: ${{ github.workspace }}/artifacts - steps: # identical to glibc + os: linux + arch: aarch64 + libc: musl + artifactsuff: linux_musl-aarch64s + steps: - name: Checkout repository uses: actions/checkout@v2 - with: - submodules: recursive - clean: true - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - name: Create artifacts directory - run: mkdir -p ${{ env.artifactsDirectory }} - - name: Download libddwaf artifact - uses: actions/download-artifact@v2 - with: - name: libddwaf_linux-aarch64 - path: ${{ env.artifactsDirectory }} - - name: Build docker linux image - run: docker build --platform linux/arm64 ${{ env.dockerfile }} -t linux_cmake - - name: Clean previous docker container - run: docker rm -f pwaf_java_build || true - - name: Build and test with release binaries - run: | - docker run --platform linux/arm64 --name pwaf_java_build -v $(pwd):${{ github.workspace }} linux_cmake bash -e -c 'export VERBOSE=1; - export LIBDDWAF_INSTALL_PREFIX=/; - tar --strip-components=1 -xvf ${{ env.artifactsDirectory }}/libddwaf-aarch64.tar.gz -C /usr/local/ && - mkdir buildAG && cd buildAG && - cmake ${{ github.workspace }} -DCMAKE_BUILD_TYPE=${{ env.buildType }} -DCMAKE_C_FLAGS="-fno-omit-frame-pointer" && - make -j && - cp -v /usr/local/lib/libddwaf.so ${{ github.workspace }}/native_libs/linux/aarch64 && - cp -v *.so ${{ github.workspace }}/native_libs/${{ env.libdir }} && - cd ${{ github.workspace }} && - ./gradlew check --info -PuseReleaseBinaries' - shell: bash - - name: Copy binaries and debug symbols - run: | - sudo chown -R $(whoami) ${{ env.tempdir }} - mkdir ${{ env.tempdir }}/packages - cd ${{ env.tempdir }}/packages - docker cp pwaf_java_build:/buildAG/libsqreen_jni.so . - docker cp pwaf_java_build:/buildAG/libsqreen_jni.so.debug . + - name: Build docker image + run: ci/build build-docker-image --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} + - name: Build JNI bindings + run: ci/build build-bindings-in-docker --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} + - name: Run tests + run: ci/build run-tests-in-docker --os ${{ env.os }} --arch ${{ env.arch }} --libc ${{ env.libc }} - name: Save Artifacts uses: actions/upload-artifact@v2 with: - path: ${{ env.tempdir }}/packages + path: build/native_libs name: jni_${{ env.artifactsuff }} - - name: Cleanup - run: sudo git clean -ffdx Native_binaries_Stage_asan: name: ASAN/static analyzer on Linux @@ -596,14 +214,13 @@ jobs: steps: - name: Checkout project uses: actions/checkout@v2 - with: - submodules: recursive + - name: Checkout libddwaf + run: git clone --depth 1 --branch "$(ci/build get-libddwaf-version)" https://github.com/DataDog/libddwaf.git - name: Install GCC and clang run: | sudo apt-get update sudo apt-get install -y libc++-dev libc++abi-dev libc++abi1 libstdc++-12-dev gcc g++ \ clang clang-tools ruby - shell: bash - name: Build libddwaf run: | set -ex @@ -619,29 +236,16 @@ jobs: -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=address -fsanitize=undefined -fsanitize=leak" VERBOSE=1 make -j DESTDIR=out make install - shell: bash - name: Run static analyzer run: | ci/static_analysis - shell: bash - - name: Cache Gradle artifacts - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - name: Run Binding Tests run: | set -ex - VERBOSE=1 ./gradlew --build-cache buildNativeLibDebug -PwithASAN --info + VERBOSE=1 ./gradlew --build-cache buildNativeLib -PwithASAN -PlibddwafPath=libddwaf/Debug/out/usr/local --info ASAN_OPTIONS="verbosity=1 handle_segv=0 fast_unwind_on_malloc=0 detect_leaks=0" \ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 \ - ./gradlew --build-cache -x buildNativeLibDebug --info test - ./gradlew --stop - shell: bash + ./gradlew --build-cache -x buildNativeLib --info test Jar_File_Stage_build_jar: name: Build & Publish runs-on: ubuntu-20.04 @@ -656,8 +260,6 @@ jobs: - Native_binaries_Stage_linux_aarch64_glibc - Native_binaries_Stage_linux_aarch64_musl - Native_binaries_Stage_asan - - Native_binaries_Stage_libddwaf_linux_x86_64 - - Native_binaries_Stage_libddwaf_linux_aarch64 steps: - name: Setup JDK 1.8 uses: actions/setup-java@v1 diff --git a/.gitmodules b/.gitmodules index c471ab8f..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "libddwaf"] - path = libddwaf - url = https://github.com/DataDog/libddwaf.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 7553c4f8..e4beaaea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,8 @@ else() message(FATAL_ERROR "Found JNI: failed") endif() +add_subdirectory(deps EXCLUDE_FROM_ALL) + find_package(libddwaf REQUIRED) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -36,8 +38,6 @@ if(CMAKE_C_COMPILER_ID STREQUAL "Clang") -Wno-implicit-fallthrough -Wno-gnu-auto-type -Wno-c++98-compat) endif() -add_subdirectory(deps EXCLUDE_FROM_ALL) - set(SOURCE_FILES src/main/c/base64.c src/main/c/byte_buffer.c diff --git a/README.md b/README.md index 8b95fd05..ec64a9e8 100644 --- a/README.md +++ b/README.md @@ -1,72 +1,69 @@ -# Introduction +# libddwaf-java -This project provides Java bindings for [libddwaf][libddwaf_repos] through JNI. -Currently supported runtimes are Linux (GNU and musl) and Windows amd64. +## Overview -# Build instructions +This project provides Java bindings for [libddwaf](https://github.com/DataDog/libddwaf) through JNI. There are 3 components: -Gradle is used to build the Java component, and CMake to build the native -components. Three components need to be built separately: +* libddwaf itself +* libsqreen\_jni, the JNI binding (native side), uses cmake +* a jar with the Java API, uses gradle -* libddwaf needs to be built and installed. -* Then the JNI binding (named libsqreen\_jni, for historical reasons). -* Finally the jar can be built with Gradle. +## Build -A jar for release purposes needs to have both the libddwaf and libsqreen -binaries for all the supported runtime environments. These are included inside -the JAR. For Linux, a shared library build of libddwaf (libddwaf.so) is included -in JAR; this DSO is used in both musl and glibc systems. On Windows, there is -only one binary, which is linked against libddwaf built as a static library. +### Simple build -For testing purposes, Gradle can invoke a debug build of libsqreen\_jni. By -default, libddwaf CMake config files are searched in -`libddwaf/Debug/out/usr/local/share/cmake/libddwaf`. +To build the JNI bindings, you can run: -The binaries for inclusion in the final release jar have to be copied into the -`native_libs` directory. Then the jar can be built with `gradle build`. +```sh +# Build JNI bindings for local testing, uses cmake under the hood +./gradlew buildNativeLib --rerun-tasks + +# Or, in recent macOS you might need to specify the architecture (x86_64, arm64): +./gradlew buildNativeLib --rerun-tasks -PmacArch=arm64 +``` -For development purposes, a debug build of libddwaf can be obtained with: +Then all tests can be run as follows: ```sh -cd libddwaf -mkdir Debug && cd Debug -cmake .. -DCMAKE_BUILD_TYPE=Debug -make -j -DESTDIR=out make install -cd ../.. +./gradlew check ``` -Then the jni lib can be built with `./gradlew buildNativeLibDebug --rerun-tasks`. - -To build native libraries, recommended use `--rerun-tasks` option to enforce rebuild task, because of gradle can skip building tasks due caches and lead to unexpected results. +### Build with custom libddwaf -Tests are run with `./gradlew check`. This implicitly invokes -`buildNativeLibDebug` if needed. +By default, cmake will download the required libddwaf build for the current sysem and use it. In some cases, you may +want to use a custom libddwaf build. This is required to use an unreleased version of libddwaf or to use custom build +arguments (e.g. ASAN). -### On MacOS libddwaf can be built with: +So, here's an example building a debug version of libddwaf from master: -You can set MacOSX SDK version with `CMAKE_OSX_SYSROOT` option (13.3) ```sh +git clone git@github.com:DataDog/libddwaf.git cd libddwaf mkdir Debug && cd Debug -cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_OSX_SYSROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX13.3.sdk +cmake .. -DCMAKE_BUILD_TYPE=Debug make -j DESTDIR=out make install cd ../.. ``` -If you need specify architecture (x86_64, arm64) with gradle parameter `macArch` + +Then you can build the JNI bindings with the following command: + ```sh - ./gradlew buildNativeLibDebug --rerun-tasks -PmacArch=arm64 +./gradlew buildNativeLib -PlibddwafPath=libddwaf/Debug/out/usr/local --rerun-tasks ``` -### On Windows libddwaf can be built with: +## Release build -```sh -cd libddwaf -mkdir Debug && cd Debug -cmake .. -DCMAKE_INSTALL_PREFIX=out\usr\local -A x64 -DCMAKE_BUILD_TYPE=Debug -cmake --build . --target install -j --config Debug +A jar for release purposes needs to have both the libddwaf and libsqreen +binaries for all the supported runtime environments. These are included inside +the JAR. For Linux, a shared library build of libddwaf (libddwaf.so) is included +in JAR; this DSO is used in both musl and glibc systems. On Windows, there is +only one binary, which is linked against libddwaf built as a static library. -``` +The binaries for inclusion in the final release jar have to be copied into the +`build/native_libs` directory. Then the jar can be built with `gradle build`. + +To build native libraries, recommended use `--rerun-tasks` option to enforce rebuild task, because of gradle can skip building tasks due caches and lead to unexpected results. + +Tests in reease mode are run with `./gradlew check -Prelease`. -[libddwaf_repos]: https://github.com/DataDog/libddwaf diff --git a/build.gradle b/build.gradle index fb3e2fb0..150e9426 100644 --- a/build.gradle +++ b/build.gradle @@ -38,73 +38,69 @@ targetCompatibility = 1.8 def WINDOWS = org.gradle.internal.os.OperatingSystem.current().windows def MACOS = org.gradle.internal.os.OperatingSystem.current().macOsX +def IS_RELEASE = project.hasProperty("release") +def cmakeBuildDir = IS_RELEASE? "$buildDir/RelWithDebInfo" : "$buildDir/Debug" -task cmakeNativeLibDebug(type: Exec) { +task cmakeNativeLib(type: Exec) { outputs.upToDateWhen { false } - description = 'Runs cmake in a debug configuration' - - def pwafConfDir = project.hasProperty('libddwafConfig') ? - project.libddwafConfig : - "$projectDir/libddwaf/Debug/out/usr/local/share/cmake/libddwaf" + description = 'Runs cmake' doFirst { - def f = file("$pwafConfDir/libddwaf-config-debug.cmake") - if (!f.exists()) { - logger.warn("The file $f does not exist. libddwaf debug must be installed globally. " + - "Run with -PlibddwafConfig=/path/to/dir/of/libddwaf-config-debug.cmake to override") - } - - file("$buildDir/Debug").mkdirs() + file(cmakeBuildDir).mkdirs() } inputs.file 'CMakeLists.txt' - outputs.dir "$buildDir/Debug/" + outputs.dir cmakeBuildDir + workingDir cmakeBuildDir logging.captureStandardError LogLevel.ERROR logging.captureStandardOutput LogLevel.INFO - def cmakeOpts + def cmakeOpts = [] if (project.hasProperty('withASAN')) { - cmakeOpts = [ + cmakeOpts += [ '-DCMAKE_C_COMPILER=clang', '-DCMAKE_CXX_COMPILER=clang++', '-DCMAKE_C_FLAGS=-fsanitize=address -fsanitize=undefined -fsanitize=leak --coverage', '-DCMAKE_SHARED_LINKER_FLAGS=-fsanitize=address -fsanitize=undefined -fsanitize=leak -lubsan -lasan --coverage', ] - if (WINDOWS) { - logger.warn('Address sanitizer not supported on Windows') - } - } else { - cmakeOpts = [ - '-DCMAKE_C_FLAGS=--coverage', - '-DCMAKE_SHARED_LINKER_FLAGS=--coverage', - ] + assert !WINDOWS, 'Address sanitizer not supported on Windows' } + if (project.hasProperty('macArch')) { cmakeOpts += ["-DCMAKE_OSX_ARCHITECTURES=${project.macArch}"] } - if (WINDOWS) { - commandLine 'cmake', "-DCMAKE_PREFIX_PATH=$pwafConfDir", '-S', projectDir, '-B', '.' + if (IS_RELEASE) { + cmakeOpts += ["-DCMAKE_BUILD_TYPE=RelWithDebInfo"] } else { - commandLine('cmake', '-DCMAKE_BUILD_TYPE=Debug', "-DCMAKE_PREFIX_PATH=$pwafConfDir", - *cmakeOpts, '-S', projectDir, '-B', '.') + cmakeOpts += [ + '-DCMAKE_BUILD_TYPE=Debug', + '-DCMAKE_C_FLAGS=--coverage', + '-DCMAKE_SHARED_LINKER_FLAGS=--coverage', + ] + } + + cmakeOpts += ['-DCMAKE_C_FLAGS="-fno-omit-frame-pointer"'] + + if (project.hasProperty('libddwafPath')) { + cmakeOpts += ["-DLIBDDWAF_CUSTOM_DIR=${project.libddwafPath}"] } - workingDir "$buildDir/Debug" + commandLine('cmake', *cmakeOpts, '-S', projectDir, '-B', '.') } -task buildNativeLibDebug(type: Exec) { - description = 'Builds the native JNI lib in a debug configuration' +task buildNativeLib(type: Exec) { + description = 'Builds the native JNI lib' group = 'test' inputs.dir 'src/main/c' if (WINDOWS) { - outputs.file "$buildDir/Debug/Debug/libsqreen_jni.dll" + outputs.file "$cmakeBuildDir/Debug/libsqreen_jni.dll" } else if (MACOS) { - outputs.file "$buildDir/Debug/libsqreen_jni.dylib" + outputs.file "$cmakeBuildDir/libsqreen_jni.dylib" } else { - outputs.file "$buildDir/Debug/libsqreen_jni.so" + outputs.file "$cmakeBuildDir/libsqreen_jni.so" } logging.captureStandardError LogLevel.ERROR @@ -116,9 +112,9 @@ task buildNativeLibDebug(type: Exec) { } else { commandLine 'make', '-j', 'sqreen_jni', 'VERBOSE=1' } - workingDir "$buildDir/Debug" + workingDir cmakeBuildDir - dependsOn cmakeNativeLibDebug + dependsOn cmakeNativeLib } jacocoTestReport { @@ -159,7 +155,7 @@ task sourcesJar(type: Jar, dependsOn: classes) { from sourceSets.main.allSource } -def nativeLibsDir = "$projectDir/native_libs" +def nativeLibsDir = "$projectDir/build/native_libs" task checkForNativeLibs { doLast { @@ -262,8 +258,8 @@ task copyNativeLibs(type: Copy) { ['linux/x86_64/musl/libsqreen_jni.so', 'linux/x86_64/libddwaf.so'], ['linux/aarch64/glibc/libsqreen_jni.so', 'linux/aarch64/libddwaf.so'], ['linux/aarch64/musl/libsqreen_jni.so', 'linux/aarch64/libddwaf.so'], - ['macos/x86_64/libsqreen_jni.dylib', 'macos/x86_64/libddwaf.dylib.dwarf'], - ['macos/aarch64/libsqreen_jni.dylib', 'macos/aarch64/libddwaf.dylib.dwarf'], + ['macos/x86_64/libsqreen_jni.dylib', 'macos/x86_64/libddwaf.dylib'], + ['macos/aarch64/libsqreen_jni.dylib', 'macos/aarch64/libddwaf.dylib'], ['windows/x86_64/sqreen_jni.dll']] doFirst { @@ -307,13 +303,13 @@ tasks.withType(Test).each { } it.jvmArgs += ['-Xcheck:jni', '-Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG', '-DPOWERWAF_EXIT_ON_LEAK=true'] - if (project.hasProperty('useReleaseBinaries')) { + if (project.hasProperty('release')) { it.jvmArgs += ['-DuseReleaseBinaries=true'] it.dependsOn copyNativeLibs } else { - def javaLibPath = WINDOWS ? "$buildDir\\Debug\\Debug" : "$buildDir/Debug" + def javaLibPath = WINDOWS ? "$cmakeBuildDir\\Debug" : "$cmakeBuildDir" it.jvmArgs += ["-Djava.library.path=$javaLibPath"] - it.dependsOn buildNativeLibDebug + it.dependsOn buildNativeLib } it.outputs.upToDateWhen { false } diff --git a/ci/alpine-abuild-libc++/.gitignore b/ci/alpine-abuild-libc++/.gitignore deleted file mode 100644 index 84c048a7..00000000 --- a/ci/alpine-abuild-libc++/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/build/ diff --git a/ci/alpine-abuild-libc++/APKBUILD b/ci/alpine-abuild-libc++/APKBUILD deleted file mode 100644 index c99d38dd..00000000 --- a/ci/alpine-abuild-libc++/APKBUILD +++ /dev/null @@ -1,66 +0,0 @@ -# Maintener: Anatol Pomozov -# Contributor: Llewelyn Trahaearn -# Contributor: Daniel Micay -# Contributor: MThinkCpp - -pkgname=libc++ -pkgver=15.0.6 -pkgrel=2 -url="https://libcxx.llvm.org/" -license=('custom:Apache 2.0 with LLVM Exception') -pkgdesc='LLVM C++ standard library.' -arch='all' -makedepends=('clang' 'cmake' 'samurai' 'python3' 'linux-headers' 'compiler-rt') -checkdepends=('llvm') -options=(!lto 'staticlibs' !strip) -source=("https://github.com/llvm/llvm-project/releases/download/llvmorg-$pkgver/llvm-project-$pkgver.src.tar.xz"{,.sig}) -sha512sums=('6fc6eeb60fac698702d1aac495fc0161eb7216a1f8db2020af8fccec5837831f7cc20dc2a169bf4f0b5f520748280b4a86621f3697d622aa58faaa45dbfaad13' - 'SKIP') -validpgpkeys=('474E22316ABF4785A88C6E8EA2C794A986419D8A') # Tom Stellard - -prepare() { - mkdir -p build -} - -build() { - cd build - - cmake \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_C_FLAGS=-fno-omit-frame-pointer \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_CXX_FLAGS=-fno-omit-frame-pointer \ - \ - -DLIBUNWIND_ENABLE_SHARED=OFF \ - -DLIBUNWIND_USE_COMPILER_RT=ON \ - \ - -DLIBCXXABI_ENABLE_SHARED=ON \ - -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ - -DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \ - -DLIBCXXABI_USE_COMPILER_RT=ON \ - \ - -DLIBCXX_ENABLE_SHARED=OFF \ - -DLIBCXX_HAS_MUSL_LIBC=ON \ - -DLIBCXX_USE_COMPILER_RT=ON \ - -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ - \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ - -DLLVM_EXTERNAL_LIT=/usr/bin/lit \ - ../llvm-project-$pkgver.src/runtimes - CXX_FLAGS="-Wno-suggest-override" ninja unwind cxx cxxabi -} - -check() { - #ninja -C build check-cxx check-cxxabi - true -} - -package () { - DESTDIR="$pkgdir" ninja -C src/build install-unwind install-cxxabi install-cxx -} -sha512sums=" -3311d85f4e02610af52f06e83c8c6d2b93950d24324b831b7afc47c39a20a546d39683a14e1a315da87d226379042de900a3f36a0351053547482af9035ab949 llvm-project-15.0.6.src.tar.xz -" diff --git a/ci/alpine-abuild-libc++/Dockerfile b/ci/alpine-abuild-libc++/Dockerfile deleted file mode 100644 index c625c071..00000000 --- a/ci/alpine-abuild-libc++/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM alpine@sha256:8914eb54f968791faf6a8638949e480fef81e697984fba772b3976835194c6d4 AS build_env -RUN apk --no-cache add alpine-sdk coreutils cmake sudo bash samurai python3 linux-headers compiler-rt \ - && adduser -G abuild -g "Alpine Package Builder" -s /bin/ash -D builder \ - && echo "builder ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \ - && mkdir /packages \ - && chown builder:abuild /packages \ - && mkdir -p /var/cache/apk \ - && ln -s /var/cache/apk /etc/apk/cache -COPY /abuilder /bin/ -USER builder -RUN mkdir /home/builder/package && chown builder /home/builder/package -WORKDIR /home/builder/package -ENV REPODEST /packages - -FROM build_env AS build_apk -ADD APKBUILD /home/builder/package/ -#ADD avoid-strtoll_l.patch /home/builder/package/ -RUN abuild-keygen -ain -RUN sudo apk update -RUN sudo rm /bin/ash && sudo ln -s /bin/bash /bin/ash -RUN abuild -r -RUN find /packages - -FROM scratch -ARG ARCH -COPY --from=build_apk /packages/builder/$ARCH/libc++-15.0.6-r2.apk /libc++-15.0.6-r2.$ARCH.apk diff --git a/ci/alpine-abuild-libc++/abuilder b/ci/alpine-abuild-libc++/abuilder deleted file mode 100644 index 29761508..00000000 --- a/ci/alpine-abuild-libc++/abuilder +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env ash - -set -e - -main() { - mkdir -p "$REPODEST" /home/builder/.abuild - abuild-apk update - [ "$RSA_PRIVATE_KEY" ] && { - echo -e "$RSA_PRIVATE_KEY" > "/home/builder/.abuild/$RSA_PRIVATE_KEY_NAME" - export PACKAGER_PRIVKEY="/home/builder/.abuild/$RSA_PRIVATE_KEY_NAME" - } - sudo chown -R builder:abuild /home/builder/package - sudo chown -R builder:abuild $REPODEST - - exec abuild "$@" -} - -main "$@" diff --git a/ci/alpine-abuild-libc++/build_aarch64.sh b/ci/alpine-abuild-libc++/build_aarch64.sh deleted file mode 100755 index e0d04f7b..00000000 --- a/ci/alpine-abuild-libc++/build_aarch64.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -docker buildx build --progress=plain --platform linux/arm64 --build-arg ARCH=aarch64 -f Dockerfile -o build . diff --git a/ci/alpine-abuild-libc++/build_x86_64.sh b/ci/alpine-abuild-libc++/build_x86_64.sh deleted file mode 100755 index b0df04d3..00000000 --- a/ci/alpine-abuild-libc++/build_x86_64.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -docker buildx build --progress=plain --platform linux/x86_64 --build-arg ARCH=x86_64 -f Dockerfile -o build . diff --git a/ci/alpine-libc++-static/aarch64/Dockerfile b/ci/alpine-libc++-static/aarch64/Dockerfile deleted file mode 100644 index 91478c3a..00000000 --- a/ci/alpine-libc++-static/aarch64/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM alpine@sha256:8914eb54f968791faf6a8638949e480fef81e697984fba772b3976835194c6d4 as build_env -RUN apk --no-cache add clang compiler-rt wget cmake make binutils musl-dev git patchelf -RUN mkdir /tmp/apk && cd /tmp/apk && \ - echo 'cc1aaa7d68b51c1b8ac1b9de198f76a93923bf120f9e23f49a9ef591c0752487f0ba678503373ef5899241087745fca4f0cd2ca0aa7ab6f8c2316fc5f0166093 libc++-15.0.6-r2.aarch64.apk' > CHECKSUM && \ - wget -q https://sqreen-ci-java.s3.amazonaws.com/apk/libc%2B%2B-15.0.6-r2.aarch64.apk -O libc++-15.0.6-r2.aarch64.apk && \ - sha512sum -c CHECKSUM && \ - apk add --allow-untrusted libc++-15.0.6-r2.aarch64.apk && \ - rm -rf /tmp/apk - -FROM build_env AS libddwaf_build -COPY . /AgentJavaNative -# NOTES: -# CMAKE_CXX_FLAGS (except -stdlib=libc++) and CMAKE_C_FLAGS are only used to -# pass the cmake compiler sanity checks. They have no effect otherwise due to -# the usage of -nodefaultlibs -# -static-libgcc is used to force libunwind to be compiled statically -# (-l:libwind.a instead of -lunwind), it has nothing to do with libgcc. -# Again, this is only needed for the cmake sanity checks. -RUN mkdir /build && cd /build && cmake /AgentJavaNative/libddwaf \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_CXX_FLAGS="-stdlib=libc++ -Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc -fno-omit-frame-pointer" \ - -DCMAKE_C_FLAGS="-Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc -fno-omit-frame-pointer" \ - -DLIBDDWAF_BUILD_STATIC=OFF \ - -DLIBDDWAF_SHARED_LINKER_FLAGS='-v -nodefaultlibs -Wl,-Bstatic -lc++ /usr/lib/clang/15.0.7/lib/linux/libclang_rt.builtins-aarch64.a -lunwind -Wl,-Bdynamic -lc /usr/lib/clang/15.0.7/lib/linux/libclang_rt.builtins-aarch64.a' -# Link line will be: -# "/usr/bin/ld" -z now -z relro --hash-style=gnu --eh-frame-hdr -m elf_x86_64 -shared -o libddwaf.so /usr/bin/../lib/crti.o /usr/lib/clang/11.1.0/lib/linux/clang_rt.crtbegin-x86_64.o -L/usr/bin/../lib -L/lib -L/usr/lib -soname libddwaf.so CMakeFiles/libddwaf_objects.dir/src/Clock.cpp.o CMakeFiles/libddwaf_objects.dir/src/parameter.cpp.o CMakeFiles/libddwaf_objects.dir/src/PowerWAF.cpp.o CMakeFiles/libddwaf_objects.dir/src/PowerWAFInterface.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWAdditive.cpp.o CMakeFiles/libddwaf_objects.dir/src/object.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWManifest.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWProcessor.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWRet.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWRetriever.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWRule.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWRuleManager.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWTransformer.cpp.o CMakeFiles/libddwaf_objects.dir/src/radixlib.c.o CMakeFiles/libddwaf_objects.dir/src/utils.cpp.o CMakeFiles/libddwaf_objects.dir/src/log.cpp.o CMakeFiles/libddwaf_objects.dir/src/rule_processor/IPWRuleProcessor.cpp.o CMakeFiles/libddwaf_objects.dir/src/rule_processor/perf_match.cpp.o CMakeFiles/libddwaf_objects.dir/src/rule_processor/re2.cpp.o -lrt -ldl -lm --no-undefined -version-script=/libddwaf/libddwaf.version --build-id=0xa75fbdf483c3016d8ae0acccea972956fe1121fe third_party/lib/libre2.a third_party/liblib_ac.a third_party/liblib_injection.a -Bstatic -lc++ /usr/lib/clang/11.1.0/lib/linux/libclang_rt.builtins-x86_64.a -lunwind -Bdynamic -lc /usr/lib/clang/11.1.0/lib/linux/libclang_rt.builtins-x86_64.a -lpthread /usr/lib/clang/11.1.0/lib/linux/clang_rt.crtend-x86_64.o /usr/bin/../lib/crtn.o -# The apparent dynamic linking to librt, libm, libpthread doesn't happen -# because in musl these are all empty libraries. The symbols live in libc, and -# will be imported. When linking to the shared library in a glibc system, -# -lm, -ldl, -lpthread need to be provided though -RUN cd /build && VERBOSE=1 make -j -RUN patchelf --remove-needed libc.musl-aarch64.so.1 /build/libddwaf.so -RUN cd /build && make package && mv libddwaf-*.tar.gz libddwaf-aarch64.tar.gz - -FROM scratch -COPY --from=libddwaf_build /build/libddwaf-aarch64.tar.gz / diff --git a/ci/alpine-libc++-static/x86_64/Dockerfile b/ci/alpine-libc++-static/x86_64/Dockerfile deleted file mode 100644 index 5ccd6cdc..00000000 --- a/ci/alpine-libc++-static/x86_64/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM alpine@sha256:8914eb54f968791faf6a8638949e480fef81e697984fba772b3976835194c6d4 as build_env -RUN apk --no-cache add clang compiler-rt compiler-rt wget cmake make binutils musl-dev git patchelf -RUN mkdir /tmp/apk && cd /tmp/apk && \ - echo 'db4c0012fabaf1aa86455d3e1cf8f15176429a799455b86d75ce450b75863dcef3fefcf183fc50ba977517398b7bf4e185820f3ebc1627456df9a2e1206d2013 libc++-15.0.6-r2.x86_64.apk' > CHECKSUM && \ - wget -q https://sqreen-ci-java.s3.amazonaws.com/apk/libc%2B%2B-15.0.6-r2.x86_64.apk -O libc++-15.0.6-r2.x86_64.apk && \ - sha512sum -c CHECKSUM && \ - apk add --allow-untrusted libc++-15.0.6-r2.x86_64.apk && \ - rm -rf /tmp/apk - -FROM build_env AS libddwaf_build -COPY .. /AgentJavaNative -# NOTES: -# CMAKE_CXX_FLAGS (except -stdlib=libc++) and CMAKE_C_FLAGS are only used to -# pass the cmake compiler sanity checks. They have no effect otherwise due to -# the usage of -nodefaultlibs -# -static-libgcc is used to force libunwind to be compiled statically -# (-l:libwind.a instead of -lunwind), it has nothing to do with libgcc. -# Again, this is only needed for the cmake sanity checks. -RUN mkdir /build && cd /build && cmake /AgentJavaNative/libddwaf \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_CXX_FLAGS="-stdlib=libc++ -Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc -fno-omit-frame-pointer" \ - -DCMAKE_C_FLAGS="-Qunused-arguments -rtlib=compiler-rt -unwindlib=libunwind -static-libgcc -fno-omit-frame-pointer" \ - -DLIBDDWAF_BUILD_STATIC=OFF \ - -DLIBDDWAF_SHARED_LINKER_FLAGS='-v -nodefaultlibs -Wl,-Bstatic -lc++ /usr/lib/clang/15.0.7/lib/linux/libclang_rt.builtins-x86_64.a -lunwind -Wl,-Bdynamic -lc /usr/lib/clang/15.0.7/lib/linux/libclang_rt.builtins-x86_64.a' -# Link line will be: -# "/usr/bin/ld" -z now -z relro --hash-style=gnu --eh-frame-hdr -m elf_x86_64 -shared -o libddwaf.so /usr/bin/../lib/crti.o /usr/lib/clang/15.0.6/lib/linux/clang_rt.crtbegin-x86_64.o -L/usr/bin/../lib -L/lib -L/usr/lib -soname libddwaf.so CMakeFiles/libddwaf_objects.dir/src/Clock.cpp.o CMakeFiles/libddwaf_objects.dir/src/parameter.cpp.o CMakeFiles/libddwaf_objects.dir/src/PowerWAF.cpp.o CMakeFiles/libddwaf_objects.dir/src/PowerWAFInterface.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWAdditive.cpp.o CMakeFiles/libddwaf_objects.dir/src/object.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWManifest.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWProcessor.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWRet.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWRetriever.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWRule.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWRuleManager.cpp.o CMakeFiles/libddwaf_objects.dir/src/PWTransformer.cpp.o CMakeFiles/libddwaf_objects.dir/src/radixlib.c.o CMakeFiles/libddwaf_objects.dir/src/utils.cpp.o CMakeFiles/libddwaf_objects.dir/src/log.cpp.o CMakeFiles/libddwaf_objects.dir/src/rule_processor/IPWRuleProcessor.cpp.o CMakeFiles/libddwaf_objects.dir/src/rule_processor/perf_match.cpp.o CMakeFiles/libddwaf_objects.dir/src/rule_processor/re2.cpp.o -lrt -ldl -lm --no-undefined -version-script=/libddwaf/libddwaf.version --build-id=0xa75fbdf483c3016d8ae0acccea972956fe1121fe third_party/lib/libre2.a third_party/liblib_ac.a third_party/liblib_injection.a -Bstatic -lc++ /usr/lib/clang/15.0.6/lib/linux/libclang_rt.builtins-x86_64.a -lunwind -Bdynamic -lc /usr/lib/clang/15.0.6/lib/linux/libclang_rt.builtins-x86_64.a -lpthread /usr/lib/clang/15.0.6/lib/linux/clang_rt.crtend-x86_64.o /usr/bin/../lib/crtn.o -# The apparent dynamic linking to librt, libm, libpthread doesn't happen -# because in musl these are all empty libraries. The symbols live in libc, and -# will be imported. When linking to the shared library in a glibc system, -# -lm, -ldl, -lpthread need to be provided though -RUN cd /build && VERBOSE=1 make -j -RUN patchelf --remove-needed libc.musl-x86_64.so.1 /build/libddwaf.so -RUN cd /build && make package && mv libddwaf-*.tar.gz libddwaf-x86_64.tar.gz - -FROM scratch -COPY --from=libddwaf_build /build/libddwaf-x86_64.tar.gz / diff --git a/ci/build b/ci/build new file mode 100755 index 00000000..cbf698d5 --- /dev/null +++ b/ci/build @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly OUTPUT_NATIVE_LIBS_DIR="build/native_libs" + +function cmd_build-docker-image() { + echo "Building docker image..." + docker build --platform "$OS/$ARCH" "$DOCKER_IMAGE" -t "libddwaf-build-$LIBC" +} + +function cmd_build-bindings-in-docker() { + echo "Building JNI bindings in Docker..." + docker run \ + --rm \ + --platform "$OS/$ARCH" \ + --user "$(id -u):$(id -g)" \ + -v "$(pwd):/work" \ + -w /work \ + "libddwaf-build-$LIBC" \ + ./ci/build build-bindings --os $OS --arch $ARCH --libc $LIBC +} + +function cmd_build-bindings() { + echo "Building JNI bindings..." + project_dir="$(pwd)" + mkdir -p "$OUTPUT_NATIVE_LIBS_DIR" + mkdir -p /tmp/build + cd /tmp/build + + cmake_args=("-DCMAKE_BUILD_TYPE=RelWithDebInfo") + + c_flags= + if [[ $OS != windows ]]; then + c_flags="$c_flags -fno-omit-frame-pointer" + fi + if [[ $OS = macos ]]; then + c_flags="$c_flags -mmacosx-version-min=11.6" + fi + cmake_args+=("-DCMAKE_C_FLAGS=$c_flags") + cmake_args+=("-DCMAKE_CXX_FLAGS=$c_flags") + + if [[ $OS = macos && $ARCH = aarch64 ]]; then + cmake_args+=("-DCMAKE_OSX_ARCHITECTURES=arm64") + fi + + generator="Unix Makefiles" + if [[ $OS = windows ]]; then + generator="NMake Makefiles" + fi + + cmake \ + "${cmake_args[@]}" \ + -G "$generator" \ + -S "$project_dir" \ + -B . + + cmake --build . + + if [[ $OS = linux ]]; then + output_path="$project_dir/$OUTPUT_NATIVE_LIBS_DIR/$OS/$ARCH/$LIBC" + mkdir -p "$output_path" + cp libsqreen_jni.so "$output_path/" + cp libsqreen_jni.so.debug "$output_path/" + cp _deps/libddwaf-src/lib/libddwaf.so "$project_dir/$OUTPUT_NATIVE_LIBS_DIR/$OS/$ARCH/" + elif [[ $OS = macos ]]; then + output_path="$project_dir/$OUTPUT_NATIVE_LIBS_DIR/$OS/$ARCH" + mkdir -p "$output_path" + cp libsqreen_jni.dylib "$output_path/" + cp libsqreen_jni.dylib.dwarf "$output_path/" + cp _deps/libddwaf-src/lib/libddwaf.dylib "$output_path/" + # TODO: .dwarf file is not generated on macOS for libddwaf releases + # cp _deps/libddwaf-src/lib/libddwaf.dylib.dwarf "$output_path/" + elif [[ $OS = windows ]]; then + output_path="$project_dir/$OUTPUT_NATIVE_LIBS_DIR/$OS/$ARCH" + mkdir -p "$output_path" + cp sqreen_jni.dll "$output_path/" + cp sqreen_jni.pdb "$output_path/" + cp _deps/libddwaf-src/lib/ddwaf.dll "$output_path/" + else + echo "Invalid OS: $OS" + exit 1 + fi +} + +function cmd_run-tests-in-docker() { + echo "Run tests in Docker..." + docker run \ + --rm \ + --platform "$OS/$ARCH" \ + --user "$(id -u):$(id -g)" \ + -v "$(pwd):/work" \ + -w /work \ + "libddwaf-build-$LIBC" \ + ./ci/build run-tests --os "$OS" --arch "$ARCH" --libc "${LIBC:-}" +} + +function cmd_run-tests() { + echo "Run tests..." + ./gradlew check --info -Prelease +} + +function get_libddwaf_version() { + grep -E 'set\(LIBDDWAF_VERSION\s+[0-9.a-zA-Z]+\)' deps/CMakeLists.txt | sed -e 's~.*\s\(.*\)).*~\1~g' +} + +function cmd_get-libddwaf-version() { + get_libddwaf_version +} + +COMMAND="$1" +shift + +# Parse options +while [[ $# -gt 0 ]]; do + case "$1" in + --arch) + ARCH="$2" + shift 2 + ;; + --os) + OS="$2" + shift 2 + ;; + --libc) + LIBC="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +case "${OS:-}-${ARCH:-}-${LIBC:-}" in + linux-*-glibc) + DOCKER_IMAGE="ci/manylinux/$ARCH" + ;; + linux-*-musl) + DOCKER_IMAGE="ci/alpine" + ;; + *) + ;; +esac + +# check if function cmd_$COMMAND exists, and then run it: +if declare -F "cmd_$COMMAND" &> /dev/null; then + "cmd_$COMMAND" "$@" +else + echo "Bad command: $COMMAND" + exit 1 +fi diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 356c5cc9..fed7bf71 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.12.0) +cmake_minimum_required(VERSION 3.14.0) include(FetchContent) set(ZLIB_VERSION v1.3) @@ -15,4 +15,46 @@ endif() target_compile_definitions(zlibstatic PUBLIC ZLIB_CONST=1) target_include_directories(zlibstatic INTERFACE ${zlib_SOURCE_DIR} ${zlib_BINARY_DIR}) +set(LIBDDWAF_VERSION 1.16.0) +set(LIBDDWAF_VERSION ${LIBDDWAF_VERSION} PARENT_SCOPE) +if(DEFINED LIBDDWAF_CUSTOM_DIR) + set(libddwaf_DIR ${LIBDDWAF_CUSTOM_DIR}/share/cmake/libddwaf PARENT_SCOPE) +else() + if(CMAKE_SYSTEM_NAME STREQUAL Linux) + if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) + set(LIBDDWAF_VARIANT x86_64-linux-musl) + # i386? + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL i386) + set(LIBDDWAF_VARIANT i386-linux-musl) + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) + set(LIBDDWAF_VARIANT aarch64-linux-musl) + else() + message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}") + endif() + elseif (CMAKE_SYSTEM_NAME STREQUAL Darwin) + if (CMAKE_SYSTEM_PROCESSOR STREQUAL arm64 OR CMAKE_OSX_ARCHITECTURES STREQUAL arm64) + set(LIBDDWAF_VARIANT darwin-arm64) + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) + set(LIBDDWAF_VARIANT darwin-x86_64) + else() + message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}") + endif() + elseif (CMAKE_SYSTEM_NAME STREQUAL Windows) + if(CMAKE_SYSTEM_PROCESSOR STREQUAL AMD64) + set(LIBDDWAF_VARIANT windows-x64) + else() + message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}") + endif() + else() + message(FATAL_ERROR "Unsupported platform: ${CMAKE_SYSTEM_NAME}") + endif() + FetchContent_Declare( + libddwaf + # TODO: gen proper url + URL https://github.com/DataDog/libddwaf/releases/download/${LIBDDWAF_VERSION}/libddwaf-${LIBDDWAF_VERSION}-${LIBDDWAF_VARIANT}.tar.gz + ) + FetchContent_MakeAvailable(libddwaf) + set(libddwaf_DIR ${libddwaf_SOURCE_DIR}/share/cmake/libddwaf PARENT_SCOPE) +endif() + # vim: set et: diff --git a/libddwaf b/libddwaf deleted file mode 160000 index 044f6752..00000000 --- a/libddwaf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 044f67522ec5df679fed4786d0ae965170bd63c1