diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..c539508e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,258 @@ +name: VariantValidator CI + +on: + push: + + pull_request: + branches: + - main + - develop + +jobs: + + test: + + runs-on: ubuntu-latest + timeout-minutes: 60 + + env: + DOCKER_NETWORK: variantvalidator-network + + steps: + + ################################################################## + # Free disk space + ################################################################## + + - name: Free disk space + run: | + echo "===== Before cleanup =====" + df -h + + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/hostedtoolcache + + docker system prune -af || true + + echo + echo "===== After cleanup =====" + df -h + + ################################################################## + # Checkout + ################################################################## + + - uses: actions/checkout@v4 + + ################################################################## + # Create Docker network + ################################################################## + + - name: Create Docker network + run: docker network create ${DOCKER_NETWORK} + + ################################################################## + # Build VVTA + ################################################################## + + - name: Build VVTA PostgreSQL + run: | + docker build \ + --no-cache \ + -f db_dockerfiles/vvta/Dockerfile \ + -t postgres-vvta \ + db_dockerfiles/vvta + + - name: Run VVTA PostgreSQL + run: | + docker run -d \ + --name vv-vvta \ + --network ${DOCKER_NETWORK} \ + --shm-size=2g \ + postgres-vvta + + ################################################################## + # Build Validator DB + ################################################################## + + - name: Build Validator MySQL + run: | + docker build \ + --no-cache \ + -f db_dockerfiles/vdb/Dockerfile \ + -t mysql-validator \ + db_dockerfiles/vdb + + - name: Run Validator MySQL + run: | + docker run -d \ + --name vv-vdb \ + --network ${DOCKER_NETWORK} \ + mysql-validator + + ################################################################## + # Build SeqRepo + ################################################################## + + - name: Build SeqRepo + run: | + docker build \ + --no-cache \ + -f db_dockerfiles/vvsr/Dockerfile \ + -t seqrepo-validator \ + db_dockerfiles/vvsr + + - name: Run SeqRepo + run: | + docker run -d \ + --name vv-seqrepo \ + --network ${DOCKER_NETWORK} \ + seqrepo-validator + + ################################################################## + # Wait for MySQL + ################################################################## + + - name: Wait for MySQL + run: | + until docker exec vv-vdb \ + mysqladmin ping \ + -u vvadmin \ + -pvar1ant \ + --silent + do + sleep 5 + done + + echo "MySQL ready." + + ################################################################## + # Wait for VVTA initialisation + ################################################################## + + - name: Wait for VVTA initialisation + run: | + until docker logs vv-vvta 2>&1 | \ + grep -q "PostgreSQL init process complete; ready for start up." + do + sleep 10 + done + + echo "VVTA ready." + + ################################################################## + # Build VariantValidator + ################################################################## + + - name: Build VariantValidator + run: | + docker build \ + --no-cache \ + -t variantvalidator \ + . + + ################################################################## + # Run VariantValidator + ################################################################## + + - name: Run VariantValidator + run: | + docker run -d \ + --name variantvalidator \ + --network ${DOCKER_NETWORK} \ + --volumes-from vv-seqrepo:ro \ + variantvalidator + + ################################################################## + # Run tests + ################################################################## + + - name: Run pytest + run: | + docker exec variantvalidator \ + pytest \ + -n 4 \ + --cov=VariantValidator \ + --cov=VariantFormatter \ + --cov-report=xml \ + tests + + ################################################################## + # Copy coverage + ################################################################## + + - name: Locate coverage.xml + if: always() + run: | + docker exec variantvalidator \ + find / -name coverage.xml 2>/dev/null || true + + - name: Copy coverage.xml + if: always() + run: | + docker cp \ + variantvalidator:/app/coverage.xml \ + coverage.xml || true + + ################################################################## + # Upload coverage + ################################################################## + + - name: Upload coverage to Codecov + if: always() + uses: codecov/codecov-action@v5 + with: + files: coverage.xml + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} + + ################################################################## + # Diagnostics + ################################################################## + + - name: Disk usage + if: always() + run: | + echo "===== Disk =====" + df -h + + echo + echo "===== Docker =====" + docker system df + + ################################################################## + # Logs on failure + ################################################################## + + - name: Container logs + if: failure() + run: | + echo "===== VariantValidator =====" + docker logs variantvalidator || true + + echo + echo "===== VVTA =====" + docker logs vv-vvta || true + + echo + echo "===== MySQL =====" + docker logs vv-vdb || true + + echo + echo "===== SeqRepo =====" + docker logs vv-seqrepo || true + + ################################################################## + # Cleanup + ################################################################## + + - name: Cleanup + if: always() + run: | + docker stop variantvalidator vv-seqrepo vv-vdb vv-vvta || true + + docker rm variantvalidator vv-seqrepo vv-vdb vv-vvta || true + + docker network rm ${DOCKER_NETWORK} || true diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index b1786088..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,140 +0,0 @@ -pipeline { - agent { - docker { - image "docker:24.0.6-git" // Set the Docker image for the Jenkins agent - } - } - environment { - CODECOV_TOKEN = credentials('CODECOV_TOKEN') // Use the Codecov token from Jenkins secret - CONTAINER_SUFFIX = "${BUILD_NUMBER}" // Use the build number as a container suffix for uniqueness - DOCKER_NETWORK = "variantvalidator_docker_network-$CONTAINER_SUFFIX" // Create a unique Docker network for this build - DATA_VOLUME = "docker-shared-space" // Define a data volume for shared data - } - stages { - stage("Clone Repository Remove dangling docker components and Create Docker Network") { - steps { - checkout scm // Checkout the source code from the configured source code management system - sh 'docker system prune --all --volumes --force' // Remove unused Docker resources - sh 'docker network create $DOCKER_NETWORK' // Create a Docker network for containers - } - } - stage("Switch to Git Branch") { - steps { - sh "git checkout ${BRANCH_NAME}" - sh "git pull" - } - } - stage("Build and Run VVTA PostgreSQL") { - steps { - script { - def dockerfile = './db_dockerfiles/vvta/Dockerfile' // Define the Dockerfile path - def vvtaContainer = docker.build("postgres-vvta-${CONTAINER_SUFFIX}", "--no-cache -f ${dockerfile} ./db_dockerfiles/vvta") - // Build and run a PostgreSQL container for VVTA - vvtaContainer.run("-p 5432:5432 -d --name vv-vvta --network $DOCKER_NETWORK --shm-size=2g") - sh 'echo Building and running VVTA PostgreSQL' // Display a message - } - } - } - stage("Build and Run Validator MySQL") { - steps { - script { - def dockerfile = './db_dockerfiles/vdb/Dockerfile' // Define the Dockerfile path - def validatorContainer = docker.build("mysql-validator-${CONTAINER_SUFFIX}", "--no-cache -f ${dockerfile} ./db_dockerfiles/vdb") - // Build and run a MySQL container for the Validator - validatorContainer.run("-p 3306:3306 -d --name vv-vdb --network $DOCKER_NETWORK") - sh 'echo Building and running Validator MySQL' // Display a message - } - } - } - stage("Build and Run SeqRepo") { - steps { - script { - def dockerfile = './db_dockerfiles/vvsr/Dockerfile' // Define the Dockerfile path - def seqRepoContainer = docker.build("sqlite-seqrepo-${CONTAINER_SUFFIX}", "--no-cache -f ${dockerfile} ./db_dockerfiles/vvsr") - // Build and run a SQLite SeqRepo container - seqRepoContainer.run("--network $DOCKER_NETWORK --name vv-seqrepo -v $DATA_VOLUME:/usr/local/share:rw") - sh 'echo Building and running SeqRepo' // Display a message - } - } - } - stage("Build and Run VariantValidator") { - steps { - script { - def dockerfile = './Dockerfile' // Define the Dockerfile path - def variantValidatorContainer = docker.build("variantvalidator-${CONTAINER_SUFFIX}", "--no-cache -f ${dockerfile} .") - - // Run variantValidatorContainer and Mount the DATA_VOLUME - variantValidatorContainer.run("-v $DATA_VOLUME:/usr/local/share:rw -d --name variantvalidator --network $DOCKER_NETWORK") - - // Display a message indicating that VariantValidator is being built and run - sh 'echo Building and running VariantValidator' - } - } - } - stage("Run Pytest and Codecov") { - steps { - script { - sh 'docker ps' // List running Docker containers - def connectionSuccessful = false - - for (int attempt = 1; attempt <= 5; attempt++) { - echo "Attempt $attempt to connect to the database..." - def exitCode = sh(script: ''' - docker exec -e PGPASSWORD=uta_admin variantvalidator psql -U uta_admin -d vvta -h vv-vvta -p 5432 - ''', returnStatus: true) - - if (exitCode == 0) { - connectionSuccessful = true - echo "Connected successfully! Running pytest..." - - // Run pytest && Run Codecov with the provided token and branch name - sh 'docker exec variantvalidator pytest -n 3 --cov=VariantValidator --cov=VariantFormatter --cov-report=term tests/' - - // Send coverage report to Codecov - sh 'docker exec variantvalidator codecov -t $CODECOV_TOKEN -b ${BRANCH_NAME}' - - // Check for test failures in the captured output - if (currentBuild.rawBuild.getLog(2000).join('\n').contains("test summary info") && currentBuild.rawBuild.getLog(2000).join('\n').contains("FAILED")) { - failure(message:"Pytest completed with test failures") - } - - // Check the Jenkins console log for pytest exit code - def pytestExitCode = currentBuild.rawBuild.getLog(2000).find { line -> line =~ /.*Pytest exit code: (\d+).*/ } - if (pytestExitCode) { - pytestExitCode = Integer.parseInt(pytestExitCode.replaceAll(/.*Pytest exit code: (\d+).*/, '$1')) - if (pytestExitCode != 0) { - failure(message:"Pytest failed with exit code $pytestExitCode") - } - } - break - } - - echo "Connection failed. Waiting for 60 seconds before the next attempt..." - sleep 60 - } - - if (!connectionSuccessful) { - failure(message:"All connection attempts failed. Exiting...") - } - } - } - } - } - post { - always { // This ensures cleanup is executed regardless of build outcome - script { - // Cleanup Docker - sh 'docker stop vv-vvta' - sh 'docker rm vv-vvta' - sh 'docker stop vv-vdb' - sh 'docker rm vv-vdb' - sh 'docker stop vv-seqrepo' - sh 'docker rm vv-seqrepo' - sh 'docker stop variantvalidator' - sh 'docker rm variantvalidator' - sh 'docker network rm $DOCKER_NETWORK' - sh 'docker system prune --all --volumes --force' - } - } - } -} diff --git a/README.md b/README.md index 8137a1a6..8085a0ed 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # VariantValidator -[![codecov](https://codecov.io/github/openvar/variantValidator/graph/badge.svg?token=QWTxw5kiY4)](https://codecov.io/github/openvar/variantValidator) +[![VariantValidator CI](https://github.com/openvar/variantValidator/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/openvar/variantValidator/actions/workflows/ci.yml) [![codecov](https://codecov.io/github/openvar/variantValidator/graph/badge.svg?token=QWTxw5kiY4)](https://codecov.io/github/openvar/variantValidator) ## About diff --git a/db_dockerfiles/vdb/Dockerfile b/db_dockerfiles/vdb/Dockerfile index e6129686..06e70d48 100644 --- a/db_dockerfiles/vdb/Dockerfile +++ b/db_dockerfiles/vdb/Dockerfile @@ -1,10 +1,10 @@ # Should run on all processors FROM ubuntu/mysql:8.0-22.04_beta -ENV MYSQL_RANDOM_ROOT_PASSWORD yes -ENV MYSQL_DATABASE validator -ENV MYSQL_USER vvadmin -ENV MYSQL_PASSWORD var1ant +ENV MYSQL_RANDOM_ROOT_PASSWORD=yes +ENV MYSQL_DATABASE=validator +ENV MYSQL_USER=vvadmin +ENV MYSQL_PASSWORD=var1ant RUN apt-get update && apt-get install -y \ wget @@ -13,7 +13,7 @@ RUN rm -rf /var/lib/apt/lists/* # Set the max_connections directly in the my.cnf RUN echo '[mysqld]' >> /etc/mysql/my.cnf && \ - echo 'max_connections=250' >> /etc/mysql/my.cnf + echo 'max_connections=1000' >> /etc/mysql/my.cnf RUN wget https://data.variantvalidator.org/vvdata/validator/validator_2025_03.sql.gz -O /docker-entrypoint-initdb.d/validator_2025_03.sql.gz diff --git a/db_dockerfiles/vvsr/Dockerfile b/db_dockerfiles/vvsr/Dockerfile index 1744ff86..9458b089 100644 --- a/db_dockerfiles/vvsr/Dockerfile +++ b/db_dockerfiles/vvsr/Dockerfile @@ -1,17 +1,21 @@ FROM ubuntu:22.04 -RUN apt-get update +RUN apt-get update RUN apt-get install -y wget RUN mkdir -p /usr/local/share/seqdata -RUN wget --output-document=/usr/local/share/seqdata/VV_SR_2025_02.tar https://data.variantvalidator.org/vvdata/vv_seqrepo/VV_SR_2025_02.tar +RUN wget --output-document=/usr/local/share/seqdata/VV_SR_2025_02.tar \ + https://data.variantvalidator.org/vvdata/vv_seqrepo/VV_SR_2025_02.tar -RUN tar -xvf /usr/local/share/seqdata/VV_SR_2025_02.tar --directory /usr/local/share/seqdata +RUN tar -xvf /usr/local/share/seqdata/VV_SR_2025_02.tar \ + --directory /usr/local/share/seqdata RUN rm /usr/local/share/seqdata/VV_SR_2025_02.tar +VOLUME /usr/local/share/seqdata + ENTRYPOINT [] -CMD ["tail", "-f", "/dev/null"] \ No newline at end of file +CMD ["tail", "-f", "/dev/null"]