From c48a6edde5f7a34d767cbfb0734e77db3b756d95 Mon Sep 17 00:00:00 2001
From: Peter-J-Freeman <peter.j.freeman@manchester.ac.uk>
Date: Tue, 23 Jun 2026 16:10:20 +0100
Subject: [PATCH 1/4] Move CI to GitHub CI and remove legacy Jenkinsfile

GitHub CI gives sufficient runners for the project and internalises
CI within the Repo. This commit adds a GitHub workflow for CI built
around Docker. Docker installs the databases and the code and then runs
Pytest, pytest-cov and codecov
---
 .github/workflows/ci.yml       | 252 +++++++++++++++++++++++++++++++++
 Jenkinsfile                    | 140 ------------------
 README.md                      |   2 +-
 db_dockerfiles/vdb/Dockerfile  |   8 +-
 db_dockerfiles/vvsr/Dockerfile |  12 +-
 5 files changed, 265 insertions(+), 149 deletions(-)
 create mode 100644 .github/workflows/ci.yml
 delete mode 100644 Jenkinsfile

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 00000000..befdbb18
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,252 @@
+name: VariantValidator CI
+
+on:
+  push:
+
+  pull_request:
+    branches:
+      - main
+      - develop
+
+jobs:
+
+  test:
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    env:
+      DOCKER_NETWORK: variantvalidator-network
+
+    steps:
+
+      ##################################################################
+      # Free disk space
+      ##################################################################
+
+      - name: Free disk space
+        run: |
+          echo "===== Before cleanup ====="
+          df -h
+
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/hostedtoolcache
+
+          docker system prune -af || true
+
+          echo
+          echo "===== After cleanup ====="
+          df -h
+
+      ##################################################################
+      # Checkout
+      ##################################################################
+
+      - uses: actions/checkout@v4
+
+      ##################################################################
+      # Create Docker network
+      ##################################################################
+
+      - name: Create Docker network
+        run: docker network create ${DOCKER_NETWORK}
+
+      ##################################################################
+      # Build VVTA
+      ##################################################################
+
+      - name: Build VVTA PostgreSQL
+        run: |
+          docker build \
+            --no-cache \
+            -f db_dockerfiles/vvta/Dockerfile \
+            -t postgres-vvta \
+            db_dockerfiles/vvta
+
+      - name: Run VVTA PostgreSQL
+        run: |
+          docker run -d \
+            --name vv-vvta \
+            --network ${DOCKER_NETWORK} \
+            --shm-size=2g \
+            postgres-vvta
+
+      ##################################################################
+      # Build Validator DB
+      ##################################################################
+
+      - name: Build Validator MySQL
+        run: |
+          docker build \
+            --no-cache \
+            -f db_dockerfiles/vdb/Dockerfile \
+            -t mysql-validator \
+            db_dockerfiles/vdb
+
+      - name: Run Validator MySQL
+        run: |
+          docker run -d \
+            --name vv-vdb \
+            --network ${DOCKER_NETWORK} \
+            mysql-validator
+
+      ##################################################################
+      # Build SeqRepo
+      ##################################################################
+
+      - name: Build SeqRepo
+        run: |
+          docker build \
+            --no-cache \
+            -f db_dockerfiles/vvsr/Dockerfile \
+            -t sqlite-seqrepo \
+            db_dockerfiles/vvsr
+
+      - name: Run SeqRepo
+        run: |
+          docker run -d \
+            --name vv-seqrepo \
+            --network ${DOCKER_NETWORK} \
+            sqlite-seqrepo
+
+      ##################################################################
+      # Wait for MySQL
+      ##################################################################
+
+      - name: Wait for MySQL
+        run: |
+          until docker exec vv-vdb \
+            mysqladmin ping \
+            -u vvadmin \
+            -pvar1ant \
+            --silent
+          do
+            sleep 5
+          done
+
+          echo "MySQL ready."
+
+      ##################################################################
+      # Wait for VVTA initialisation
+      ##################################################################
+
+      - name: Wait for VVTA initialisation
+        run: |
+          until docker logs vv-vvta 2>&1 | \
+            grep -q "PostgreSQL init process complete; ready for start up."
+          do
+            sleep 10
+          done
+
+          echo "VVTA ready."
+
+      ##################################################################
+      # Build VariantValidator
+      ##################################################################
+
+      - name: Build VariantValidator
+        run: |
+          docker build \
+            --no-cache \
+            -t variantvalidator \
+            .
+
+      ##################################################################
+      # Run VariantValidator
+      ##################################################################
+
+      - name: Run VariantValidator
+        run: |
+          docker run -d \
+            --name variantvalidator \
+            --network ${DOCKER_NETWORK} \
+            variantvalidator
+
+      ##################################################################
+      # Run tests
+      ##################################################################
+
+      - name: Run pytest
+        run: |
+          docker exec variantvalidator \
+            pytest \
+            -n 4 \
+            --cov=VariantValidator \
+            --cov=VariantFormatter \
+            --cov-report=term-missing \
+            --cov-report=xml \
+            tests
+
+      ##################################################################
+      # Copy coverage
+      ##################################################################
+
+      - name: Copy coverage.xml
+        if: always()
+        run: |
+          docker cp \
+            variantvalidator:/VariantValidator/coverage.xml \
+            coverage.xml
+
+      ##################################################################
+      # Upload coverage
+      ##################################################################
+
+      - name: Upload coverage to Codecov
+        if: always()
+        uses: codecov/codecov-action@v5
+        with:
+          files: coverage.xml
+          fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      ##################################################################
+      # Diagnostics
+      ##################################################################
+
+      - name: Disk usage
+        if: always()
+        run: |
+          echo "===== Disk ====="
+          df -h
+
+          echo
+          echo "===== Docker ====="
+          docker system df
+
+      ##################################################################
+      # Logs on failure
+      ##################################################################
+
+      - name: Container logs
+        if: failure()
+        run: |
+          echo "===== VariantValidator ====="
+          docker logs variantvalidator || true
+
+          echo
+          echo "===== VVTA ====="
+          docker logs vv-vvta || true
+
+          echo
+          echo "===== MySQL ====="
+          docker logs vv-vdb || true
+
+          echo
+          echo "===== SeqRepo ====="
+          docker logs vv-seqrepo || true
+
+      ##################################################################
+      # Cleanup
+      ##################################################################
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker stop variantvalidator vv-seqrepo vv-vdb vv-vvta || true
+
+          docker rm variantvalidator vv-seqrepo vv-vdb vv-vvta || true
+
+          docker network rm ${DOCKER_NETWORK} || true
diff --git a/Jenkinsfile b/Jenkinsfile
deleted file mode 100644
index b1786088..00000000
--- a/Jenkinsfile
+++ /dev/null
@@ -1,140 +0,0 @@
-pipeline {
-    agent {
-        docker {
-            image "docker:24.0.6-git" // Set the Docker image for the Jenkins agent
-        }
-    }
-    environment {
-        CODECOV_TOKEN = credentials('CODECOV_TOKEN') // Use the Codecov token from Jenkins secret
-        CONTAINER_SUFFIX = "${BUILD_NUMBER}" // Use the build number as a container suffix for uniqueness
-        DOCKER_NETWORK = "variantvalidator_docker_network-$CONTAINER_SUFFIX" // Create a unique Docker network for this build
-        DATA_VOLUME = "docker-shared-space" // Define a data volume for shared data
-    }
-    stages {
-        stage("Clone Repository Remove dangling docker components and Create Docker Network") {
-            steps {
-                checkout scm // Checkout the source code from the configured source code management system
-                sh 'docker system prune --all --volumes --force' // Remove unused Docker resources
-                sh 'docker network create $DOCKER_NETWORK' // Create a Docker network for containers
-            }
-        }
-        stage("Switch to Git Branch") {
-            steps {
-                sh "git checkout ${BRANCH_NAME}"
-                sh "git pull"
-            }
-        }
-        stage("Build and Run VVTA PostgreSQL") {
-            steps {
-                script {
-                    def dockerfile = './db_dockerfiles/vvta/Dockerfile' // Define the Dockerfile path
-                    def vvtaContainer = docker.build("postgres-vvta-${CONTAINER_SUFFIX}", "--no-cache -f ${dockerfile} ./db_dockerfiles/vvta")
-                    // Build and run a PostgreSQL container for VVTA
-                    vvtaContainer.run("-p 5432:5432 -d --name vv-vvta --network $DOCKER_NETWORK --shm-size=2g")
-                    sh 'echo Building and running VVTA PostgreSQL' // Display a message
-                }
-            }
-        }
-        stage("Build and Run Validator MySQL") {
-            steps {
-                script {
-                    def dockerfile = './db_dockerfiles/vdb/Dockerfile' // Define the Dockerfile path
-                    def validatorContainer = docker.build("mysql-validator-${CONTAINER_SUFFIX}", "--no-cache -f ${dockerfile} ./db_dockerfiles/vdb")
-                    // Build and run a MySQL container for the Validator
-                    validatorContainer.run("-p 3306:3306 -d --name vv-vdb --network $DOCKER_NETWORK")
-                    sh 'echo Building and running Validator MySQL' // Display a message
-                }
-            }
-        }
-        stage("Build and Run SeqRepo") {
-            steps {
-                script {
-                    def dockerfile = './db_dockerfiles/vvsr/Dockerfile' // Define the Dockerfile path
-                    def seqRepoContainer = docker.build("sqlite-seqrepo-${CONTAINER_SUFFIX}", "--no-cache -f ${dockerfile} ./db_dockerfiles/vvsr")
-                    // Build and run a SQLite SeqRepo container
-                    seqRepoContainer.run("--network $DOCKER_NETWORK --name vv-seqrepo -v $DATA_VOLUME:/usr/local/share:rw")
-                    sh 'echo Building and running SeqRepo' // Display a message
-                }
-            }
-        }
-        stage("Build and Run VariantValidator") {
-            steps {
-                script {
-                    def dockerfile = './Dockerfile' // Define the Dockerfile path
-                    def variantValidatorContainer = docker.build("variantvalidator-${CONTAINER_SUFFIX}", "--no-cache -f ${dockerfile} .")
-
-                    // Run variantValidatorContainer and Mount the DATA_VOLUME
-                    variantValidatorContainer.run("-v $DATA_VOLUME:/usr/local/share:rw -d --name variantvalidator --network $DOCKER_NETWORK")
-
-                    // Display a message indicating that VariantValidator is being built and run
-                    sh 'echo Building and running VariantValidator'
-                }
-            }
-        }
-        stage("Run Pytest and Codecov") {
-            steps {
-                script {
-                    sh 'docker ps' // List running Docker containers
-                    def connectionSuccessful = false
-
-                    for (int attempt = 1; attempt <= 5; attempt++) {
-                        echo "Attempt $attempt to connect to the database..."
-                        def exitCode = sh(script: '''
-                            docker exec -e PGPASSWORD=uta_admin variantvalidator psql -U uta_admin -d vvta -h vv-vvta -p 5432
-                        ''', returnStatus: true)
-
-                        if (exitCode == 0) {
-                            connectionSuccessful = true
-                            echo "Connected successfully! Running pytest..."
-
-                            // Run pytest && Run Codecov with the provided token and branch name
-                            sh 'docker exec variantvalidator pytest -n 3 --cov=VariantValidator --cov=VariantFormatter --cov-report=term tests/'
-
-                            // Send coverage report to Codecov
-                            sh 'docker exec variantvalidator codecov -t $CODECOV_TOKEN -b ${BRANCH_NAME}'
-
-                            // Check for test failures in the captured output
-                            if (currentBuild.rawBuild.getLog(2000).join('\n').contains("test summary info") && currentBuild.rawBuild.getLog(2000).join('\n').contains("FAILED")) {
-                                failure(message:"Pytest completed with test failures")
-                            }
-
-                            // Check the Jenkins console log for pytest exit code
-                            def pytestExitCode = currentBuild.rawBuild.getLog(2000).find { line -> line =~ /.*Pytest exit code: (\d+).*/ }
-                            if (pytestExitCode) {
-                                pytestExitCode = Integer.parseInt(pytestExitCode.replaceAll(/.*Pytest exit code: (\d+).*/, '$1'))
-                                if (pytestExitCode != 0) {
-                                    failure(message:"Pytest failed with exit code $pytestExitCode")
-                                }
-                            }
-                            break
-                        }
-
-                        echo "Connection failed. Waiting for 60 seconds before the next attempt..."
-                        sleep 60
-                    }
-
-                    if (!connectionSuccessful) {
-                        failure(message:"All connection attempts failed. Exiting...")
-                    }
-                }
-            }
-        }
-    }
-    post {
-        always { // This ensures cleanup is executed regardless of build outcome
-            script {
-                // Cleanup Docker
-                sh 'docker stop vv-vvta'
-                sh 'docker rm vv-vvta'
-                sh 'docker stop vv-vdb'
-                sh 'docker rm vv-vdb'
-                sh 'docker stop vv-seqrepo'
-                sh 'docker rm vv-seqrepo'
-                sh 'docker stop variantvalidator'
-                sh 'docker rm variantvalidator'
-                sh 'docker network rm $DOCKER_NETWORK'
-                sh 'docker system prune --all --volumes --force'
-            }
-        }
-    }
-}
diff --git a/README.md b/README.md
index 8137a1a6..8085a0ed 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
 # VariantValidator
-[![codecov](https://codecov.io/github/openvar/variantValidator/graph/badge.svg?token=QWTxw5kiY4)](https://codecov.io/github/openvar/variantValidator)
+[![VariantValidator CI](https://github.com/openvar/variantValidator/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/openvar/variantValidator/actions/workflows/ci.yml) [![codecov](https://codecov.io/github/openvar/variantValidator/graph/badge.svg?token=QWTxw5kiY4)](https://codecov.io/github/openvar/variantValidator)
 
 ## About
 
diff --git a/db_dockerfiles/vdb/Dockerfile b/db_dockerfiles/vdb/Dockerfile
index e6129686..fd89e15b 100644
--- a/db_dockerfiles/vdb/Dockerfile
+++ b/db_dockerfiles/vdb/Dockerfile
@@ -1,10 +1,10 @@
 # Should run on all processors
 FROM ubuntu/mysql:8.0-22.04_beta
 
-ENV MYSQL_RANDOM_ROOT_PASSWORD yes
-ENV MYSQL_DATABASE validator
-ENV MYSQL_USER vvadmin
-ENV MYSQL_PASSWORD var1ant
+ENV MYSQL_RANDOM_ROOT_PASSWORD=yes
+ENV MYSQL_DATABASE=validator
+ENV MYSQL_USER=vvadmin
+ENV MYSQL_PASSWORD=var1ant
 
 RUN apt-get update && apt-get install -y \
     wget
diff --git a/db_dockerfiles/vvsr/Dockerfile b/db_dockerfiles/vvsr/Dockerfile
index 1744ff86..9458b089 100644
--- a/db_dockerfiles/vvsr/Dockerfile
+++ b/db_dockerfiles/vvsr/Dockerfile
@@ -1,17 +1,21 @@
 FROM ubuntu:22.04
 
-RUN  apt-get update
+RUN apt-get update
 
 RUN apt-get install -y wget
 
 RUN mkdir -p /usr/local/share/seqdata
 
-RUN wget --output-document=/usr/local/share/seqdata/VV_SR_2025_02.tar https://data.variantvalidator.org/vvdata/vv_seqrepo/VV_SR_2025_02.tar
+RUN wget --output-document=/usr/local/share/seqdata/VV_SR_2025_02.tar \
+    https://data.variantvalidator.org/vvdata/vv_seqrepo/VV_SR_2025_02.tar
 
-RUN tar -xvf /usr/local/share/seqdata/VV_SR_2025_02.tar --directory /usr/local/share/seqdata
+RUN tar -xvf /usr/local/share/seqdata/VV_SR_2025_02.tar \
+    --directory /usr/local/share/seqdata
 
 RUN rm /usr/local/share/seqdata/VV_SR_2025_02.tar
 
+VOLUME /usr/local/share/seqdata
+
 ENTRYPOINT []
 
-CMD ["tail", "-f", "/dev/null"]
\ No newline at end of file
+CMD ["tail", "-f", "/dev/null"]

From e2e1e81c5c31d6806090801b4c0ff02fccf23542 Mon Sep 17 00:00:00 2001
From: Peter-J-Freeman <peter.j.freeman@manchester.ac.uk>
Date: Tue, 23 Jun 2026 16:50:44 +0100
Subject: [PATCH 2/4] Update Dockerfile for SeqRepo to create internal Volume

Creating the internal volume cuts down substantial disc space usage, and
we can instead mount the VV container to the SR container volume
---
 .github/workflows/ci.yml       | 18 ++++++++++++------
 db_dockerfiles/vvsr/Dockerfile |  2 --
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index befdbb18..5119a140 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -160,9 +160,10 @@ jobs:
       - name: Run VariantValidator
         run: |
           docker run -d \
-            --name variantvalidator \
-            --network ${DOCKER_NETWORK} \
-            variantvalidator
+              --name variantvalidator \
+              --network ${DOCKER_NETWORK} \
+              --volumes-from vv-seqrepo:ro \
+              variantvalidator
 
       ##################################################################
       # Run tests
@@ -175,7 +176,6 @@ jobs:
             -n 4 \
             --cov=VariantValidator \
             --cov=VariantFormatter \
-            --cov-report=term-missing \
             --cov-report=xml \
             tests
 
@@ -183,12 +183,18 @@ jobs:
       # Copy coverage
       ##################################################################
 
+      - name: Locate coverage.xml
+        if: always()
+        run: |
+          docker exec variantvalidator \
+            find / -name coverage.xml 2>/dev/null || true
+
       - name: Copy coverage.xml
         if: always()
         run: |
           docker cp \
-            variantvalidator:/VariantValidator/coverage.xml \
-            coverage.xml
+            variantvalidator:/app/coverage.xml \
+            coverage.xml || true
 
       ##################################################################
       # Upload coverage
diff --git a/db_dockerfiles/vvsr/Dockerfile b/db_dockerfiles/vvsr/Dockerfile
index 9458b089..423a1e6e 100644
--- a/db_dockerfiles/vvsr/Dockerfile
+++ b/db_dockerfiles/vvsr/Dockerfile
@@ -4,8 +4,6 @@ RUN apt-get update
 
 RUN apt-get install -y wget
 
-RUN mkdir -p /usr/local/share/seqdata
-
 RUN wget --output-document=/usr/local/share/seqdata/VV_SR_2025_02.tar \
     https://data.variantvalidator.org/vvdata/vv_seqrepo/VV_SR_2025_02.tar
 

From 1300f53d031ea0415fec2f106c2f24d1097874da Mon Sep 17 00:00:00 2001
From: Peter-J-Freeman <peter.j.freeman@manchester.ac.uk>
Date: Tue, 23 Jun 2026 17:56:05 +0100
Subject: [PATCH 3/4] Update dockerfiles, ci workflow and container naming

Fixes a bug where the CI workflow is not creating the correct container
names and is not building SeqRepo
---
 .github/workflows/ci.yml       | 4 ++--
 db_dockerfiles/vvsr/Dockerfile | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5119a140..c539508e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -101,7 +101,7 @@ jobs:
           docker build \
             --no-cache \
             -f db_dockerfiles/vvsr/Dockerfile \
-            -t sqlite-seqrepo \
+            -t seqrepo-validator \
             db_dockerfiles/vvsr
 
       - name: Run SeqRepo
@@ -109,7 +109,7 @@ jobs:
           docker run -d \
             --name vv-seqrepo \
             --network ${DOCKER_NETWORK} \
-            sqlite-seqrepo
+            seqrepo-validator
 
       ##################################################################
       # Wait for MySQL
diff --git a/db_dockerfiles/vvsr/Dockerfile b/db_dockerfiles/vvsr/Dockerfile
index 423a1e6e..9458b089 100644
--- a/db_dockerfiles/vvsr/Dockerfile
+++ b/db_dockerfiles/vvsr/Dockerfile
@@ -4,6 +4,8 @@ RUN apt-get update
 
 RUN apt-get install -y wget
 
+RUN mkdir -p /usr/local/share/seqdata
+
 RUN wget --output-document=/usr/local/share/seqdata/VV_SR_2025_02.tar \
     https://data.variantvalidator.org/vvdata/vv_seqrepo/VV_SR_2025_02.tar
 

From 2d2405274cceac44fd8c60982b794cb23d3b7b3d Mon Sep 17 00:00:00 2001
From: Peter-J-Freeman <peter.j.freeman@manchester.ac.uk>
Date: Tue, 23 Jun 2026 19:59:14 +0100
Subject: [PATCH 4/4] Raise max MySQL connections in Docker

---
 db_dockerfiles/vdb/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_dockerfiles/vdb/Dockerfile b/db_dockerfiles/vdb/Dockerfile
index fd89e15b..06e70d48 100644
--- a/db_dockerfiles/vdb/Dockerfile
+++ b/db_dockerfiles/vdb/Dockerfile
@@ -13,7 +13,7 @@ RUN rm -rf /var/lib/apt/lists/*
 
 # Set the max_connections directly in the my.cnf
 RUN echo '[mysqld]' >> /etc/mysql/my.cnf && \
-    echo 'max_connections=250' >> /etc/mysql/my.cnf
+    echo 'max_connections=1000' >> /etc/mysql/my.cnf
 
 RUN wget https://data.variantvalidator.org/vvdata/validator/validator_2025_03.sql.gz -O /docker-entrypoint-initdb.d/validator_2025_03.sql.gz