Gh v2 580b czarte and Dan

.githooks/pre-commit

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# File to check (relative to repo root)
+FILENAME="ip_country/src/dbip_country.rs"
+
+if ! git diff --cached --name-only | grep -q "^$FILENAME$"; then
+ exit 0
+fi
+
+# find 'Nonexistents' country in dbip_country.rs
+@cat "$FILENAME" | grep '("NOEX", "Nonexistent")'
+NONEXISTENT=$?
+
+# Get local file size (staged version, not working copy)
+LOCAL_SIZE=$(git ls-files --stage | grep "$FILENAME" | awk '{print $2}' | xargs git cat-file -s)
+
+if [[ $LOCAL_SIZE -gt 1000000 || NONEXISTENT -gt 0 ]]; then
+ if [[ $LOCAL_SIZE -gt 1000000 ]]; then
+ echo "❌ Commit blocked: $FILENAME size it not dev version, we want to keep dev version under 1 MB."
+ echo " Local file size: $LOCAL_SIZE bytes"
+ fi
+ if [[ $NONEXISTENT -gt 0 ]]; then
+ echo "❌ Commit blocked: Dev file must contain '(\"NOEX\", \"Nonexistents\")' country"
+ fi
+ echo ""
+ echo " ⚡ Please sync the file with the remote before committing using:"
+ echo " git checkout HEAD -- ip_country/src/dbip_country.rs"
+ exit 1
+fi
+
+# All good
+exit 0

.github/workflows/ci-matrix.yml

@@ -20,7 +20,7 @@ jobs:
  name: Build node on ${{ matrix.target.os }}
  runs-on: ${{ matrix.target.os }}
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  with:
  ref: ${{ github.event.pull_request.head.sha }}
 
@@ -36,7 +36,8 @@ jobs:
  toolchain: 1.63.0
  components: rustfmt, clippy
  override: true
- - uses: actions/cache@v3
+
+ - uses: actions/cache@v4
  with:
  path: |
  ~/.cargo/bin/
@@ -45,10 +46,13 @@ jobs:
  ~/.cargo/registry/cache/
  ~/.cargo/git/db/
  target/
- key: ${{ runner.os }}-cargo
+ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-cargo-
 
  - name: Build ${{ matrix.target.os }}
  run: |
+ git fetch
  ./ci/all.sh
  ./ci/multinode_integration_test.sh
  ./ci/collect_results.sh
@@ -66,46 +70,52 @@ jobs:
  df -h /Users/runner/work/Node/Node/node/target/release
  deploy_to_s3:
  needs: build
+ if: success() && (startsWith(github.head_ref, 'GH') || startsWith(github.head_ref, 'v'))
  strategy:
  matrix:
-os: [linux, macos, windows]
- runs-on: ubuntu-22.04
+ os: [linux, macos, windows]
+ runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  with:
  ref: ${{ github.event.pull_request.head.sha }}
+ fetch-depth: 1
 
  - name: Download artifacts
  uses: actions/download-artifact@v4
 
  - name: Display structure of downloaded files
  run: ls -R
 
+ - name: Check artifacts exist
+ run: |
+ if [ ! -d "Node-${{ matrix.os }}/generated/bin/" ]; then
+ echo "Error: Build artifacts not found"
+ exit 1
+ fi
+
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ aws-region: us-west-2
+
  - if: startsWith(github.head_ref, 'GH')
  name: Versioned S3 Sync
- uses: jakejarvis/s3-sync-action@v0.5.1
- with:
- args: --acl private --follow-symlinks --delete
- env:
- AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- AWS_REGION: 'us-west-2'
- DEST_DIR: 'Node/${{ github.head_ref }}/Node-${{ matrix.os }}'
- SOURCE_DIR: 'Node-${{ matrix.os }}/generated/bin/'
+ run: |
+ aws s3 sync "Node-${{ matrix.os }}/generated/bin/" "s3://${{ secrets.AWS_S3_BUCKET }}/Node/${{ github.head_ref }}/Node-${{ matrix.os }}" \
+ --delete \
+ --no-progress \
+ --acl private
 
  - if: startsWith(github.head_ref, 'v')
  name: Latest S3 Sync
- uses: jakejarvis/s3-sync-action@v0.5.1
- with:
- args: --acl private --follow-symlinks --delete
- env:
- AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- AWS_REGION: 'us-west-2'
- DEST_DIR: 'Node/latest/Node-${{ matrix.os }}'
- SOURCE_DIR: 'Node-${{ matrix.os }}/generated/bin/'
+ run: |
+ aws s3 sync "Node-${{ matrix.os }}/generated/bin/" "s3://${{ secrets.AWS_S3_BUCKET }}/Node/latest/Node-${{ matrix.os }}" \
+ --delete \
+ --no-progress \
+ --acl private
 
  - name: Invalidate Binaries CloudFront
  uses: chetan/invalidate-cloudfront-action@v2.4

.github/workflows/dbip_download.yml

@@ -0,0 +1,63 @@
+name: Download DBIP data and generate dbip_country.rs
+
+on:
+ schedule:
+ - cron: "0 0 3 * *" # Runs at midnight on the 3rd of every month
+
+env:
+ TEMP_DIR: '/tmp'
+ YEAR_MONTH: '01-9999'
+
+permissions:
+ contents: write
+
+jobs:
+ generate:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up Rust
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: stable
+
+ - name: Download DBIP data in MMDB format
+ run: |
+ export YEAR_MONTH=$(date +%Y-%m)
+ echo "TEMP_DIR=$(mktemp -d)" >> $GITHUB_ENV
+ echo "YEAR_MONTH=$YEAR_MONTH" >> $GITHUB_ENV
+ cd ip_country
+ mkdir -p dbip-data
+ curl -L -o dbip-data/dbip-country-lite.mmdb.gz "https://download.db-ip.com/free/dbip-country-lite-$YEAR_MONTH.mmdb.gz"
+ gunzip dbip-data/dbip-country-lite.mmdb.gz
+
+ - name: Generate Rust source file
+ run: |
+ cd ip_country
+ cargo run < "dbip-data/dbip-country-lite.mmdb" > "$TEMP_DIR"/dbip_country.rs
+ ls "$TEMP_DIR"
+
+ - name: Commit and push generated file
+ run: |
+ git config user.name "github-actions[bot]"
+ git config user.email "github-actions[bot]@users.noreply.github.com"
+ git fetch
+ BRANCH="$(git ls-remote --exit-code --heads origin generated-source 2>/dev/null || true)"
+ if [[ "$BRANCH" == "" ]]
+ then
+ git checkout -B generated-source
+ git rm -rf .
+ mkdir -p ip_country/src
+ else
+ git checkout generated-source
+ fi
+ DIFF="$(diff "${TEMP_DIR}"/dbip_country.rs ip_country/src/dbip_country.rs || true)"
+ if [[ "$DIFF" != "" || "$BRANCH" == "" ]]
+ then
+ mv "${TEMP_DIR}"/dbip_country.rs ip_country/src/dbip_country.rs
+ git add ip_country/src/dbip_country.rs
+ git commit -m "Update generated dbip_country ${YEAR_MONTH} Rust source file" || true
+ git push -u origin HEAD
+ fi

ci/all.sh

@@ -1,12 +1,19 @@
 #!/bin/bash -xev
 # Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved.
 CI_DIR="$( cd "$( dirname "$0" )" && pwd )"
-PARENT_DIR="$1"
+export ARGV_1="$1"
 
 ci/format.sh
 
 export RUST_BACKTRACE=1
-
+#if github checkout fails, you can download csv database from db-ip.com and convert it locally by cargo run ip_country
+if [[ $GITHUB_ACTIONS || "$ARGV_1" == "dbip" ]]; then
+ git checkout origin/generated-source -- ip_country/src/dbip_country.rs
+ ls -la ip_country/src/dbip_country.rs
+fi
+if [[ -z "${GITHUB_ACTIONS}" ]]; then
+ bash ./install-hooks.sh
+fi
 # Remove these two lines to slow down the build
 which sccache || cargo install --version 0.4.1 sccache || echo "Skipping sccache installation" # Should do significant work only once
 #export CARGO_TARGET_DIR="$CI_DIR/../cargo-cache"
@@ -18,19 +25,19 @@ export RUSTFLAGS="-D warnings -Anon-snake-case"
 echo "*********************************************************************************************************"
 echo "*** MASQ_LIB HEAD ***"
 cd "$CI_DIR/../masq_lib"
-ci/all.sh "$PARENT_DIR"
+ci/all.sh
 echo "*** MASQ_LIB TAIL ***"
 echo "*********************************************************************************************************"
 echo "*********************************************************************************************************"
 echo "*** NODE HEAD ***"
 cd "$CI_DIR/../node"
-ci/all.sh "$PARENT_DIR"
+ci/all.sh
 echo "*** NODE TAIL ***"
 echo "*********************************************************************************************************"
 echo "*********************************************************************************************************"
 echo "*** DNS UTILITY HEAD ***"
 cd "$CI_DIR/../dns_utility"
-ci/all.sh "$PARENT_DIR"
+ci/all.sh
 echo "*** DNS UTILITY TAIL ***"
 echo "*********************************************************************************************************"
 echo "*********************************************************************************************************"
@@ -42,13 +49,12 @@ echo "**************************************************************************
 echo "*********************************************************************************************************"
 echo "*** AUTOMAP HEAD ***"
 cd "$CI_DIR/../automap"
-ci/all.sh "$PARENT_DIR"
+ci/all.sh
 echo "*** AUTOMAP TAIL ***"
 echo "*********************************************************************************************************"
 echo "*********************************************************************************************************"
 echo "*** IP COUNTRY HEAD ***"
 cd "$CI_DIR/../ip_country"
-ci/all.sh "$PARENT_DIR"
+ci/all.sh
 echo "*** IP COUNTRY TAIL ***"
 echo "*********************************************************************************************************"
-