Add OSV-Scanner security gate + clear all CVEs (Python floor to 3.10)

.github/workflows/code-quality-checks.yml

@@ -34,7 +34,7 @@ jobs:
  labels: linux-ubuntu-latest
  strategy:
  matrix:
- python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+ python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
  dependency-version: ["default", "min"]
  exclude:
  - python-version: "3.12"
@@ -96,7 +96,7 @@ jobs:
  labels: linux-ubuntu-latest
  strategy:
  matrix:
- python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+ python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
  dependency-version: ["default", "min"]
  exclude:
  - python-version: "3.12"
@@ -246,7 +246,7 @@ jobs:
  labels: linux-ubuntu-latest
  strategy:
  matrix:
- python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+ python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
  steps:
  - name: Check out repository
  uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
@@ -273,7 +273,7 @@ jobs:
  labels: linux-ubuntu-latest
  strategy:
  matrix:
- python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+ python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
  steps:
  - name: Check out repository
  uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

.github/workflows/securityScan.yml

@@ -0,0 +1,295 @@
+name: Security Scan
+
+# Single workflow, single job. Triggered three ways with DIFFERENT
+# thresholds:
+#
+# - pull_request to main: fail the job on any unsuppressed
+# CVSS >= 7 finding (HIGH+), or any unscored finding. MEDIUM/LOW
+# findings show in the step summary but don't block merges. Not yet
+# required-to-merge in branch protection.
+#
+# - cron (weekly): report ALL findings regardless of severity, fail the
+# job on any finding, and upload the raw scan output as an artifact.
+# The intent is full situational awareness -- emerging MEDIUM risks
+# should be visible before they cross the PR gate. A separate
+# cross-repo action collates the uploaded artifacts across all driver
+# repos and sends a single digest; this job does NOT email directly.
+#
+# - workflow_dispatch: behaves like the cron run (full reporting).
+#
+# Scanner: OSV-Scanner v2.3.8 (purl-based via OSV.dev; federates GHSA,
+# NVD, PyPA, RustSec, Go vuln DB). Reads `poetry.lock` natively --
+# no separate SBOM tool needed.
+#
+# NOTE: this scans BOTH runtime and dev dependencies (OSV treats
+# everything in poetry.lock equally). If a finding is dev-only and
+# shouldn't block merges, suppress it via osv-scanner.toml with a
+# justification ("dev-only, not shipped in the wheel").
+#
+# Suppressions live in `osv-scanner.toml` as [[IgnoredVulns]] entries
+# (CVE-id global; OSV-Scanner v2.3.8 doesn't support per-package CVE
+# scoping). Each entry has a justification comment and an `ignoreUntil`
+# expiry so suppressions re-surface for re-review rather than lingering.
+
+on:
+ pull_request:
+ branches: [main]
+ schedule:
+ - cron: '0 0 * * 0' # Run every Sunday at midnight UTC
+ workflow_dispatch:
+
+permissions:
+ id-token: write
+ contents: read
+
+jobs:
+ security-scan:
+ name: Security Scan
+ runs-on:
+ group: databricks-protected-runner-group
+ labels: linux-ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+ # JFrog OIDC + pip: skipped on fork PRs (no OIDC token from
+ # GitHub's perspective). OSV-Scanner reads poetry.lock directly
+ # without needing to download wheels, so fork PRs still work; we
+ # keep setup-jfrog here only for parity with the other workflows
+ # in this repo. If you remove it later, also remove the
+ # `id-token: write` permission above.
+ - name: Setup JFrog
+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+ uses: ./.github/actions/setup-jfrog
+
+ - name: Install osv-scanner
+ run: |
+ set -euo pipefail
+ curl -fsSL -o /tmp/osv-scanner \
+ https://github.com/google/osv-scanner/releases/download/v2.3.8/osv-scanner_linux_amd64
+ chmod +x /tmp/osv-scanner
+ /tmp/osv-scanner --version
+
+ - name: Run OSV-Scanner
+ # osv-scanner's exit codes are meaningful and we must NOT blanket
+ # `|| true` them: exit 0 = no vulns, exit 1 = vulns found (expected
+ # -- our real gate is the CVSS>=7 filter below), any OTHER non-zero
+ # (126/127/128+, network error reaching OSV.dev, corrupt binary,
+ # partial DB load) = a scanner error that must fail the job CLOSED.
+ # A blanket `|| true` + zero-byte guard let an errored-but-non-empty
+ # output pass as "clean" (fail-open); capture and classify the code.
+ run: |
+ set -uo pipefail
+
+ if [ ! -f poetry.lock ]; then
+ echo "::error::poetry.lock not found at repo root."
+ exit 1
+ fi
+
+ scan_rc=0
+ /tmp/osv-scanner scan source \
+ --lockfile=poetry.lock \
+ --config=osv-scanner.toml \
+ --format=json \
+ --output-file=/tmp/osv-out.json \
+ || scan_rc=$?
+
+ # Tolerate only 0 (clean) and 1 (findings present). Anything else
+ # is a scanner failure -> fail closed rather than reporting zero.
+ if [ "$scan_rc" -ne 0 ] && [ "$scan_rc" -ne 1 ]; then
+ echo "::error::OSV-Scanner exited $scan_rc (scanner error, not a findings result). Failing closed."
+ exit 1
+ fi
+
+ if [ ! -s /tmp/osv-out.json ]; then
+ echo "::error::OSV-Scanner did not produce an output file."
+ exit 1
+ fi
+
+ # Validate the output is well-formed JSON with a results array
+ # before any downstream parsing. A truncated/partial write is
+ # non-empty (defeats the -s guard) but unparseable -- catch it
+ # here so it fails closed instead of parsing to zero findings.
+ if ! jq -e 'has("results") and (.results | type == "array")' /tmp/osv-out.json >/dev/null 2>&1; then
+ echo "::error::OSV-Scanner output is not valid JSON with a .results array (partial/corrupt scan). Failing closed."
+ exit 1
+ fi
+
+ # Parse OSV's JSON into job outputs. The terminal steps below
+ # (PR-fail and scheduled-fail) consume these outputs.
+ #
+ # Two thresholds: PR gating uses CVSS >= 7 (high_count) so we don't
+ # block merges on MEDIUM/LOW noise; the weekly reports everything
+ # (total_findings) so the team has full situational awareness of
+ # emerging risk before it crosses the gate.
+ - name: Collect findings
+ id: findings
+ run: |
+ set -uo pipefail
+
+ # All findings (sorted by severity desc).
+ #
+ # Severity resolution is defense-in-depth against fail-open:
+ # OSV's group-level `.max_severity` is EMPTY ("") for advisory
+ # groups that lack a CVSS vector (common for GHSA-only and MAL-*
+ # malware advisories). jq's `//` only coalesces null/false -- an
+ # empty string is truthy and would pass through, then
+ # `"" | tonumber? // 0` scores it 0, silently sailing a real HIGH
+ # past the CVSS>=7 gate. So we do NOT trust max_severity alone:
+ # 1. Try group `.max_severity` (numeric only; empty/"" -> null).
+ # 2. Fall back to the max CVSS score across the group's own
+ # vulnerabilities' `.severities[].score` (parsed from the
+ # CVSS vector's numeric base score when present).
+ # 3. If still unresolved, emit the sentinel "UNKNOWN" (a
+ # non-numeric string), never scored 0.
+ #
+ # BLOCKING RULE: unlike the Go driver (whose scoreless findings are
+ # mostly Go-stdlib advisories delivered via GOTOOLCHAIN and thus
+ # report-only), PyPA advisories reliably carry a CVSS score. A
+ # scoreless UNKNOWN here means a GHSA-only or MAL-* malware advisory
+ # on a package we depend on -- always BLOCKING (fail closed).
+ #
+ # NOTE ON jq NUMBER PARSING: use `try (x|tonumber) catch null`,
+ # NOT `x|tonumber?`. On a non-numeric string, `tonumber?` yields
+ # EMPTY (not null); inside an `... as $var` binding that makes the
+ # entire finding row vanish -- silently dropping a scoreless HIGH.
+ # try/catch normalizes non-numeric to null so the row survives and
+ # the fallback logic runs.
+ ALL_FINDINGS=$(jq -c '
+ def cvss_num($sev):
+ # OSV severity entries: {type:"CVSS_V3", score:"9.8"} (numeric)
+ # or a full vector string. Take numeric scores only; vectors
+ # without a bare numeric score contribute nothing (null).
+ ($sev // []) | map(try (.score | tonumber) catch null)
+ | map(select(. != null)) | (max // null);
+ [
+ .results[].packages[]? |
+ .package as $pkg |
+ (.vulnerabilities // []) as $vulns |
+ .groups[]? |
+ .ids as $gids |
+ # max CVSS across the vulnerabilities referenced by this group
+ ([ $vulns[] | select(.id as $id | ($gids | index($id)) != null) | cvss_num(.severities) ]
+ | map(select(. != null)) | (max // null)) as $vuln_score |
+ (try (.max_severity | tonumber) catch null) as $grp_score |
+ ($grp_score // $vuln_score) as $resolved |
+ {
+ pkg: ($pkg.name + "@" + $pkg.version),
+ ids: .ids,
+ severity: (if $resolved == null then "UNKNOWN" else ($resolved | tostring) end)
+ }
+ ] | sort_by(if (try (.severity | tonumber) catch null) == null then -1 else - (.severity | tonumber) end)
+ ' /tmp/osv-out.json)
+ TOTAL_FINDINGS=$(echo "$ALL_FINDINGS" | jq 'length')
+
+ # Scoreless (UNKNOWN) findings -- all blocking (see note above).
+ UNKNOWN_COUNT=$(echo "$ALL_FINDINGS" | jq '[.[] | select(.severity == "UNKNOWN")] | length')
+
+ # Blocking findings = CVSS >= 7 (any package) OR scoreless UNKNOWN.
+ HIGH_FINDINGS=$(echo "$ALL_FINDINGS" | jq -c '[.[] | select(((.severity | tonumber? // 0) >= 7) or (.severity == "UNKNOWN"))]')
+ HIGH_COUNT=$(echo "$HIGH_FINDINGS" | jq 'length')
+
+ # Guard against empty counts propagating to the numeric gates
+ # below. If any jq above failed, the var would be "" and
+ # `[ "$X" -gt 0 ]` errors / `'' != '0'` reads true. Default to 0
+ # and, since a failed parse should never be silently "clean",
+ # fail closed if the counts didn't resolve to integers.
+ TOTAL_FINDINGS=${TOTAL_FINDINGS:-}
+ HIGH_COUNT=${HIGH_COUNT:-}
+ UNKNOWN_COUNT=${UNKNOWN_COUNT:-}
+ if ! [[ "$TOTAL_FINDINGS" =~ ^[0-9]+$ ]] || ! [[ "$HIGH_COUNT" =~ ^[0-9]+$ ]]; then
+ echo "::error::Could not compute finding counts from OSV output (parse failure). Failing closed."
+ exit 1
+ fi
+
+ # Persist the full findings list to a file rather than a job
+ # output -- GitHub Actions outputs are size-capped at 1 MB and
+ # the formatted finding list can be larger than that.
+ echo "$ALL_FINDINGS" > /tmp/all-findings.json
+
+ echo "total_findings=$TOTAL_FINDINGS" >> "$GITHUB_OUTPUT"
+ echo "high_count=$HIGH_COUNT" >> "$GITHUB_OUTPUT"
+ echo "unknown_count=$UNKNOWN_COUNT" >> "$GITHUB_OUTPUT"
+
+ # Step summary so findings are visible in the GH Actions UI
+ # without downloading artifacts.
+ {
+ echo "## OSV-Scanner Findings"
+ echo ""
+ echo "- Total findings (any severity): \`$TOTAL_FINDINGS\`"
+ echo "- Blocking findings (CVSS >= 7, or unscored; PR-blocking): \`$HIGH_COUNT\`"
+ echo "- Unscored/UNKNOWN findings: \`$UNKNOWN_COUNT\` (all blocking)"
+ if [ "$TOTAL_FINDINGS" -gt 0 ]; then
+ echo ""
+ echo "All findings (sorted by severity desc):"
+ echo ""
+ echo "| Severity | Package | IDs |"
+ echo "|---|---|---|"
+ echo "$ALL_FINDINGS" | jq -r '.[] | "| \(.severity) | \(.pkg) | \(.ids | join(",")) |"'
+ fi
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ # Also dump the findings to the job log so they're visible in
+ # the default "Logs" view, not just the step summary panel.
+ echo "OSV: $TOTAL_FINDINGS total findings, $HIGH_COUNT blocking (CVSS>=7 or unscored)"
+ if [ "$TOTAL_FINDINGS" -gt 0 ]; then
+ echo ""
+ echo "All findings (sorted by severity desc):"
+ echo "$ALL_FINDINGS" | jq -r '.[] | " [\(.severity)] \(.pkg) \(.ids | join(", "))"'
+ fi
+
+ # --- Terminal: PR event ---
+ # Fail the job so the PR's check goes red. No email.
+ # PR gate is CVSS >= 7 (or unscored) only; MEDIUM/LOW findings show
+ # up in the step summary but don't block merges.
+ - name: Fail on findings (PR)
+ if: github.event_name == 'pull_request' && steps.findings.outputs.high_count != '0'
+ run: |
+ set -uo pipefail
+ # List the actual blocking findings inline so the author sees what
+ # needs fixing without clicking through to the step summary
+ # panel or downloading artifacts.
+ HIGH_FINDINGS=$(jq -c '[.[] | select(((.severity | tonumber? // 0) >= 7) or (.severity == "UNKNOWN"))]' /tmp/all-findings.json)
+
+ echo "::error::${{ steps.findings.outputs.high_count }} unsuppressed blocking finding(s) (CVSS>=7, or unscored) in this PR:"
+ echo ""
+ echo "$HIGH_FINDINGS" | jq -r '.[] | " [\(.severity)] \(.pkg) \(.ids | join(", "))"'
+ echo ""
+ echo "Fix by either:"
+ echo " 1. Bumping the affected dependency to a patched version, or"
+ echo " 2. Adding a documented [[IgnoredVulns]] entry to osv-scanner.toml"
+ echo " with a clear justification for why the CVE doesn't apply to our usage."
+ echo ""
+ echo "Full step summary: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+ exit 1
+ # --- Terminal: scheduled/manual event ---
+ # Weekly reports ALL findings (not just CVSS >= 7) so emerging risk is
+ # visible before it crosses the PR gate. PR-time is narrower to avoid
+ # blocking on MEDIUM/LOW noise; weekly is broader for situational
+ # awareness.
+ #
+ # Notification is intentionally NOT done here: a separate cross-repo
+ # action collates findings from all driver repos and sends a single
+ # digest. This job's job is to (a) fail so the scheduled run is red
+ # when anything is found, and (b) upload the raw osv-out.json artifact
+ # for the collator to consume.
+ - name: Fail on findings (scheduled/manual)
+ if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && steps.findings.outputs.total_findings != '0'
+ run: |
+ echo "::error::${{ steps.findings.outputs.total_findings }} OSV finding(s) on main (${{ steps.findings.outputs.high_count }} blocking at CVSS>=7 or unscored). See the security-scan-reports artifact."
+ exit 1
+
+ # Always upload the raw scan output so triagers -- and the planned
+ # cross-repo collation/notification action -- can pull findings
+ # without rerunning. This is the machine-readable source of truth now
+ # that per-repo email has been removed.
+ - name: Upload reports
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: security-scan-reports
+ path: |
+ /tmp/osv-out.json
+ /tmp/all-findings.json
+ if-no-files-found: ignore

.github/workflows/warm-deps-cache.yml

@@ -86,7 +86,7 @@ jobs:
  # Mirror code-quality-checks.yml exactly so every fork check has a
  # matching warmed entry. extras: "" is the base unit-test / lint / type
  # environment; pyarrow and kernel back the two extra unit-test tiers.
- python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+ python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
  dependency-version: ["default", "min"]
  extras: ["", "pyarrow", "kernel"]
  exclude:
@@ -98,10 +98,6 @@ jobs:
  dependency-version: "min"
  - python-version: "3.13"
  dependency-version: "min"
- # The kernel wheel is cp310-abi3 (Requires-Python >=3.10); the
- # [kernel] extra is a no-op on 3.9, so there's no kernel leg to warm.
- - python-version: "3.9"
- extras: "kernel"
 
  name: "Warm (py ${{ matrix.python-version }}, ${{ matrix.dependency-version }} deps, extras=${{ matrix.extras || 'base' }})"
 

osv-scanner.toml

@@ -0,0 +1,18 @@
+# OSV-Scanner suppressions for the databricks-sql-python security gate.
+#
+# Each entry suppresses a CVE that is a documented ecosystem false
+# positive against an artifact we ship. Every entry has a justification.
+#
+# Trade-off worth noting: [[IgnoredVulns]] entries are CVE-id global --
+# they ignore the CVE across all packages OSV reports it against, not
+# just the artifact we have in mind. The alternative
+# ([[PackageOverrides]] with `vulnerability.ignore = true`) is
+# per-package but blanket-ignores ALL vulnerabilities on that package,
+# which is much worse. OSV-Scanner v2.3.8 does NOT support an
+# intersection ("this CVE on this package only").
+#
+# See google.github.io/osv-scanner/configuration/ for the schema.
+#
+# This file starts empty -- populate iteratively as the first scan run
+# surfaces real false positives. Do not pre-populate with speculative
+# suppressions.