Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion scripts/install-k8s.ps1
Original file line numberDiff line numberDiff line change
Expand Up@@ -924,6 +924,34 @@ function Test-ToolsPresent {
return $true
}

# The CLI floor the installer actively repairs (client#707). 0.10.0 is the release
# where the CLI gained its own update nudge: at or above it a user can keep
# themselves current, below it NOTHING on the machine can tell them they are
# behind. A floor, not a "must be latest" — so this needs no network call and
# never needs raising.
$script:TB_CLI_MIN_VERSION = "0.10.0"

# Is the tracebloc CLI present AND new enough to maintain itself?
#
# The fast path must not shortcut past Install-TraceblocCli when it isn't, for
# TWO reasons on Windows:
# * Test-ToolsPresent covers docker/kubectl/k3d/helm — the CLI is not in it at
# all, so a stale CLI was invisible; and
# * `completed` is set purely from ClientState -eq "connected", which says
# nothing about the CLI, while Install-TraceblocCli is deliberately
# non-fatal. A machine whose CLI install FAILED was therefore marked complete
# and never retried — permanently CLI-less, not merely stale.
#
# Fails OPEN on an unreadable version: that is not evidence of staleness, and
# re-running the CLI install on every invocation would be worse than the problem.
function Test-TraceblocCliCurrent {
if (-not (Has "tracebloc")) { return $false }
$ver = ""
try { $ver = (& tracebloc version 2>$null | Select-Object -First 1) } catch { $ver = "" }
if ($ver -notmatch '(\d+(?:\.\d+)+)') { return $true }
try { return ([version]$Matches[1] -ge [version]$script:TB_CLI_MIN_VERSION) } catch { return $true }
}

# Pure TRI-STATE classifier from a FULL `k3d cluster list -o json` (no name filter)
# output. Distinguishes "confidently not ours" from "can't tell" so callers never
# conflate an indeterminate read with a definitive answer (#557 Bugbot 3728340365,
Expand DownExpand Up@@ -5963,7 +5991,7 @@ Find-Gpu
# verifies live health (not just the checkpoint), so a stopped cluster or a down
# client falls through to the repairing walk. Skipped on -Resume (a resume must
# finish the interrupted walk).
if ((-not $Resume) -and $script:InstallState.completed -and (Test-ToolsPresent) -and (Test-ClusterRunning) -and (Test-ClientHealthy)) {
if ((-not $Resume) -and $script:InstallState.completed -and (Test-ToolsPresent) -and (Test-TraceblocCliCurrent) -and (Test-ClusterRunning) -and (Test-ClientHealthy)) {
# GPU is "fully enabled" only when the node ACTUALLY advertises a GPU AND the live release
# requests one. If an NVIDIA GPU is present but EITHER is missing, do NOT shortcut -- the state is
# inconsistent in one of two ways (Bugbot), both of which a re-run should fix:
Expand Down
79 changes: 73 additions & 6 deletions scripts/lib/assess.sh
Original file line numberDiff line numberDiff line change
Expand Up@@ -131,13 +131,71 @@ _assess_runtime_down() {
grep -qiE 'cannot connect to the docker daemon|is the docker daemon running|docker daemon is not running|dial unix|the system cannot find the file specified|open //./pipe/docker_engine' <<<"$_out"
}

# _assess_cli_present — is the tracebloc CLI available? Counts a binary in
# ~/.local/bin (where the CLI installer drops it when /usr/local/bin isn't
# writable) even if THIS shell's PATH predates that dir — the same place
# provision.sh / install-cli.sh resolve it.
# _assess_cli_bin — the tracebloc CLI to interrogate, or non-zero when there is
# none. Counts a binary in ~/.local/bin (where the CLI installer drops it when
# /usr/local/bin isn't writable) even if THIS shell's PATH predates that dir —
# the same place provision.sh / install-cli.sh resolve it.
_assess_cli_bin() {
has tracebloc && { printf 'tracebloc'; return 0; }
if [[ -x "${HOME}/.local/bin/tracebloc" ]]; then
printf '%s' "${HOME}/.local/bin/tracebloc"
return 0
fi
return 1
}

# _assess_cli_present — is the tracebloc CLI available at all?
_assess_cli_present() {
has tracebloc && return 0
[[ -x "${HOME}/.local/bin/tracebloc" ]]
_assess_cli_bin >/dev/null
}

# _version_lt A B — true when dotted-numeric A < B. Pure and self-contained: no
# `sort -V` (BSD sort predates it, and this must behave the same on macOS) and no
# jq. Missing or non-numeric components read as 0, so "0.10" < "0.10.1" and a
# pre-release suffix ("0.10.0-rc.1") compares as its base version.
_version_lt() {
local a="${1%%-*}" b="${2%%-*}" i av bv
local -a _A _B
IFS=. read -r -a _A <<<"$a"
IFS=. read -r -a _B <<<"$b"
for ((i = 0; i < 3; i++)); do
av="${_A[i]:-0}"; bv="${_B[i]:-0}"
[[ "$av" =~ ^[0-9]+$ ]] || av=0
[[ "$bv" =~ ^[0-9]+$ ]] || bv=0
(( av < bv )) && return 0
(( av > bv )) && return 1
done
return 1
}

# _assess_cli_outdated — is the installed CLI below the floor the installer will
# actively repair? (client#707)
#
# The floor is 0.10.0 because that is the release where the CLI gained its own
# update nudge. At or above it a user can keep themselves current; below it
# NOTHING on the machine can tell them they are behind — the cluster auto-
# upgrades hourly around a host binary that never moves. A field machine sat on
# v0.5.1 for five weeks that way.
#
# It is a FLOOR, not a "must be latest": the installer repairs users up to the
# point where the CLI maintains itself, and then stops caring. That keeps this
# free (no network call on every run) and means the constant never needs raising.
: "${TB_CLI_MIN_VERSION:=0.10.0}"

_assess_cli_outdated() {
local bin ver
bin="$(_assess_cli_bin)" || return 1 # absent — cli-missing already covers it
# No pipe: `tracebloc version | head -1` would let head close the pipe first,
# and pipefail turns that into a 141 the caller never asked for (backend#1778).
ver="$("$bin" version 2>/dev/null || true)"
ver="${ver%%$'\n'*}" # first line
ver="${ver#* }"; ver="${ver%% *}" # "tracebloc 0.10.5 (darwin/arm64)" -> "0.10.5"
ver="${ver#v}"
# FAIL OPEN on anything unparseable. A version we cannot read is not evidence
# of staleness, and treating it as outdated would reinstall the CLI on every
# single run — worse than the staleness this exists to fix.
[[ "$ver" =~ ^[0-9]+(\.[0-9]+)*$ ]] || return 1
_version_lt "$ver" "$TB_CLI_MIN_VERSION"
}

# _assess_release_pending NS — true when a release in NS is wedged (pending-* or
Expand DownExpand Up@@ -238,6 +296,14 @@ _assess_classify() {
return 0
fi

# Present, but too old to keep itself current (client#707). Without this the
# healthy fast-path pins a user to whatever CLI they first installed, forever
# and silently — the ONE thing on the machine that no auto-upgrade reaches.
if _assess_cli_outdated; then
INSTALL_STATE="degraded"; INSTALL_STATE_REASON="cli-outdated"
return 0
fi

INSTALL_STATE="healthy"; INSTALL_STATE_REASON="ns:${ns}"
return 0
}
Expand DownExpand Up@@ -309,6 +375,7 @@ assess_existing_install() {
cluster-stopped) info "Your secure environment is stopped — starting it and finishing setup." ;;
workload-not-ready) info "Your secure environment is still starting up — finishing setup." ;;
cli-missing) info "The tracebloc CLI isn't installed yet — setting it up." ;;
cli-outdated) info "Your tracebloc CLI is out of date — updating it." ;;
pending-wedge) info "A previous update was interrupted — recovering it and finishing setup." ;;
*) info "Your secure environment is only partly set up — finishing setup." ;;
esac
Expand Down
4 changes: 2 additions & 2 deletions scripts/manifest.sha256
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,8 +11,8 @@ f33495d298a2461df5c56aae05412b7c7d75a9c50fcf68a25b2e8afe998d2232 scripts/lib/se
a96e42571fce6b1da5e6bcfe2fd7d5ecfd6d659207d23c7a2b46756548a3e269 scripts/lib/install-client-helm.sh
61c1c887d158af52d4da4734b3bfa83205b2600ae7a291bfb3074daf3d9ffb55 scripts/lib/install-cli.sh
b0bf0a4966461e4257b6765dc4f12877762eee1e72f60ed4b2e322cb6291315a scripts/lib/provision.sh
898ab612c8ec53db62d5da8eff58592d400e63112d071aadb8f4bc7f263ce687 scripts/lib/assess.sh
8bd0deb458e7649723b722d28022018eeeff318068fd7c166756cbdcc3d65806 scripts/lib/assess.sh
911fd0714b17357bb205fc8a8fa8e13eedc1a9632a2f63d4ead9f8d8c7ee546f scripts/lib/probe.sh
1d06cb2de1a97fa614b41ec12e347aacd57b8bfef780623620639bb451a2f9e3 scripts/lib/summary.sh
77e03332ebfab1ef759c6148a57afcf479c02c5dc6cc7b0e0e680f58e20cd364 scripts/lib/diagnose.sh
1796f7384cb369471d03a44a1b0d7b1d8dea909c4b396355bf06b38c7717c275 scripts/install-k8s.ps1
78424926a7623f98121ae238741a484100928d4620be3aab2e5e0bca87ffe537 scripts/install-k8s.ps1
99 changes: 99 additions & 0 deletions scripts/tests/assess.bats
Original file line numberDiff line numberDiff line change
Expand Up@@ -163,6 +163,105 @@ _depname() {
[ "$status" -eq 0 ] || return 1
}

# ── CLI version floor (client#707) ──────────────────────────────────────────
# The healthy fast-path used to pin a user to whatever CLI they first installed,
# forever: presence was checked, version never was. The cluster auto-upgrades
# hourly around a host binary nothing touches — a field machine sat on v0.5.1
# for five weeks that way.
@test "_version_lt: orders dotted versions, including the 2-vs-10 trap" {
_version_lt 0.5.1 0.10.0 || return 1 # the field case
_version_lt 0.9.9 0.10.0 || return 1 # 9 < 10, not a string compare
_version_lt 0.10.0 0.10.1 || return 1
_version_lt 0.10 0.10.1 || return 1 # missing component reads as 0
! _version_lt 0.10.0 0.10.0 || return 1 # equal is not less
! _version_lt 0.10.5 0.10.0 || return 1
! _version_lt 1.0.0 0.10.0 || return 1
_version_lt 0.10.0-rc.1 0.10.0 && return 1 # pre-release compares as its base
return 0
}

@test "_assess_cli_outdated: a CLI below the floor is outdated" {
has() { [ "$1" = tracebloc ]; }
tracebloc() { echo "tracebloc 0.5.1 (darwin/arm64)"; }
run _assess_cli_outdated
[ "$status" -eq 0 ] || return 1
}

@test "_assess_cli_outdated: a CLI at the floor is NOT outdated" {
has() { [ "$1" = tracebloc ]; }
tracebloc() { echo "tracebloc 0.10.0 (darwin/arm64)"; }
run _assess_cli_outdated
[ "$status" -ne 0 ] || return 1
}

@test "_assess_cli_outdated: a CLI above the floor is NOT outdated" {
has() { [ "$1" = tracebloc ]; }
tracebloc() { echo "tracebloc 0.10.6 (darwin/arm64)"; }
run _assess_cli_outdated
[ "$status" -ne 0 ] || return 1
}

# Fail OPEN: an unreadable version is not evidence of staleness, and treating it
# as outdated would reinstall the CLI on EVERY run.
@test "_assess_cli_outdated: unreadable version -> treated as current (no churn)" {
has() { [ "$1" = tracebloc ]; }
tracebloc() { echo "tracebloc (unknown build)"; }
run _assess_cli_outdated
[ "$status" -ne 0 ] || return 1
}

@test "_assess_cli_outdated: a CLI that errors on 'version' -> treated as current" {
has() { [ "$1" = tracebloc ]; }
tracebloc() { return 1; }
run _assess_cli_outdated
[ "$status" -ne 0 ] || return 1
}

@test "_assess_cli_outdated: no CLI at all -> not 'outdated' (cli-missing owns that)" {
has() { return 1; }
HOME="$BATS_TEST_TMPDIR/empty"; mkdir -p "$HOME"
run _assess_cli_outdated
[ "$status" -ne 0 ] || return 1
}

# The regression guard: mutation-real against the pre-fix classify, which went
# straight to healthy on any CLI that merely existed.
@test "_assess_classify: a stale CLI is degraded/cli-outdated, NEVER healthy" {
has() { return 0; }
_cluster_exists() { return 0; }
_assess_cluster_servers_running() { echo 1; }
detect_installed_client() { INSTALLED_CLIENT_ID=uuid; INSTALLED_CLIENT_NS=tracebloc; }
_assess_release_pending() { return 1; }
_assess_workload_ready() { return 0; }
_assess_cli_present() { return 0; }
_assess_cli_outdated() { return 0; }
_assess_classify
[ "$INSTALL_STATE" = degraded ] || return 1
[ "$INSTALL_STATE_REASON" = cli-outdated ] || return 1
}

@test "_assess_classify: a current CLI still reaches healthy (floor doesn't over-fire)" {
has() { return 0; }
_cluster_exists() { return 0; }
_assess_cluster_servers_running() { echo 1; }
detect_installed_client() { INSTALLED_CLIENT_ID=uuid; INSTALLED_CLIENT_NS=tracebloc; }
_assess_release_pending() { return 1; }
_assess_workload_ready() { return 0; }
_assess_cli_present() { return 0; }
_assess_cli_outdated() { return 1; }
_assess_classify
[ "$INSTALL_STATE" = healthy ] || return 1
}

@test "assess_existing_install: cli-outdated says so, continues, and does NOT hand off" {
_assess_classify() { INSTALL_STATE=degraded; INSTALL_STATE_REASON=cli-outdated; }
tracebloc() { echo "HOME_SCREEN"; }
run assess_existing_install
[ "$status" -eq 0 ] || return 1
assert_has "out of date" "$output"
refute_has "HOME_SCREEN" "$output"
}

@test "_assess_cli_present: absent everywhere -> not present (1)" {
has() { return 1; }
HOME="$BATS_TEST_TMPDIR/empty"; mkdir -p "$HOME"
Expand Down
61 changes: 59 additions & 2 deletions scripts/tests/install-k8s.Tests.ps1
Original file line numberDiff line numberDiff line change
Expand Up@@ -426,8 +426,12 @@ Describe "Resume-after-reboot wiring (#420 source guards)" {
$script:PSRC | Should -Not -Match "Set-StageComplete"
$script:PSRC | Should -Not -Match "function Add-CompletedStage"
}
It "gates the fast nothing-to-do path on tools + running cluster + HEALTHY client" {
$script:PSRC | Should -Match '\$script:InstallState\.completed -and \(Test-ToolsPresent\) -and \(Test-ClusterRunning\) -and \(Test-ClientHealthy\)'
It "gates the fast nothing-to-do path on tools + a CURRENT CLI + running cluster + HEALTHY client" {
# Test-TraceblocCliCurrent is load-bearing here (client#707): Test-ToolsPresent
# covers docker/kubectl/k3d/helm only, so without it the fast path shortcuts
# past Install-TraceblocCli and the CLI is never updated — nor even retried on
# a machine where its (non-fatal) install had failed.
$script:PSRC | Should -Match '\$script:InstallState\.completed -and \(Test-ToolsPresent\) -and \(Test-TraceblocCliCurrent\) -and \(Test-ClusterRunning\) -and \(Test-ClientHealthy\)'
$script:PSRC | Should -Match 'already installed and the client is healthy -- nothing to do'
}
It "names the ACTUAL state-file path in the force-reinstall hint (honours HOST_DATA_DIR)" {
Expand DownExpand Up@@ -941,6 +945,59 @@ Describe "Install-TraceblocCli" {
}
}

Describe "Test-TraceblocCliCurrent" {
# client#707. The fast path gates on Test-ToolsPresent, which covers
# docker/kubectl/k3d/helm and NOT the CLI — so a stale CLI was invisible and a
# machine whose CLI install failed (it is non-fatal) was marked completed and
# never retried. This predicate is what puts the CLI back in that gate, both
# for presence and for version.

It "a CLI below the floor is not current -> fast path must fall through" {
Mock Has { $true }
Mock tracebloc { "tracebloc 0.5.1 (windows/amd64)" } # the field version
Test-TraceblocCliCurrent | Should -BeFalse
}

It "a CLI at the floor is current" {
Mock Has { $true }
Mock tracebloc { "tracebloc 0.10.0 (windows/amd64)" }
Test-TraceblocCliCurrent | Should -BeTrue
}

It "a CLI above the floor is current" {
Mock Has { $true }
Mock tracebloc { "tracebloc 0.10.6 (windows/amd64)" }
Test-TraceblocCliCurrent | Should -BeTrue
}

# 9 vs 10: a string comparison would call 0.9.9 newer than 0.10.0.
It "orders 0.9.9 below 0.10.0 numerically, not lexically" {
Mock Has { $true }
Mock tracebloc { "tracebloc 0.9.9 (windows/amd64)" }
Test-TraceblocCliCurrent | Should -BeFalse
}

# The Windows-only hole: absent entirely, yet `completed` can still be true.
It "a MISSING CLI is not current -> no longer fast-paths past the install" {
Mock Has { $false }
Test-TraceblocCliCurrent | Should -BeFalse
}

# Fail OPEN — an unreadable version is not evidence of staleness, and churning
# a reinstall on every run would be worse than the staleness.
It "an unreadable version is treated as current (no reinstall churn)" {
Mock Has { $true }
Mock tracebloc { "tracebloc (unknown build)" }
Test-TraceblocCliCurrent | Should -BeTrue
}

It "a CLI that errors on 'version' is treated as current" {
Mock Has { $true }
Mock tracebloc { throw "boom" }
Test-TraceblocCliCurrent | Should -BeTrue
}
}

Describe "Test-TraceblocCli" {
# Post-install self-verification (#738). Proves the CLI is usable from a fresh
# terminal and prints a VERIFIED next command, or the Windows-correct fix if a
Expand Down
Loading