Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 36 additions & 9 deletions scripts/install-k8s.ps1
Original file line numberDiff line numberDiff line change
Expand Up@@ -563,7 +563,18 @@ echo "NCT installed successfully."

$scriptPath = [System.IO.Path]::Combine($env:TEMP, "install-nct-$(Get-Random -Maximum 999999).sh")
[System.IO.File]::WriteAllText($scriptPath, $nctScript.Replace("`r`n", "`n"))
$wslPath = "/mnt/" + ($scriptPath -replace '\\','/' -replace '^([A-Za-z]):/', { $_.Groups[1].Value.ToLower() + '/' })
# Build the WSL path WITHOUT a scriptblock -replace: scriptblock substitution in
# the -replace operator is PowerShell 6.1+, but the bootstrap (install.ps1) runs
# this via powershell.exe (Windows PowerShell 5.1, per #Requires -Version 5.1),
# where the scriptblock is coerced to its literal text and the drive letter is
# NOT lowercased -> a malformed $wslPath and a 180s NCT-install timeout. -match
# / $Matches is 5.1-safe.
$fwd = $scriptPath -replace '\\','/'
if ($fwd -match '^([A-Za-z]):/(.*)$') {
$wslPath = "/mnt/" + $Matches[1].ToLower() + '/' + $Matches[2]
} else {
$wslPath = "/mnt/" + $fwd
}

$nctInstallJob = Start-Job -ScriptBlock {
param($d, $p)
Expand DownExpand Up@@ -1176,8 +1187,16 @@ function Install-ClientHelm {
# fails OPEN and lets a re-install silently overwrite an existing client we
# simply couldn't identify. Record it and fail CLOSED below (#200 follow-up).
$unreadableNs = ""
# $listUnknown: `helm list` itself failed or returned non-JSON, so we couldn't
# even ENUMERATE releases. Same fail-open risk one level up from $unreadableNs —
# skipping the guard here would let a re-install overwrite a different client.
$listUnknown = $false
$listJson = (helm list -A -o json 2>$null) | Out-String
if ($LASTEXITCODE -eq 0 -and $listJson.Trim()) {
if ($LASTEXITCODE -ne 0) {
# helm list failed (wedged/unreachable API, kubeconfig glitch) -> unknown.
# (helm returns 0 with an empty `[]` when there are genuinely no releases.)
$listUnknown = $true
} elseif ($listJson.Trim()) {
try {
foreach ($rel in ($listJson | ConvertFrom-Json)) {
if ($rel.chart -and $rel.chart.StartsWith("client-")) {
Expand All@@ -1202,17 +1221,25 @@ function Install-ClientHelm {
if ($id) { $existingId = $id; $existingNs = $rel.namespace; break }
}
}
} catch { }
} catch {
# helm list returned non-JSON/garbage -> can't trust the enumeration.
$listUnknown = $true
}
}
# Fail closed: a client release exists here but we couldn't read its clientId,
# and no OTHER release gave us a definitive id. Refuse rather than overwrite an
# Fail closed when we couldn't identify a client we can see ($unreadableNs) OR
# couldn't enumerate at all ($listUnknown). Refuse rather than overwrite an
# unknown client -- the operator must resolve it explicitly.
if (-not $existingId -and $unreadableNs) {
if (-not $existingId -and ($unreadableNs -or $listUnknown)) {
Write-Host ""
Warn "A tracebloc client release is installed here (namespace '$unreadableNs') but its configuration could not be read."
if ($listUnknown) {
Warn "Couldn't determine which tracebloc client (if any) is already installed here -- helm could not enumerate releases."
} else {
Warn "A tracebloc client release is installed here (namespace '$unreadableNs') but its configuration could not be read."
}
Hint "tracebloc runs one client per machine, so the installer will not overwrite"
Hint "a client it cannot identify. Inspect or remove it, then re-run:"
Hint " helm get values -A (see what is installed)"
Hint "a client it cannot see (usually the cluster API is briefly unreachable). Check and re-run:"
Hint " kubectl cluster-info (is the API reachable?)"
Hint " helm get values -A (see what is installed)"
Hint " k3d cluster delete $CLUSTER_NAME (wipes this client + its local data)"
Write-Host ""
Err "Refusing to replace an unidentifiable existing client."
Expand Down
18 changes: 10 additions & 8 deletions scripts/install.sh
Original file line numberDiff line numberDiff line change
Expand Up@@ -277,8 +277,10 @@ download_with_retry() {
for attempt in 1 2 3; do
# --tlsv1.2 floor; honor any proxy / custom-CA env the corporate-proxy
# segment relies on (#172/#722) — curl picks up HTTPS_PROXY/NO_PROXY/
# CURL_CA_BUNDLE from the environment automatically.
if curl -fsSL --tlsv1.2 "$url" -o "$dest"; then return 0; fi
# CURL_CA_BUNDLE from the environment automatically. --connect-timeout/--max-time
# (added to every fetch in this file) turn a stalled endpoint into a retriable
# failure instead of hanging the "1. Downloading" phase forever.
if curl -fsSL --tlsv1.2 --connect-timeout 30 --max-time 300 "$url" -o "$dest"; then return 0; fi
if [[ $attempt -ge $max_attempts ]]; then
echo "[ERROR] Failed to download $url after $max_attempts attempts."
exit 1
Expand DownExpand Up@@ -372,11 +374,11 @@ download_manifest() {
# Authoritative source: the signed release asset. Fall back to the in-repo
# copy in the tag tree only under the unverified dev opt-in (a branch checkout
# has no release assets).
if curl -fsSL --tlsv1.2 "$REPO_REL/manifest.sha256" -o "$dest" 2>/dev/null; then
if curl -fsSL --tlsv1.2 --connect-timeout 30 --max-time 300 "$REPO_REL/manifest.sha256" -o "$dest" 2>/dev/null; then
return 0
fi
if [[ "$ALLOW_UNVERIFIED" == "1" ]]; then
curl -fsSL --tlsv1.2 "$REPO_RAW/scripts/manifest.sha256" -o "$dest" 2>/dev/null
curl -fsSL --tlsv1.2 --connect-timeout 30 --max-time 300 "$REPO_RAW/scripts/manifest.sha256" -o "$dest" 2>/dev/null
return $?
fi
return 1
Expand DownExpand Up@@ -406,8 +408,8 @@ verify_manifest_signature() {
exit 1
fi

if ! curl -fsSL --tlsv1.2 "$REPO_REL/manifest.sha256.sig" -o "$sig" 2>/dev/null \
|| ! curl -fsSL --tlsv1.2 "$REPO_REL/manifest.sha256.cert" -o "$cert" 2>/dev/null; then
if ! curl -fsSL --tlsv1.2 --connect-timeout 30 --max-time 300 "$REPO_REL/manifest.sha256.sig" -o "$sig" 2>/dev/null \
|| ! curl -fsSL --tlsv1.2 --connect-timeout 30 --max-time 300 "$REPO_REL/manifest.sha256.cert" -o "$cert" 2>/dev/null; then
if [[ "$ALLOW_UNVERIFIED" == "1" ]]; then
echo "[WARN] manifest signature/cert not published for ref '$REF' — not verified (TRACEBLOC_ALLOW_UNVERIFIED=1)." >&2
return 0
Expand DownExpand Up@@ -462,8 +464,8 @@ ensure_cosign() {
local sums="$TMPDIR/cosign_checksums.txt"

echo " · Fetching the signature-verification tool (cosign)…"
curl -fsSL --tlsv1.2 "$base/$asset" -o "$bin" 2>/dev/null || return 1
curl -fsSL --tlsv1.2 "$base/cosign_checksums.txt" -o "$sums" 2>/dev/null || return 1
curl -fsSL --tlsv1.2 --connect-timeout 30 --max-time 300 "$base/$asset" -o "$bin" 2>/dev/null || return 1
curl -fsSL --tlsv1.2 --connect-timeout 30 --max-time 300 "$base/cosign_checksums.txt" -o "$sums" 2>/dev/null || return 1

local want got
want="$(grep " ${asset}\$" "$sums" | awk '{print $1}' | head -1)"
Expand Down
6 changes: 5 additions & 1 deletion scripts/lib/cluster.sh
Original file line numberDiff line numberDiff line change
Expand Up@@ -408,7 +408,11 @@ _wait_for_api() {
local frames=('⠋' '⠙' '⠹' '⠸' '⠼' '⠴' '⠦' '⠧' '⠇' '⠏')
local f=0
for attempt in $(seq 1 $max); do
if kubectl cluster-info &>/dev/null 2>&1; then
# --request-timeout bounds the call itself: the 60s cap here is only re-checked
# BETWEEN iterations, so an unbounded cluster-info against an API that accepts
# the TCP connection but never responds (corporate-proxy intercept of
# localhost, half-booted apiserver) would hang this gate forever.
if kubectl cluster-info --request-timeout=5s &>/dev/null 2>&1; then
printf "\r\033[K"
tput cnorm 2>/dev/null || true
success "Secure environment ready"
Expand Down
18 changes: 15 additions & 3 deletions scripts/lib/common.sh
Original file line numberDiff line numberDiff line change
Expand Up@@ -48,7 +48,11 @@ has() { command -v "$1" &>/dev/null; }
_chart_version() {
local ns="${1:-${TB_NAMESPACE:-tracebloc}}"
has helm || return 0
helm list -n "$ns" 2>/dev/null | grep -oE 'client-[0-9][^[:space:]]*' | head -1 | sed 's/^client-//'
# Trailing `|| true`: when no client-* release exists, `grep` exits 1 and, under
# `set -o pipefail`, the pipeline (this function's last command) returns 1 —
# which would abort callers that assign it under `set -e` (e.g. diagnose.sh).
# The version (or empty) has already been emitted to stdout regardless.
helm list -n "$ns" 2>/dev/null | grep -oE 'client-[0-9][^[:space:]]*' | head -1 | sed 's/^client-//' || true
}

# The client's core workload Deployments in namespace $1 — the set whose
Expand DownExpand Up@@ -176,7 +180,9 @@ download_with_progress() {
local url="$1" dest="$2" label="$3"

local total_bytes
total_bytes=$(curl -fsSLI "$url" 2>/dev/null \
# -m bounds the HEAD probe so a stalled server can't hang it (it's not
# retry-wrapped and its failure just means "no total" -> indeterminate bar).
total_bytes=$(curl -fsSLI -m 15 "$url" 2>/dev/null \
| awk 'tolower($0) ~ /content-length/ {gsub(/[^0-9]/,"",$2); print $2}' \
| tail -1)

Expand All@@ -192,7 +198,13 @@ download_with_progress() {
local logfile="${LOG_FILE:-/tmp/tracebloc-spin.log}"
rm -f "$dest"

curl -fSL -o "$dest" "$url" >> "$logfile" 2>&1 &
# --connect-timeout bounds the dial; --speed-limit/--speed-time abort a STALLED
# transfer (<1 KB/s for 60s) without capping a legitimately slow-but-progressing
# large download. Without these the backgrounded curl is monitored only by
# `kill -0` (no deadline, no kill), so a slow-loris / mid-stream stall would
# hang the progress loop forever.
curl -fSL --connect-timeout 30 --speed-limit 1024 --speed-time 60 \
-o "$dest" "$url" >> "$logfile" 2>&1 &
local curl_pid=$!

local bar_width=30
Expand Down
32 changes: 21 additions & 11 deletions scripts/lib/diagnose.sh
Original file line numberDiff line numberDiff line change
Expand Up@@ -51,9 +51,14 @@ run_diagnose() {

# Namespace discovery — TB_NAMESPACE isn't set on a standalone diagnose run,
# so find the namespace of the jobs-manager pod (falls back to "default").
# Every kubectl call below carries --request-timeout: run_diagnose runs `set +e`
# so a non-zero exit is harmless, but that does NOT bound an indefinite BLOCK —
# and --diagnose is exactly the "API may be wedged" path, where an unbounded
# call would freeze the bundle this function exists to produce.
local kt="--request-timeout=5s"
ns="${TB_NAMESPACE:-}"
if [[ -z "$ns" ]] && has kubectl; then
ns="$(kubectl get pods -A 2>/dev/null | awk '/-jobs-manager/{print $1; exit}')"
ns="$(kubectl get pods -A $kt 2>/dev/null | awk '/-jobs-manager/{print $1; exit}')"
fi
[[ -z "$ns" ]] && ns="default"

Expand DownExpand Up@@ -106,28 +111,31 @@ run_diagnose() {
# ── kubectl overview + per-pod detail ──
if has kubectl; then
{
echo "## nodes"; kubectl get nodes -o wide 2>&1
echo; echo "## pods (all namespaces)"; kubectl get pods -A -o wide 2>&1
echo; echo "## workloads"; kubectl get deploy,ds,sts -A 2>&1
echo; echo "## recent events"; kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -120
echo "## nodes"; kubectl get nodes -o wide $kt 2>&1
echo; echo "## pods (all namespaces)"; kubectl get pods -A -o wide $kt 2>&1
echo; echo "## workloads"; kubectl get deploy,ds,sts -A $kt 2>&1
echo; echo "## recent events"; kubectl get events -A --sort-by=.lastTimestamp $kt 2>&1 | tail -120
} > "$d/02-kubectl.txt" 2>&1
{
echo "## describe of non-Running pods in namespace '$ns'"
for p in $(kubectl get pods -n "$ns" --no-headers 2>/dev/null | awk '$3!="Running" && $3!="Completed"{print $1}'); do
echo; echo "### $p"; kubectl describe pod -n "$ns" "$p" 2>&1
for p in $(kubectl get pods -n "$ns" --no-headers $kt 2>/dev/null | awk '$3!="Running" && $3!="Completed"{print $1}'); do
echo; echo "### $p"; kubectl describe pod -n "$ns" "$p" $kt 2>&1
done
} > "$d/03-describe.txt" 2>&1
# workload logs (current + previous)
local w
for w in mysql-client "${ns}-jobs-manager" "${ns}-requests-proxy"; do
kubectl logs -n "$ns" "deploy/$w" --all-containers --tail=500 > "$d/logs/${w}.log" 2>&1
kubectl logs -n "$ns" "deploy/$w" --all-containers --previous --tail=500 > "$d/logs/${w}.previous.log" 2>&1
kubectl logs -n "$ns" "deploy/$w" --all-containers --tail=500 $kt > "$d/logs/${w}.log" 2>&1
kubectl logs -n "$ns" "deploy/$w" --all-containers --previous --tail=500 $kt > "$d/logs/${w}.previous.log" 2>&1
done
kubectl logs -n "$ns" "daemonset/tracebloc-resource-monitor" --tail=300 > "$d/logs/resource-monitor.log" 2>&1
kubectl logs -n "$ns" "daemonset/tracebloc-resource-monitor" --tail=300 $kt > "$d/logs/resource-monitor.log" 2>&1
fi

# ── helm (redacted afterwards) ──
if has helm; then
# helm has no --request-timeout; it talks to the same API. Only run it when a
# BOUNDED probe confirms the API is reachable, so a wedged API can't hang the
# bundle here (the kubectl output above already captured the degraded state).
if has helm && { ! has kubectl || kubectl cluster-info --request-timeout=5s >/dev/null 2>&1; }; then
# NOTE: deliberately NOT collecting `helm get manifest` — it renders the
# Secret objects with base64-encoded credentials (CLIENT_PASSWORD,
# .dockerconfigjson), which the text redaction can't see. `helm get values`
Expand All@@ -136,6 +144,8 @@ run_diagnose() {
echo "## helm list -A"; helm list -A 2>&1
echo; echo "## helm get values $ns"; helm get values "$ns" -n "$ns" 2>&1
} > "$d/04-helm.txt" 2>&1
elif has helm; then
echo "## helm skipped — API unreachable (bounded cluster-info probe failed)" > "$d/04-helm.txt" 2>&1
fi

# ── install artifacts (copied, redacted afterwards) ──
Expand Down
8 changes: 7 additions & 1 deletion scripts/lib/gpu-nvidia.sh
Original file line numberDiff line numberDiff line change
Expand Up@@ -54,7 +54,13 @@ install_nvidia_drivers() {
log "TRACEBLOC_SKIP_REBOOT_PROMPT set — skipping reboot prompt."
exit 2
fi
read -r -p " Reboot now? [y/N]: " _choice
# Read the terminal directly: the main install path is `curl … | bash`, where
# this shell's stdin is the (EOF) install pipe — a bare `read` there returns
# non-zero and, with `set -e` active in this code path, would ABORT the whole
# installer right after a successful driver install. No tty (unattended) => treat
# as "no reboot" (same as TRACEBLOC_SKIP_REBOOT_PROMPT).
local _choice=""
if [[ -r /dev/tty ]]; then read -r -p " Reboot now? [y/N]: " _choice </dev/tty || _choice=""; fi
[[ "$_choice" =~ ^[Yy]$ ]] && sudo reboot
warn "Skipping reboot. GPU may not be available until you restart."
}
Expand Down
4 changes: 3 additions & 1 deletion scripts/lib/gpu-plugins.sh
Original file line numberDiff line numberDiff line change
Expand Up@@ -67,7 +67,9 @@ verify_gpu() {
log "Verifying GPU on node..."

for i in {1..18}; do
RAW=$(kubectl get nodes -o json 2>/dev/null \
# --request-timeout bounds the call: the 18×5s cap is only re-checked between
# iterations, so an unbounded get-nodes against a wedged API would hang here.
RAW=$(kubectl get nodes -o json --request-timeout=5s 2>/dev/null \
| grep -o '"[^"]*gpu[^"]*"\s*:\s*"[^"]*"' \
| sed 's/"//g; s/\s*:\s*/=/g' | head -5 \
2>/dev/null || echo "")
Expand Down
5 changes: 4 additions & 1 deletion scripts/lib/install-cli.sh
Original file line numberDiff line numberDiff line change
Expand Up@@ -163,7 +163,10 @@ install_tracebloc_cli() {

# 1) Download the released installer. A failure here is a download problem,
# distinct from an install problem below.
if ! curl -fsSL "$CURL_SECURE" "$TRACEBLOC_CLI_INSTALL_URL" -o "$installer" 2>>"${LOG_FILE:-/dev/null}"; then
# --connect-timeout/--max-time so a stalled CDN turns into a clean "install later"
# failure below instead of hanging the CLI-install step (this call isn't retry-
# wrapped, and a hang is not a failure the graceful fallback would otherwise catch).
if ! curl -fsSL "$CURL_SECURE" --connect-timeout 30 --max-time 120 "$TRACEBLOC_CLI_INSTALL_URL" -o "$installer" 2>>"${LOG_FILE:-/dev/null}"; then
warn "Couldn't download the tracebloc CLI installer — your client is set up fine."
hint "Install it later: curl -fsSL ${TRACEBLOC_CLI_INSTALL_URL} | sh"
rm -f "$installer"
Expand Down
Loading
Loading