From e3422f573de2fa1732a6e9ba9290de10b5f944be Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 31 Aug 2026 07:36:25 +0200 Subject: [PATCH 1/3] y-k8s-ingress-hosts: escalate the file swap, not the cluster read The script ran the whole k8s-ingress-hosts binary under sudo, so every /etc/hosts update read the cluster as root: root ran gke-gcloud-auth-plugin, which ran gcloud, which left root-owned files in the invoking user's ~/.config/gcloud and ~/.kube. The binary has a -host-file flag, so the merge can happen on a copy as the invoking user and only the resulting file needs privilege. The new y-etc-hosts-write takes no arguments, refuses input that is empty or has no 127.0.0.1 entry, and swaps the file in with rename(2) so a reader never sees a half-written /etc/hosts. That also closes an escalation. The NOPASSWD rule granted $YBIN/y-k8s-ingress-hosts-v*-bin * and -host-file turns those wildcards into a write-any-file-as-root grant, e.g. -host-file /etc/sudoers.d/x -write. Also fixes a pre-existing set -e abort: grep -c prints 0 but exits 1 when no GatewayClass carries yolean.se/dns-hint-ip, which killed the script before it could fall back to the legacy annotation. No flag or exit code changes. Help migrated to the YHELP/help-subcommand convention. Verified against a stub binary: -check reports drift and exits 1, --ensure writes and is then idempotent, a no-flag run still previews entries, and Y_ETC_HOSTS makes the whole path testable without root. Co-Authored-By: Claude Opus 5 (1M context) --- bin/y-cluster-sudoers | 5 +- bin/y-etc-hosts-write | 68 ++++++++++++++++++ bin/y-k8s-ingress-hosts | 155 ++++++++++++++++++++++++---------------- 3 files changed, 167 insertions(+), 61 deletions(-) create mode 100755 bin/y-etc-hosts-write diff --git a/bin/y-cluster-sudoers b/bin/y-cluster-sudoers index 29df93de..68a1393e 100755 --- a/bin/y-cluster-sudoers +++ b/bin/y-cluster-sudoers @@ -50,7 +50,10 @@ rules() { $u ALL=(root) NOPASSWD: $YBIN/y-localhost * # y-k8s-ingress-hosts: /etc/hosts management ($u) -$u ALL=(root) NOPASSWD: $YBIN/y-k8s-ingress-hosts-v*-bin * +# Argument-free on purpose: the rule this replaced covered the +# k8s-ingress-hosts binary with any flags, and its -host-file flag +# turned that into a write-any-file-as-root grant. +$u ALL=(root) NOPASSWD: $YBIN/y-etc-hosts-write EOF done } diff --git a/bin/y-etc-hosts-write b/bin/y-etc-hosts-write new file mode 100755 index 00000000..7ffa7622 --- /dev/null +++ b/bin/y-etc-hosts-write @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +[ -z "${DEBUG:-}" ] || set -x +set -eo pipefail + +YHELP='y-etc-hosts-write - Replace /etc/hosts with content read from stdin + +Usage: y-etc-hosts-write < newhosts + +The single privileged step of the /etc/hosts tooling. It takes no arguments, so +the NOPASSWD rule y-cluster-sudoers grants for it cannot be pointed at another +path the way a rule for a flag-taking binary can. + +Refuses input that is empty or has no 127.0.0.1 entry, and swaps the file in +with rename(2) so a reader never sees a half-written hosts file. + +Environment: + Y_ETC_HOSTS Target path (default: /etc/hosts) + +Dependencies: + +Exit codes: + 0 Written, or already identical + 1 Usage error + 2 Input refused by the safety checks + 3 Target not writable (run through sudo) +' + +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; +esac + +if [ $# -ne 0 ]; then + echo "ERROR: takes no arguments, reads the new file from stdin" >&2 + exit 1 +fi + +TARGET="${Y_ETC_HOSTS:-/etc/hosts}" +TARGET_DIR="$(dirname "$TARGET")" + +if [ ! -w "$TARGET" ] || [ ! -w "$TARGET_DIR" ]; then + echo "ERROR: $TARGET is not writable as $(id -un), run through sudo" >&2 + exit 3 +fi + +# Same directory as the target, so the swap below is a rename within one filesystem +TMP=$(mktemp "$TARGET_DIR/.hosts.XXXXXX") +trap 'rm -f "$TMP"' EXIT + +cat > "$TMP" + +if [ ! -s "$TMP" ]; then + echo "ERROR: refusing to write an empty $TARGET" >&2 + exit 2 +fi +if ! grep -qE '^[[:space:]]*127\.0\.0\.1[[:space:]]' "$TMP"; then + echo "ERROR: refusing input without a 127.0.0.1 entry" >&2 + exit 2 +fi + +if cmp -s "$TMP" "$TARGET"; then + echo "# $TARGET already up to date" + exit 0 +fi + +chmod 0644 "$TMP" +mv "$TMP" "$TARGET" +echo "# wrote $TARGET" diff --git a/bin/y-k8s-ingress-hosts b/bin/y-k8s-ingress-hosts index 6405bfae..f720551f 100755 --- a/bin/y-k8s-ingress-hosts +++ b/bin/y-k8s-ingress-hosts @@ -1,44 +1,68 @@ #!/usr/bin/env bash -[ -z "$DEBUG" ] || set -x +[ -z "${DEBUG:-}" ] || set -x set -eo pipefail -YBIN="$(dirname $0)" +YBIN="$(dirname "$0")" -[ -z "$KUBECONFIG" ] && echo "This script requires a KUBECONFIG" && exit 1 +YHELP='y-k8s-ingress-hosts - Sync /etc/hosts with cluster ingress and gateway hosts -CTX="" -CHECK=false -ENSURE=false -EXPLICIT_HOST_IP="" -PASSTHROUGH=() +Usage: y-k8s-ingress-hosts --context=NAME [options] -while [ $# -gt 0 ]; do - case "$1" in - -h|--help) - cat >&2 < spec.gatewayClassName -> GatewayClass metadata.annotations[yolean.se/dns-hint-ip] 2. Any GatewayClass carrying yolean.se/dns-hint-ip (used when no - consumer Gateway/ystack.ystack exists, e.g. y-cluster's + consumer Gateway/ystack.ystack exists, e.g. the y-cluster appliance flow where the GatewayClass is the only artefact). 3. Gateway/ystack.ystack metadata.annotations[yolean.se/override-ip] (legacy, pre-dates the dns-hint-ip contract). y-cluster provision stamps the dns-hint-ip annotation when the host forwards guest:80. -EOF - exit 0 ;; + +Environment: + KUBECONFIG required + Y_HOST_IP same as --host-ip + Y_ETC_HOSTS target hosts file (default: /etc/hosts) + +Dependencies: + +Exit codes: + 0 Up to date, written, or no entries to write + 1 Usage error, or -check found drift without --ensure + 3 The k8s-ingress-hosts binary failed +' + +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; + -h) echo "$YHELP"; exit 0 ;; +esac + +[ -z "$KUBECONFIG" ] && echo "ERROR: this script requires a KUBECONFIG" >&2 && exit 1 + +CTX="" +CHECK=false +ENSURE=false +WRITE=false +EXPLICIT_HOST_IP="" +PASSTHROUGH=() + +while [ $# -gt 0 ]; do + case "$1" in --context=*) CTX="${1#*=}"; shift ;; + -write|--write) WRITE=true; shift ;; -check|--check) CHECK=true; shift ;; --ensure) ENSURE=true; shift ;; --host-ip=*) EXPLICIT_HOST_IP="${1#*=}"; shift ;; @@ -55,10 +79,13 @@ EOF esac done -[ -z "$CTX" ] && echo "Required: --context=NAME" && exit 1 +[ -z "$CTX" ] && echo "ERROR: required: --context=NAME" >&2 && exit 1 + +HOSTS_FILE="${Y_ETC_HOSTS:-/etc/hosts}" CONTEXT_KUBECONFIG=$(mktemp) -trap "rm -f $CONTEXT_KUBECONFIG" EXIT +NEW_HOSTS=$(mktemp) +trap 'rm -f "$CONTEXT_KUBECONFIG" "$NEW_HOSTS"' EXIT kubectl config view --raw --minify --context="$CTX" --request-timeout=5s > "$CONTEXT_KUBECONFIG" # Resolve the host-side dial IP, in priority order: @@ -96,7 +123,7 @@ if [ -z "$HOST_IP" ]; then -o go-template='{{range .items}}{{$ip := index .metadata.annotations "yolean.se/dns-hint-ip"}}{{if $ip}}{{.metadata.name}}={{$ip}}{{"\n"}}{{end}}{{end}}' \ 2>/dev/null || true) # y-script-lint:disable=or-true # no GatewayClasses is a normal pre-install state DISTINCT_IPS=$(echo "$HINT_LIST" | awk -F= 'NF==2 && $2!="" {print $2}' | sort -u) - IP_COUNT=$(echo "$DISTINCT_IPS" | grep -c .) + IP_COUNT=$(echo "$DISTINCT_IPS" | grep -c . || true) # y-script-lint:disable=or-true # grep -c prints 0 but exits 1 with no matches, which is the no-annotation case if [ "$IP_COUNT" = "1" ]; then HOST_IP="$DISTINCT_IPS" GC_NAME=$(echo "$HINT_LIST" | awk -F= 'NF==2 && $2!="" {print $1; exit}') @@ -118,16 +145,24 @@ if [ -n "$HOST_IP" ]; then PASSTHROUGH+=("-override-ip" "$HOST_IP") fi -version=$(y-bin-download $YBIN/y-bin.optional.yaml k8s-ingress-hosts) +version=$(y-bin-download "$YBIN/y-bin.optional.yaml" k8s-ingress-hosts) +BIN="$YBIN/y-k8s-ingress-hosts-v${version}-bin" + +echo "# reading k8s ingress resources..." +if ! RENDERED=$("$BIN" -kubeconfig "$CONTEXT_KUBECONFIG" "${PASSTHROUGH[@]}" 2>&1); then + echo "ERROR: $BIN failed:" >&2 + echo "$RENDERED" >&2 + exit 3 +fi +ENTRIES=$(echo "$RENDERED" | grep -v '^#' || true) # y-script-lint:disable=or-true # grep exits 1 when the binary emitted comments only, which is the no-entries case handled below if $CHECK || $ENSURE; then - NEEDED=$($YBIN/y-k8s-ingress-hosts-v${version}-bin -kubeconfig "$CONTEXT_KUBECONFIG" "${PASSTHROUGH[@]}" 2>/dev/null | grep -v '^#') STALE=0 while IFS= read -r line; do [ -z "$line" ] && continue EXPECTED_IP=$(echo "$line" | awk '{print $1}') HOST=$(echo "$line" | awk '{print $2}') - ACTUAL=$(grep -E "^[^#]*[[:space:]]$HOST([[:space:]]|$)" /etc/hosts 2>/dev/null || true) # y-script-lint:disable=or-true # grep exits 1 on no match -- expected for a missing-host check + ACTUAL=$(grep -E "^[^#]*[[:space:]]$HOST([[:space:]]|$)" "$HOSTS_FILE" 2>/dev/null || true) # y-script-lint:disable=or-true # grep exits 1 on no match -- expected for a missing-host check if [ -z "$ACTUAL" ]; then echo "Missing: $line" STALE=1 @@ -136,46 +171,46 @@ if $CHECK || $ENSURE; then echo "Stale: $HOST has $ACTUAL_IP, expected $EXPECTED_IP" STALE=1 fi - done <<< "$NEEDED" + done <<< "$ENTRIES" if [ $STALE -eq 0 ]; then - echo "# /etc/hosts is up to date" + echo "# $HOSTS_FILE is up to date" exit 0 fi if ! $ENSURE; then - echo "# /etc/hosts needs updating. Run with -write or --ensure to fix." + echo "# $HOSTS_FILE needs updating. Run with -write or --ensure to fix." exit 1 fi - echo "# /etc/hosts needs updating, writing ..." - PASSTHROUGH+=("-write") + echo "# $HOSTS_FILE needs updating, writing ..." + WRITE=true fi -# Guard: don't write an empty block that clears existing entries. -# Preview without -write to check if there are entries. -_PREVIEW_ARGS=() -for _a in "${PASSTHROUGH[@]}"; do - [ "$_a" = "-write" ] || _PREVIEW_ARGS+=("$_a") -done -echo "# reading k8s ingress resources..." -_PREVIEW=$($YBIN/y-k8s-ingress-hosts-v${version}-bin -kubeconfig "$CONTEXT_KUBECONFIG" "${_PREVIEW_ARGS[@]}" 2>/dev/null | grep -v '^#') -if [ -z "$_PREVIEW" ]; then - echo "# no ingress/gateway entries found, skipping write to preserve existing /etc/hosts" +# Without -write/-check/--ensure this is a preview, like the binary itself +if ! $WRITE; then + echo "$RENDERED" exit 0 fi -# One-line stdout log when this invocation will actually mutate -# /etc/hosts (i.e. -write is in PASSTHROUGH, set either explicitly -# by the caller or appended above by --ensure on detected drift). -# Useful as a converge-trace breadcrumb so a yconverge exec check -# that ran y-k8s-ingress-hosts is visibly attributable. -WRITE_MODE=false -for _a in "${PASSTHROUGH[@]}"; do - [ "$_a" = "-write" ] && WRITE_MODE=true -done -if $WRITE_MODE; then - HOST_COUNT=$(echo "$_PREVIEW" | wc -l | tr -d ' ') - echo "y-k8s-ingress-hosts: writing $HOST_COUNT host entries to /etc/hosts" +# Guard: an empty render would otherwise clear the managed block +if [ -z "$ENTRIES" ]; then + echo "# no ingress/gateway entries found, skipping write to preserve existing $HOSTS_FILE" + exit 0 fi -[ $(id -u) -ne 0 ] && exec sudo $YBIN/y-k8s-ingress-hosts-v${version}-bin -kubeconfig "$CONTEXT_KUBECONFIG" "${PASSTHROUGH[@]}" +HOST_COUNT=$(echo "$ENTRIES" | wc -l | tr -d ' ') +echo "y-k8s-ingress-hosts: writing $HOST_COUNT host entries to $HOSTS_FILE" + +# Render as the invoking user: the binary merges its managed block into a copy, +# and only that copy is handed to the privileged step. +cp "$HOSTS_FILE" "$NEW_HOSTS" +"$BIN" -kubeconfig "$CONTEXT_KUBECONFIG" -host-file "$NEW_HOSTS" -write "${PASSTHROUGH[@]}" >/dev/null -$YBIN/y-k8s-ingress-hosts-v${version}-bin -kubeconfig "$CONTEXT_KUBECONFIG" "${PASSTHROUGH[@]}" || exit $? +if cmp -s "$NEW_HOSTS" "$HOSTS_FILE"; then + echo "# $HOSTS_FILE already up to date" + exit 0 +fi + +if [ "$HOSTS_FILE" = "/etc/hosts" ] && [ "$(id -u)" -ne 0 ]; then + sudo "$YBIN/y-etc-hosts-write" < "$NEW_HOSTS" +else + Y_ETC_HOSTS="$HOSTS_FILE" "$YBIN/y-etc-hosts-write" < "$NEW_HOSTS" +fi From 181562b8254549534cccca937e4971b9ed1c6bcf Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 31 Aug 2026 07:36:38 +0200 Subject: [PATCH 2/3] y-kubefwd: upgrade to 1.25.16 and keep the credential out of root kubefwd itself cannot avoid root. Upstream 1.25.16 says so in its own help text -- "kubefwd needs sudo for /etc/hosts and network interfaces" -- and its only no-sudo path is the mcp subcommand talking to an already-escalated instance over the REST API. But sudo -E handed root the entire environment: KUBECONFIG, HOME and PATH. Root therefore ran gke-gcloud-auth-plugin and gcloud, which is what put root-owned files under the user's ~/.kube and ~/.config/gcloud, and made every token refresh a root-run gcloud. Now the script generates a kubeconfig minified to the context and rewrites its exec credential to drop back to the invoking user (sudo -u USER -H), so the forwarder is the only thing running as root and all it ever receives is a bearer token. Contexts without an exec block (client certs, local clusters) pass through untouched. Preflight runs as the user so an expired login is reported in your own session instead of from inside the root process. The env hop in those exec args is load-bearing: sudo resets PATH to secure_path, where the plugin fails with exec: "gcloud": executable file not found in $PATH so PATH is restored to the plugin's own directory, where gcloud is a sibling. 1.25.12 -> 1.25.16 also brings idle mode: with no -n/-A kubefwd now starts its REST API and waits instead of forwarding, which the script now says out loud rather than looking hung. Running the script itself as root is refused, and the already-running guard exits 10 with pgrep -l (BSD pgrep has no -a). Verified with a stub sudo: argument assembly, the generated credential, that the rewritten exec command really returns an ExecCredential, and that 1.25.16 accepts our flags ahead of the svc subcommand. Co-Authored-By: Claude Opus 5 (1M context) --- bin/y-bin.optional.yaml | 10 ++-- bin/y-kubefwd | 129 +++++++++++++++++++++++++++++++++------- 2 files changed, 113 insertions(+), 26 deletions(-) diff --git a/bin/y-bin.optional.yaml b/bin/y-bin.optional.yaml index 7ed7c309..364cceab 100755 --- a/bin/y-bin.optional.yaml +++ b/bin/y-bin.optional.yaml @@ -25,14 +25,14 @@ k3d: linux_arm64: 0b8110f2229631af7402fb828259330985918b08fefd38b7f1b788a1c8687216 kubefwd: - version: 1.25.12 + version: 1.25.16 templates: download: https://github.com/txn2/kubefwd/releases/download/v${version}/kubefwd_${Os}_${xarch}.tar.gz sha256: - darwin_amd64: a9edf5398e0bcebf1a77eb8a74185498035cd624749d9b9ab08cf3ba832b7543 - darwin_arm64: 70d60486d7d03b2dda99eee1a10d12549b861ae354df0225956a7552d8fb9a83 - linux_amd64: 047b596f0d672111199265ccc7f830f1578eb9ad9be7dc5a30f2174b261534f2 - linux_arm64: 04bc8d17e2417ead85bd2ba12a4c0eb517aa873c5611903656d7f579fe146a00 + darwin_amd64: 905a3dd8a70d5acfcfe2a03f41feec264ac0c11cf7e5633b489716d719b64a6b + darwin_arm64: fc7a0126559cdd9e5cf8ddaf6a271b2d06381787bc3d3005ae48739a88f7d56c + linux_amd64: 07275cad05b2427069071160125b8cb29e94dd44582f685ce6d966fa9e7fb7d7 + linux_arm64: e01ade02d919be2c7e306543f0a65de2e629c254ef16b51ecb45830b0044a3e8 archive: tool: tar path: kubefwd diff --git a/bin/y-kubefwd b/bin/y-kubefwd index 14091734..9e578391 100755 --- a/bin/y-kubefwd +++ b/bin/y-kubefwd @@ -1,34 +1,121 @@ #!/usr/bin/env bash -[ -z "$DEBUG" ] || set -x +[ -z "${DEBUG:-}" ] || set -x set -eo pipefail -YBIN="$(dirname $0)" +YBIN="$(dirname "$0")" -ctx=$1 -case $ctx in - "--context="*) shift 1 ;; - *) echo "Initial arg must be --context=" && exit 1 ;; +YHELP='y-kubefwd - Port-forward cluster services, escalating the forwarder only + +Usage: y-kubefwd --context=NAME [kubefwd args] + +Options: + --context=NAME kubeconfig context name (required, must be the first argument) + +kubefwd itself has to run as root: upstream states it needs sudo for /etc/hosts +and the loopback aliases, and offers no unprivileged mode. What this script does +not do is give root the cluster credential. It hands kubefwd a generated +kubeconfig whose credential plugin is re-invoked as the calling user, so the +plugin, gcloud and their caches stay out of root. + +Defaults added unless present in the arguments: + -l ystack-kubefwd!=never skip services opting out + --domain=NAME for contexts other than local + +Examples: + y-kubefwd --context=local svc -n ystack -m 443:59443 + +Environment: + KUBECONFIG required + YSTACK_BUILDKIT_REQUIRE when true, require buildkitd before forwarding + +Dependencies: + +Exit codes: + 0 kubefwd exited normally + 1 usage error + 2 the kubeconfig context has a credential plugin that is not on PATH + 10 another kubefwd is already running +' + +case "${1:-}" in + help) echo "$YHELP"; exit 0 ;; + --help) echo "$YHELP"; exit 0 ;; esac +ctx="${1:-}" +case "$ctx" in + --context=*) shift ;; + *) echo "ERROR: initial arg must be --context=NAME" >&2; exit 1 ;; +esac CONTEXT_NAME="${ctx#--context=}" -[ "$YSTACK_BUILDKIT_REQUIRE" != "true" ] || [ $(id -u) -eq 0 ] || [ -z "$ctx" ] || y-buildkitd-available $ctx - -version=$(y-bin-download $YBIN/y-bin.optional.yaml kubefwd) +[ -z "$KUBECONFIG" ] && echo "ERROR: this script requires a KUBECONFIG" >&2 && exit 1 -# Fail early if there's already a kubefwd process running -# corrupt /etc/hosts and cause DNS resolution failures -if [ $(id -u) -ne 0 ] && pgrep -f "y-kubefwd-v.*-bin" >/dev/null 2>&1; then - echo "ERROR: An existing kubefwd process is already running:" - pgrep -la "y-kubefwd-v.*-bin" - echo "Kill it first: sudo pkill -f 'y-kubefwd-v.*-bin'" +if [ "$(id -u)" -eq 0 ]; then + echo "ERROR: run y-kubefwd as yourself, it escalates the forwarder on its own" >&2 exit 1 fi -[ $(id -u) -eq 0 ] || kubectl $ctx get pods >/dev/null -[ $(id -u) -ne 0 ] && echo "su privileges required for kubefwd" && exec sudo -E $0 $ctx "$@" +[ "${YSTACK_BUILDKIT_REQUIRE:-}" != "true" ] || y-buildkitd-available "$ctx" -addargs="$ctx" -[[ "$*" == *-l* ]] || addargs="$addargs -l ystack-kubefwd!=never" -[[ "$CONTEXT_NAME" == "local" ]] || [[ "$*" == *--domain* ]] || addargs="$addargs --domain=$CONTEXT_NAME" +# A second kubefwd would fight over the same /etc/hosts block +if pgrep -f "y-kubefwd-v.*-bin" >/dev/null 2>&1; then + echo "ERROR: a kubefwd process is already running:" >&2 + pgrep -l -f "y-kubefwd-v.*-bin" >&2 + echo "Kill it first: sudo pkill -f 'y-kubefwd-v.*-bin'" >&2 + exit 10 +fi + +version=$(y-bin-download "$YBIN/y-bin.optional.yaml" kubefwd) +BIN="$YBIN/y-kubefwd-v${version}-bin" + +KCONF=$(mktemp) +trap 'rm -f "$KCONF"' EXIT +kubectl config view --raw --minify --context="$CONTEXT_NAME" --request-timeout=5s > "$KCONF" + +# Preflight as yourself, so an expired login is reported here rather than from +# inside the root process +kubectl --kubeconfig="$KCONF" --request-timeout=10s get pods >/dev/null + +USER_NAME=$(kubectl --kubeconfig="$KCONF" config view -o jsonpath='{.users[0].name}') +EXEC_CMD=$(kubectl --kubeconfig="$KCONF" config view --raw -o jsonpath='{.users[0].user.exec.command}') + +if [ -n "$EXEC_CMD" ]; then + # sudo resets PATH, so the plugin has to be named absolutely + if ! EXEC_ABS=$(command -v "$EXEC_CMD"); then + echo "ERROR: credential plugin $EXEC_CMD of context $CONTEXT_NAME not found in PATH" >&2 + exit 2 + fi + EXEC_API=$(kubectl --kubeconfig="$KCONF" config view --raw -o jsonpath='{.users[0].user.exec.apiVersion}') + EXEC_ARGS=() + while IFS= read -r a; do + [ -z "$a" ] || EXEC_ARGS+=(--exec-arg="$a") + done <<< "$(kubectl --kubeconfig="$KCONF" config view --raw \ + -o jsonpath='{range .users[0].user.exec.args[*]}{@}{"\n"}{end}')" + + # Drop back to the invoking user for the credential, so the plugin and the + # gcloud it calls never run as root. provideClusterInfo is turned off because + # sudo would strip KUBERNETES_EXEC_INFO from the plugin environment anyway. + # env restores a PATH holding the plugin siblings: under sudo secure_path, + # gke-gcloud-auth-plugin cannot find the gcloud it shells out to. + kubectl --kubeconfig="$KCONF" config set-credentials "$USER_NAME" \ + --exec-command="$(command -v sudo)" \ + --exec-api-version="${EXEC_API:-client.authentication.k8s.io/v1beta1}" \ + --exec-interactive-mode=IfAvailable \ + --exec-provide-cluster-info=false \ + --exec-arg=-u --exec-arg="$(id -un)" --exec-arg=-H --exec-arg=-- \ + --exec-arg=/usr/bin/env --exec-arg="PATH=$(dirname "$EXEC_ABS"):/usr/bin:/bin" \ + --exec-arg="$EXEC_ABS" "${EXEC_ARGS[@]}" >/dev/null +fi + +addargs=() +[[ "$*" == *-l* ]] || addargs+=(-l "ystack-kubefwd!=never") +[[ "$CONTEXT_NAME" == "local" ]] || [[ "$*" == *--domain* ]] || addargs+=("--domain=$CONTEXT_NAME") + +case " $* " in + *" -n "*|*" --namespace"*|*" -A "*|*" --all-namespaces "*) ;; + *) echo "# note: no namespace given, kubefwd $version starts in idle mode with its REST API enabled" ;; +esac -$YBIN/y-kubefwd-v${version}-bin $addargs "$@" || exit $? +# Only the forwarder is escalated, and without -E: root gets no KUBECONFIG, no +# HOME and no PATH of yours, just the generated kubeconfig above. +sudo -- "$BIN" -c "$KCONF" "${addargs[@]}" "$@" From 57f414ce929208666489aba92f5b104b3ce8ce81 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 31 Aug 2026 07:57:06 +0200 Subject: [PATCH 3/3] y-kubefwd: hand root a static credential, not a plugin to run The previous commit kept the exec plugin in the kubeconfig and only moved it back to the invoking user with sudo -u. Root still drove it, and kubefwd re-invokes the credential every minute or two (552 config-helper calls on 2026-08-11, 1071 on 08-12), so every one of those became a sudo as well. It also left a gcloud-shaped hop in a script that has no business knowing about gcloud. Resolve the credential here instead, once, as the invoking user, and give kubefwd a kubeconfig holding only the token that came back. This is the client-go exec protocol, so nothing about it is gcloud specific: args, env and KUBERNETES_EXEC_INFO all come from the context's own exec block, whatever plugin it names. Contexts without one, client certificates on local clusters or a static token, pass through untouched, and that is also where the long sessions happen. The trade is now stated instead of implicit: a GKE token lasts an hour and is not renewed for a running kubefwd, so its expiry is printed at startup. Two details worth keeping: - The exec block has to be dropped, not just overwritten with a token. kubectl keeps both and exec wins. Done with a fresh user entry plus a re-minify, because unsetting users.NAME.exec needs a dotted path that breaks on user names containing dots. - Plugin stderr is captured separately. Folding it into stdout corrupts the ExecCredential of any plugin that warns while succeeding, which a stub reproduced. Verified with stub plugins: the token path, a failing plugin, a plugin answering with client certificates, that KUBERNETES_EXEC_INFO arrives well-formed, and that a client-certificate context reaches kubefwd unchanged. No cluster was contacted. Co-Authored-By: Claude Opus 5 (1M context) --- bin/y-kubefwd | 88 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 30 deletions(-) diff --git a/bin/y-kubefwd b/bin/y-kubefwd index 9e578391..b1b0b835 100755 --- a/bin/y-kubefwd +++ b/bin/y-kubefwd @@ -12,9 +12,14 @@ Options: kubefwd itself has to run as root: upstream states it needs sudo for /etc/hosts and the loopback aliases, and offers no unprivileged mode. What this script does -not do is give root the cluster credential. It hands kubefwd a generated -kubeconfig whose credential plugin is re-invoked as the calling user, so the -plugin, gcloud and their caches stay out of root. +not do is let root near your login. Whatever credential plugin the context names +is run here, as you, and kubefwd gets a kubeconfig holding only the static +credential it returned. No plugin, no gcloud and no token cache runs as root. + +A credential that carries an expiry is not renewed for a running kubefwd, so the +expiry is printed at startup. That is an hour for a GKE token. Contexts that +authenticate with client certificates, the usual local cluster setup, and static +tokens in the kubeconfig do not expire and are passed through untouched. Defaults added unless present in the arguments: -l ystack-kubefwd!=never skip services opting out @@ -32,7 +37,7 @@ Dependencies: Exit codes: 0 kubefwd exited normally 1 usage error - 2 the kubeconfig context has a credential plugin that is not on PATH + 2 the context credential could not be resolved into a static one 10 another kubefwd is already running ' @@ -69,44 +74,67 @@ version=$(y-bin-download "$YBIN/y-bin.optional.yaml" kubefwd) BIN="$YBIN/y-kubefwd-v${version}-bin" KCONF=$(mktemp) -trap 'rm -f "$KCONF"' EXIT +KCONF_STATIC=$(mktemp) +PLUGIN_ERR=$(mktemp) +trap 'rm -f "$KCONF" "$KCONF_STATIC" "$PLUGIN_ERR"' EXIT kubectl config view --raw --minify --context="$CONTEXT_NAME" --request-timeout=5s > "$KCONF" -# Preflight as yourself, so an expired login is reported here rather than from -# inside the root process -kubectl --kubeconfig="$KCONF" --request-timeout=10s get pods >/dev/null - -USER_NAME=$(kubectl --kubeconfig="$KCONF" config view -o jsonpath='{.users[0].name}') EXEC_CMD=$(kubectl --kubeconfig="$KCONF" config view --raw -o jsonpath='{.users[0].user.exec.command}') if [ -n "$EXEC_CMD" ]; then - # sudo resets PATH, so the plugin has to be named absolutely - if ! EXEC_ABS=$(command -v "$EXEC_CMD"); then - echo "ERROR: credential plugin $EXEC_CMD of context $CONTEXT_NAME not found in PATH" >&2 - exit 2 - fi - EXEC_API=$(kubectl --kubeconfig="$KCONF" config view --raw -o jsonpath='{.users[0].user.exec.apiVersion}') + command -v jq >/dev/null || { echo "ERROR: jq is needed to read the credential plugin output" >&2; exit 2; } + + # The client-go exec protocol, not anything gcloud specific: run the plugin + # the context names, once, as the invoking user, and keep only its answer. EXEC_ARGS=() while IFS= read -r a; do - [ -z "$a" ] || EXEC_ARGS+=(--exec-arg="$a") + [ -z "$a" ] || EXEC_ARGS+=("$a") done <<< "$(kubectl --kubeconfig="$KCONF" config view --raw \ -o jsonpath='{range .users[0].user.exec.args[*]}{@}{"\n"}{end}')" + EXEC_ENV=() + while IFS= read -r e; do + [ -z "$e" ] || EXEC_ENV+=("$e") + done <<< "$(kubectl --kubeconfig="$KCONF" config view --raw \ + -o jsonpath='{range .users[0].user.exec.env[*]}{.name}={.value}{"\n"}{end}')" + EXEC_API=$(kubectl --kubeconfig="$KCONF" config view --raw -o jsonpath='{.users[0].user.exec.apiVersion}') + if [ "$(kubectl --kubeconfig="$KCONF" config view --raw -o jsonpath='{.users[0].user.exec.provideClusterInfo}')" = "true" ]; then + EXEC_ENV+=("KUBERNETES_EXEC_INFO=$(jq -n \ + --arg api "${EXEC_API:-client.authentication.k8s.io/v1beta1}" \ + --arg server "$(kubectl --kubeconfig="$KCONF" config view --raw -o jsonpath='{.clusters[0].cluster.server}')" \ + --arg ca "$(kubectl --kubeconfig="$KCONF" config view --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}')" \ + '{apiVersion:$api,kind:"ExecCredential",spec:{cluster:{server:$server,"certificate-authority-data":$ca,config:null},interactive:false}}')") + fi - # Drop back to the invoking user for the credential, so the plugin and the - # gcloud it calls never run as root. provideClusterInfo is turned off because - # sudo would strip KUBERNETES_EXEC_INFO from the plugin environment anyway. - # env restores a PATH holding the plugin siblings: under sudo secure_path, - # gke-gcloud-auth-plugin cannot find the gcloud it shells out to. - kubectl --kubeconfig="$KCONF" config set-credentials "$USER_NAME" \ - --exec-command="$(command -v sudo)" \ - --exec-api-version="${EXEC_API:-client.authentication.k8s.io/v1beta1}" \ - --exec-interactive-mode=IfAvailable \ - --exec-provide-cluster-info=false \ - --exec-arg=-u --exec-arg="$(id -un)" --exec-arg=-H --exec-arg=-- \ - --exec-arg=/usr/bin/env --exec-arg="PATH=$(dirname "$EXEC_ABS"):/usr/bin:/bin" \ - --exec-arg="$EXEC_ABS" "${EXEC_ARGS[@]}" >/dev/null + # Plugin stderr goes to its own file: merging it into stdout would corrupt the + # ExecCredential of any plugin that warns while succeeding + if ! CREDENTIAL=$(env "${EXEC_ENV[@]}" "$EXEC_CMD" "${EXEC_ARGS[@]}" 2>"$PLUGIN_ERR"); then + echo "ERROR: credential plugin $EXEC_CMD of context $CONTEXT_NAME failed:" >&2 + cat "$PLUGIN_ERR" >&2 + exit 2 + fi + TOKEN=$(echo "$CREDENTIAL" | jq -r '.status.token // empty' 2>/dev/null || true) # y-script-lint:disable=or-true # a plugin answering with something other than an ExecCredential is reported below + if [ -z "$TOKEN" ]; then + echo "ERROR: credential plugin $EXEC_CMD returned no usable token" >&2 + echo "Only a bearer token can be made static here, not client certificates or a non-ExecCredential answer" >&2 + cat "$PLUGIN_ERR" >&2 + exit 2 + fi + + # A fresh user entry, then re-minify to drop the exec one: editing the exec + # block in place needs a dotted config path, which breaks on names with dots + kubectl --kubeconfig="$KCONF" config set-credentials y-kubefwd --token="$TOKEN" >/dev/null + kubectl --kubeconfig="$KCONF" config set-context "$CONTEXT_NAME" --user=y-kubefwd >/dev/null + kubectl --kubeconfig="$KCONF" config view --raw --minify > "$KCONF_STATIC" + mv "$KCONF_STATIC" "$KCONF" + + EXPIRY=$(echo "$CREDENTIAL" | jq -r '.status.expirationTimestamp // empty') + [ -z "$EXPIRY" ] || echo "# note: this credential expires $EXPIRY, kubefwd loses cluster access then" fi +# Preflight with exactly what kubefwd will use, so a bad or expired credential +# is reported here rather than from inside the root process +kubectl --kubeconfig="$KCONF" --request-timeout=10s get pods >/dev/null + addargs=() [[ "$*" == *-l* ]] || addargs+=(-l "ystack-kubefwd!=never") [[ "$CONTEXT_NAME" == "local" ]] || [[ "$*" == *--domain* ]] || addargs+=("--domain=$CONTEXT_NAME")