From a88d38be9e71d7d7fe73bcdddf9f35a5593f2774 Mon Sep 17 00:00:00 2001
From: Wu Yi <typhoonzero1986@gmail.com>
Date: Wed, 10 Jun 2026 15:35:53 +0800
Subject: [PATCH 1/2] test: remove environment-specific e2e defaults

---
 e2e/cases/c11_qwen3_06b_mindspore.sh     |  15 +++-
 e2e/cases/c12_kueue_preemption.sh        |  26 +++---
 e2e/cases/c1_smoke_gpu.sh                |  13 +--
 e2e/cases/c2_kubeflow_trainer_mnist.sh   |  13 +--
 e2e/cases/c3_traininghub_sft.sh          |  14 ++--
 e2e/cases/c4_traininghub_osft.sh         |  11 +--
 e2e/cases/c5_trainer_v2_llamafactory.sh  |  36 ++++----
 e2e/cases/c6_volcanojob_llamafactory.sh  |  22 +++--
 e2e/cases/c7_smoke_npu.sh                |  26 +++---
 e2e/cases/c8_trainer_v2_mindspeed_npu.sh |  18 ++--
 e2e/cases/c9_qwen3_finetune_verify.sh    |  15 +++-
 e2e/lib.sh                               | 102 ++++++++++++++++++++---
 e2e/run_all.sh                           |   2 +-
 13 files changed, 209 insertions(+), 104 deletions(-)

diff --git a/e2e/cases/c11_qwen3_06b_mindspore.sh b/e2e/cases/c11_qwen3_06b_mindspore.sh
index 1d1659b..51faa08 100755
--- a/e2e/cases/c11_qwen3_06b_mindspore.sh
+++ b/e2e/cases/c11_qwen3_06b_mindspore.sh
@@ -10,9 +10,16 @@ set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 source "${HERE}/../lib.sh"
 
+require_env NPU_NAMESPACE "namespace for NPU e2e resources"
+require_env NPU_RESOURCE_NAME "extended resource name for one NPU, for example huawei.com/Ascend910B4"
 NS="${NPU_NAMESPACE}"
 JOB_NAME="c11-mindspore-cann-smoke-$(printf '%05x' $$)"
 IMAGE="${C11_IMAGE:-docker.io/alaudadockerhub/alauda-workbench-jupyter-mindspore-cann-py312-ubi9:v0.1.7}"
+IMAGE_PULL_SECRET="${C11_IMAGE_PULL_SECRET:-${E2E_IMAGE_PULL_SECRET:-}}"
+NPU_RESOURCE_VALUE="${NPU_RESOURCE_VALUE:-1}"
+NPU_MEMORY_RESOURCE_NAME="${NPU_MEMORY_RESOURCE_NAME:-}"
+NPU_MEMORY_RESOURCE_VALUE="${NPU_MEMORY_RESOURCE_VALUE:-8192}"
+NPU_RUNTIME_CLASS="${NPU_RUNTIME_CLASS:-}"
 
 cleanup() {
   npu_kc -n "${NS}" delete job "${JOB_NAME}" --ignore-not-found --wait=false || true
@@ -20,7 +27,7 @@ cleanup() {
 trap cleanup EXIT
 
 log "C11: submitting Job ${JOB_NAME} (image=${IMAGE})"
-cat <<YAML | retry_create npu_kc >/dev/null
+cat <<YAML | mirror_dockerhub "${NPU_DH_MIRROR}" | retry_create npu_kc >/dev/null
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -35,7 +42,8 @@ spec:
       labels: { e2e.alauda.io/case: c11 }
     spec:
       restartPolicy: Never
-      runtimeClassName: ascend
+$(yaml_scalar_field 6 runtimeClassName "${NPU_RUNTIME_CLASS}")
+$(yaml_image_pull_secrets 6 "${IMAGE_PULL_SECRET}")
       securityContext: { runAsNonRoot: true, runAsUser: 1001, runAsGroup: 0, fsGroup: 1000 }
       containers:
         - name: probe
@@ -51,7 +59,8 @@ spec:
             limits:
               cpu: "2"
               memory: 8Gi
-              huawei.com/Ascend910: "1"
+$(yaml_resource_limit 14 "${NPU_RESOURCE_NAME}" "${NPU_RESOURCE_VALUE}")
+$(yaml_resource_limit 14 "${NPU_MEMORY_RESOURCE_NAME}" "${NPU_MEMORY_RESOURCE_VALUE}")
           command: [bash, -lc]
           args:
             - |
diff --git a/e2e/cases/c12_kueue_preemption.sh b/e2e/cases/c12_kueue_preemption.sh
index 797a31c..ec50bc1 100755
--- a/e2e/cases/c12_kueue_preemption.sh
+++ b/e2e/cases/c12_kueue_preemption.sh
@@ -24,6 +24,7 @@ set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 source "${HERE}/../lib.sh"
 
+require_env GPU_NAMESPACE "namespace for GPU e2e resources"
 NS="${GPU_NAMESPACE}"
 RUN_ID="$(printf '%05x' $$)-$(date -u +%s)"
 
@@ -40,13 +41,16 @@ RUNTIME="c12-checkpoint-runtime-${RUN_ID}"
 PVC_NAME="c12-ckpt-${RUN_ID}"
 TRAIN_LABEL="e2e.alauda.io/c12-run=${RUN_ID}"
 
-# Same harbor image C5/C6 already use — guaranteed cached on the GPU node.
-IMAGE="${LF_IMAGE:-build-harbor.alauda.cn/mlops/llamafactory0.9-cu126-amd64:v0.1.0-build.20260603021903}"
+IMAGE="${LF_IMAGE:-docker.io/alaudadockerhub/llamafactory0.9-cu126-amd64:v0.1.0}"
+IMAGE_PULL_SECRET="${LF_IMAGE_PULL_SECRET:-${E2E_IMAGE_PULL_SECRET:-}}"
+RWX_STORAGE_CLASS="${C12_RWX_STORAGE_CLASS:-${E2E_RWX_STORAGE_CLASS:-}}"
+NODE_SELECTOR_KEY="${C12_NODE_SELECTOR_KEY:-${E2E_GPU_NODE_SELECTOR_KEY:-}}"
+NODE_SELECTOR_VALUE="${C12_NODE_SELECTOR_VALUE:-${E2E_GPU_NODE_SELECTOR_VALUE:-}}"
 
 # --- Preflight: Kueue must be installed; skip otherwise. -----------------
 if ! gpu_kc api-resources --api-group=kueue.x-k8s.io 2>/dev/null | grep -q clusterqueues; then
   log "C12: kueue.x-k8s.io API group not present — install Kueue (see preemptible-trainjobs-with-kueue.mdx) and re-run"
-  exit 77
+  exit "${E2E_SKIP_RC}"
 fi
 
 cleanup() {
@@ -63,7 +67,7 @@ cleanup() {
 trap cleanup EXIT
 
 # --- Step 1: cohort + queues + priorities. -------------------------------
-# Quotas are sized for one Tesla P100 + HAMI (gpucores 50%, 4 GiB GPU mem):
+# Quotas are sized for one fractional HAMI GPU slice:
 # both inference and training Workloads each ask for that slice, so the
 # inference reclaim path empties exactly one borrowed training slot.
 log "C12: applying cohort (CQ ${CQ_INF}, ${CQ_TRAIN}; flavor ${FLAVOR})"
@@ -142,7 +146,7 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata: { name: ${PVC_NAME}, namespace: ${NS} }
 spec:
-  storageClassName: cephfs
+$(yaml_storage_class 2 "${RWX_STORAGE_CLASS}")
   accessModes: ["ReadWriteMany"]
   resources: { requests: { storage: 1Gi } }
 YAML
@@ -153,7 +157,7 @@ YAML
 # elsewhere. HuggingFace Trainer auto-resumes from the newest
 # checkpoint-N/ in CKPT_DIR if we pass it to .train(resume_from_checkpoint=).
 log "C12: applying TrainingRuntime ${RUNTIME}"
-cat <<YAML | retry_apply gpu_kc
+cat <<YAML | mirror_dockerhub "${GPU_DH_MIRROR}" | retry_apply gpu_kc
 apiVersion: trainer.kubeflow.org/v1alpha1
 kind: TrainingRuntime
 metadata:
@@ -176,8 +180,8 @@ spec:
               template:
                 spec:
                   terminationGracePeriodSeconds: 60
-                  nodeSelector: { kubernetes.io/hostname: 192.168.138.15 }
-                  imagePullSecrets: [{ name: harbor-mlops-regcred }]
+$(yaml_node_selector 18 "${NODE_SELECTOR_KEY}" "${NODE_SELECTOR_VALUE}")
+$(yaml_image_pull_secrets 18 "${IMAGE_PULL_SECRET}")
                   securityContext: { runAsNonRoot: true, runAsUser: 65534, runAsGroup: 65534, fsGroup: 65534 }
                   volumes:
                     - { name: workspace, emptyDir: {} }
@@ -330,7 +334,7 @@ log "C12: checkpoint visible on PVC"
 
 # --- Step 5: high-priority preemptor. -----------------------------------
 log "C12: submitting high-priority preemptor Job"
-INF_JOB=$(cat <<YAML | retry_create gpu_kc -o jsonpath='{.metadata.name}'
+INF_JOB=$(cat <<YAML | mirror_dockerhub "${GPU_DH_MIRROR}" | retry_create gpu_kc -o jsonpath='{.metadata.name}'
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -350,8 +354,8 @@ spec:
         e2e.alauda.io/c12-run: "${RUN_ID}"
     spec:
       restartPolicy: Never
-      nodeSelector: { kubernetes.io/hostname: 192.168.138.15 }
-      imagePullSecrets: [{ name: harbor-mlops-regcred }]
+$(yaml_node_selector 6 "${NODE_SELECTOR_KEY}" "${NODE_SELECTOR_VALUE}")
+$(yaml_image_pull_secrets 6 "${IMAGE_PULL_SECRET}")
       securityContext: { runAsNonRoot: true, runAsUser: 65534, runAsGroup: 65534, fsGroup: 65534 }
       containers:
         - name: serve
diff --git a/e2e/cases/c1_smoke_gpu.sh b/e2e/cases/c1_smoke_gpu.sh
index 049306b..01dc2dc 100755
--- a/e2e/cases/c1_smoke_gpu.sh
+++ b/e2e/cases/c1_smoke_gpu.sh
@@ -7,18 +7,21 @@ set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 source "${HERE}/../lib.sh"
 
+require_env GPU_NAMESPACE "namespace for GPU e2e resources"
 NS="${GPU_NAMESPACE}"
 ASSETS="${E2E_ROOT}/../docs/en/training_guides/assets/training-runtimes"
 
-log "C1: applying torch2.6-cu126-amd64 TrainingRuntime to ns/${NS} (rewriting docker.io → ${GPU_DH_MIRROR})"
-sed "s/namespace: kubeflow-admin-cpaas-io/namespace: ${NS}/" \
-  "${ASSETS}/torch2.6-cu126-amd64-trainingruntime.yaml" \
+if [ -n "${GPU_DH_MIRROR}" ]; then
+  log "C1: applying torch2.6-cu126-amd64 TrainingRuntime to ns/${NS} (Docker Hub mirror=${GPU_DH_MIRROR})"
+else
+  log "C1: applying torch2.6-cu126-amd64 TrainingRuntime to ns/${NS}"
+fi
+set_metadata_namespace "${NS}" < "${ASSETS}/torch2.6-cu126-amd64-trainingruntime.yaml" \
   | mirror_dockerhub "${GPU_DH_MIRROR}" \
   | retry_apply gpu_kc
 
 log "C1: submitting TrainJob from trainjob-smoke.yaml"
-TJ_NAME=$(sed -e "s/namespace: kubeflow-admin-cpaas-io/namespace: ${NS}/" \
-              "${ASSETS}/trainjob-smoke.yaml" \
+TJ_NAME=$(set_metadata_namespace "${NS}" < "${ASSETS}/trainjob-smoke.yaml" \
           | retry_create gpu_kc -o jsonpath='{.metadata.name}')
 log "C1: trainjob=${TJ_NAME}"
 
diff --git a/e2e/cases/c2_kubeflow_trainer_mnist.sh b/e2e/cases/c2_kubeflow_trainer_mnist.sh
index ffe4e48..b53b1ef 100755
--- a/e2e/cases/c2_kubeflow_trainer_mnist.sh
+++ b/e2e/cases/c2_kubeflow_trainer_mnist.sh
@@ -11,16 +11,17 @@ set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 source "${HERE}/../lib.sh"
 
+require_env GPU_NAMESPACE "namespace for GPU e2e resources"
 NS="${GPU_NAMESPACE}"
 
 log "C2: applying ClusterTrainingRuntime torch-distributed (from kubeflow-trainer-quick-start.md)"
 # Pulled directly out of docs/en/training_guides/kubeflow-trainer-quick-start.md.
-# Image override: docker-mirrors.alauda.cn proxies docker.io but the lazy-cache
-# returns EOF on the torch-distributed blobs (only torch2.6-* are cached). The
-# alauda-local registry mirror has the image pre-pulled. Doc itself stays at
-# the public alaudadockerhub/ name; only the test overrides for network reasons.
-TORCH_DIST_IMAGE="${TORCH_DIST_IMAGE:-152-231-registry.alauda.cn:60070/mlops/torch-distributed:v2.9.1-aml2}"
-cat <<'YAML' | sed "s@alaudadockerhub/torch-distributed:v2.9.1-aml2@${TORCH_DIST_IMAGE}@g" | retry_apply gpu_kc
+# Override TORCH_DIST_IMAGE only when the public Docker Hub image is mirrored locally.
+TORCH_DIST_IMAGE="${TORCH_DIST_IMAGE:-alaudadockerhub/torch-distributed:v2.9.1-aml2}"
+cat <<'YAML' \
+  | sed "s@alaudadockerhub/torch-distributed:v2.9.1-aml2@${TORCH_DIST_IMAGE}@g" \
+  | mirror_dockerhub "${GPU_DH_MIRROR}" \
+  | retry_apply gpu_kc
 apiVersion: trainer.kubeflow.org/v1alpha1
 kind: ClusterTrainingRuntime
 metadata:
diff --git a/e2e/cases/c3_traininghub_sft.sh b/e2e/cases/c3_traininghub_sft.sh
index e41a0a2..d420659 100755
--- a/e2e/cases/c3_traininghub_sft.sh
+++ b/e2e/cases/c3_traininghub_sft.sh
@@ -2,16 +2,17 @@
 # C3 — exercises training_hub.sft against the published
 # traininghub0.1-cu126-amd64:v0.1.0 runtime image (used by sft-comprehensive-tutorial.ipynb).
 # Tiny synthetic Qwen2-style HF checkpoint + synthetic JSONL are generated inside the
-# Pod so the case has no external dependency. nproc_per_node is forced to 1 because
-# the dev GPU node exposes only 1 whole GPU.
+# Pod so the case has no external dependency. nproc_per_node is forced to 1
+# to keep the smoke case small.
 set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 source "${HERE}/../lib.sh"
 
+require_env GPU_NAMESPACE "namespace for GPU e2e resources"
 NS="${GPU_NAMESPACE}"
 JOB_NAME="c3-traininghub-sft-$(printf '%05x' $$)"
-# Bundled-training_hub image built from kubeflow-plugin/training-runtimes/traininghub0.1-cu126-amd64.
-IMAGE="${C3_IMAGE:-build-harbor.alauda.cn/mlops/traininghub0.1-cu126-amd64:v0.1.0-build.20260609030710}"
+IMAGE="${C3_IMAGE:-docker.io/alaudadockerhub/traininghub0.1-cu126-amd64:v0.1.0}"
+IMAGE_PULL_SECRET="${C3_IMAGE_PULL_SECRET:-${E2E_IMAGE_PULL_SECRET:-}}"
 
 cleanup() {
   gpu_kc -n "${NS}" delete job "${JOB_NAME}" --ignore-not-found --wait=false || true
@@ -24,7 +25,7 @@ log "C3: submitting Job ${JOB_NAME} (image=${IMAGE})"
 # single_gpu_dev preset; we do the same here. Liger is disabled because the
 # image's CUDA *runtime* lacks nvcc (see training-runtimes.mdx — same caveat
 # applies to JIT op compilation).
-cat <<YAML | gpu_kc -n "${NS}" create -f -
+cat <<YAML | mirror_dockerhub "${GPU_DH_MIRROR}" | gpu_kc -n "${NS}" create -f -
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -44,8 +45,7 @@ spec:
         runAsUser: 1000
         runAsGroup: 1000
         fsGroup: 1000
-      imagePullSecrets:
-        - name: harbor-mlops-regcred
+$(yaml_image_pull_secrets 6 "${IMAGE_PULL_SECRET}")
       volumes:
         - name: workspace
           emptyDir: {}
diff --git a/e2e/cases/c4_traininghub_osft.sh b/e2e/cases/c4_traininghub_osft.sh
index 977118d..2928f62 100755
--- a/e2e/cases/c4_traininghub_osft.sh
+++ b/e2e/cases/c4_traininghub_osft.sh
@@ -6,9 +6,11 @@ set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 source "${HERE}/../lib.sh"
 
+require_env GPU_NAMESPACE "namespace for GPU e2e resources"
 NS="${GPU_NAMESPACE}"
 JOB_NAME="c4-traininghub-osft-$(printf '%05x' $$)"
-IMAGE="${C4_IMAGE:-build-harbor.alauda.cn/mlops/traininghub0.1-cu126-amd64:v0.1.0-build.20260609030710}"
+IMAGE="${C4_IMAGE:-docker.io/alaudadockerhub/traininghub0.1-cu126-amd64:v0.1.0}"
+IMAGE_PULL_SECRET="${C4_IMAGE_PULL_SECRET:-${E2E_IMAGE_PULL_SECRET:-}}"
 
 cleanup() {
   gpu_kc -n "${NS}" delete job "${JOB_NAME}" --ignore-not-found --wait=false || true
@@ -17,7 +19,7 @@ trap cleanup EXIT
 
 log "C4: submitting Job ${JOB_NAME} (image=${IMAGE})"
 
-cat <<YAML | gpu_kc -n "${NS}" create -f -
+cat <<YAML | mirror_dockerhub "${GPU_DH_MIRROR}" | gpu_kc -n "${NS}" create -f -
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -37,8 +39,7 @@ spec:
         runAsUser: 1000
         runAsGroup: 1000
         fsGroup: 1000
-      imagePullSecrets:
-        - name: harbor-mlops-regcred
+$(yaml_image_pull_secrets 6 "${IMAGE_PULL_SECRET}")
       volumes:
         - name: workspace
           emptyDir: {}
@@ -80,7 +81,7 @@ spec:
               cd /workspace
               # training_hub is bundled in the runtime image. mini_trainer (the
               # OSFT backend) unconditionally `import flash_attn`, which only
-              # supports sm_75+; the dev GPU node is Tesla P100 (sm_60).
+              # supports sm_75+; older GPU architectures need this stub.
               # Stub the module so the import succeeds; transformers falls
               # through to torch SDPA, which is what training_hub.sft uses too.
               mkdir -p /workspace/stubs/flash_attn
diff --git a/e2e/cases/c5_trainer_v2_llamafactory.sh b/e2e/cases/c5_trainer_v2_llamafactory.sh
index fd9838c..28f4c73 100755
--- a/e2e/cases/c5_trainer_v2_llamafactory.sh
+++ b/e2e/cases/c5_trainer_v2_llamafactory.sh
@@ -12,20 +12,20 @@ set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 source "${HERE}/../lib.sh"
 
+require_env GPU_NAMESPACE "namespace for GPU e2e resources"
 NS="${GPU_NAMESPACE}"
 # Per-run suffix so re-runs / concurrent runs don't share PVC contents or fight
 # over the same TrainingRuntime. The trap below tears down both.
 RUN_ID="$(printf '%05x' $$)-$(date -u +%s)"
 RUNTIME="c5-llamafactory-finetune-runtime-${RUN_ID}"
 PVC_NAME="c5-models-${RUN_ID}"
-# The doc notebook says "Use `alaudadockerhub/fine_tune_with_llamafactory:v0.1.11`,
-# or build your own". The Dockerhub copy is firewalled (mirror blob EOF on large
-# images), the harbor rebuild at the same tag lacks torch, and the catalog runtime
-# image `llamafactory0.9-cu126-amd64:v0.1.0` has the same blob-EOF issue. Use the
-# build-suffixed copy of the catalog image already cached on the dev cluster.
-IMAGE="${LF_IMAGE:-build-harbor.alauda.cn/mlops/llamafactory0.9-cu126-amd64:v0.1.0-build.20260603021903}"
+IMAGE="${LF_IMAGE:-docker.io/alaudadockerhub/llamafactory0.9-cu126-amd64:v0.1.0}"
+IMAGE_PULL_SECRET="${LF_IMAGE_PULL_SECRET:-${E2E_IMAGE_PULL_SECRET:-}}"
+RWX_STORAGE_CLASS="${C5_RWX_STORAGE_CLASS:-${E2E_RWX_STORAGE_CLASS:-}}"
+NODE_SELECTOR_KEY="${C5_NODE_SELECTOR_KEY:-${E2E_GPU_NODE_SELECTOR_KEY:-}}"
+NODE_SELECTOR_VALUE="${C5_NODE_SELECTOR_VALUE:-${E2E_GPU_NODE_SELECTOR_VALUE:-}}"
 
-log "C5: ensuring shared PVC ${PVC_NAME} exists (RWX on cephfs so any node can mount)"
+log "C5: ensuring shared RWX PVC ${PVC_NAME} exists"
 cat <<YAML | retry_apply gpu_kc
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -33,13 +33,13 @@ metadata:
   name: ${PVC_NAME}
   namespace: ${NS}
 spec:
-  storageClassName: cephfs
+$(yaml_storage_class 2 "${RWX_STORAGE_CLASS}")
   accessModes: ["ReadWriteMany"]
   resources: { requests: { storage: 4Gi } }
 YAML
 
 log "C5: applying TrainingRuntime ${RUNTIME} to ns/${NS} (image=${IMAGE})"
-cat <<YAML | retry_apply gpu_kc
+cat <<YAML | mirror_dockerhub "${GPU_DH_MIRROR}" | retry_apply gpu_kc
 apiVersion: trainer.kubeflow.org/v1alpha1
 kind: TrainingRuntime
 metadata:
@@ -64,10 +64,8 @@ spec:
               template:
                 spec:
                   securityContext: { runAsNonRoot: true, runAsUser: 65534, runAsGroup: 65534, fsGroup: 65534 }
-                  nodeSelector:
-                    kubernetes.io/hostname: 192.168.138.15
-                  imagePullSecrets:
-                    - name: harbor-mlops-regcred
+$(yaml_node_selector 18 "${NODE_SELECTOR_KEY}" "${NODE_SELECTOR_VALUE}")
+$(yaml_image_pull_secrets 18 "${IMAGE_PULL_SECRET}")
                   volumes:
                     - name: models
                       persistentVolumeClaim: { claimName: ${PVC_NAME} }
@@ -123,10 +121,8 @@ spec:
               template:
                 spec:
                   securityContext: { runAsNonRoot: true, runAsUser: 65534, runAsGroup: 65534, fsGroup: 65534 }
-                  nodeSelector:
-                    kubernetes.io/hostname: 192.168.138.15
-                  imagePullSecrets:
-                    - name: harbor-mlops-regcred
+$(yaml_node_selector 18 "${NODE_SELECTOR_KEY}" "${NODE_SELECTOR_VALUE}")
+$(yaml_image_pull_secrets 18 "${IMAGE_PULL_SECRET}")
                   volumes:
                     - name: models
                       persistentVolumeClaim: { claimName: ${PVC_NAME} }
@@ -192,10 +188,8 @@ spec:
               template:
                 spec:
                   securityContext: { runAsNonRoot: true, runAsUser: 65534, runAsGroup: 65534, fsGroup: 65534 }
-                  nodeSelector:
-                    kubernetes.io/hostname: 192.168.138.15
-                  imagePullSecrets:
-                    - name: harbor-mlops-regcred
+$(yaml_node_selector 18 "${NODE_SELECTOR_KEY}" "${NODE_SELECTOR_VALUE}")
+$(yaml_image_pull_secrets 18 "${IMAGE_PULL_SECRET}")
                   volumes:
                     - { name: workspace, emptyDir: {} }
                     - name: models
diff --git a/e2e/cases/c6_volcanojob_llamafactory.sh b/e2e/cases/c6_volcanojob_llamafactory.sh
index 52dc863..cde7425 100755
--- a/e2e/cases/c6_volcanojob_llamafactory.sh
+++ b/e2e/cases/c6_volcanojob_llamafactory.sh
@@ -9,14 +9,20 @@ set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 source "${HERE}/../lib.sh"
 
+require_env GPU_NAMESPACE "namespace for GPU e2e resources"
 NS="${GPU_NAMESPACE}"
 JOB_NAME="c6-vcjob-sft-$(printf '%05x' $$)-$(date -u +%s)"
 # PVC derived from the run-id so concurrent / repeat runs each get a clean
 # /mnt/models. Torn down in cleanup below.
 PVC_NAME="c6-models-${JOB_NAME#c6-vcjob-sft-}"
-IMAGE="${LF_IMAGE:-build-harbor.alauda.cn/mlops/llamafactory0.9-cu126-amd64:v0.1.0-build.20260603021903}"
+IMAGE="${LF_IMAGE:-docker.io/alaudadockerhub/llamafactory0.9-cu126-amd64:v0.1.0}"
+IMAGE_PULL_SECRET="${LF_IMAGE_PULL_SECRET:-${E2E_IMAGE_PULL_SECRET:-}}"
+RWX_STORAGE_CLASS="${C6_RWX_STORAGE_CLASS:-${E2E_RWX_STORAGE_CLASS:-}}"
+NODE_SELECTOR_KEY="${C6_NODE_SELECTOR_KEY:-${E2E_GPU_NODE_SELECTOR_KEY:-}}"
+NODE_SELECTOR_VALUE="${C6_NODE_SELECTOR_VALUE:-${E2E_GPU_NODE_SELECTOR_VALUE:-}}"
+VOLCANO_QUEUE="${C6_VOLCANO_QUEUE:-${E2E_VOLCANO_QUEUE:-}}"
 
-log "C6: ensuring shared PVC ${PVC_NAME} (RWX cephfs)"
+log "C6: ensuring shared RWX PVC ${PVC_NAME}"
 cat <<YAML | retry_apply gpu_kc
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -24,13 +30,13 @@ metadata:
   name: ${PVC_NAME}
   namespace: ${NS}
 spec:
-  storageClassName: cephfs
+$(yaml_storage_class 2 "${RWX_STORAGE_CLASS}")
   accessModes: ["ReadWriteMany"]
   resources: { requests: { storage: 4Gi } }
 YAML
 
 log "C6: submitting VolcanoJob ${JOB_NAME} (image=${IMAGE})"
-cat <<YAML | retry_create gpu_kc -o jsonpath='{.metadata.name}' >/dev/null
+cat <<YAML | mirror_dockerhub "${GPU_DH_MIRROR}" | retry_create gpu_kc -o jsonpath='{.metadata.name}' >/dev/null
 apiVersion: batch.volcano.sh/v1alpha1
 kind: Job
 metadata:
@@ -41,7 +47,7 @@ spec:
   minAvailable: 1
   schedulerName: volcano
   maxRetry: 0
-  queue: default
+$(yaml_scalar_field 2 queue "${VOLCANO_QUEUE}")
   tasks:
     - name: train
       replicas: 1
@@ -50,11 +56,9 @@ spec:
           labels: { e2e.alauda.io/case: c6 }
         spec:
           restartPolicy: Never
-          # Whole NVIDIA GPU is on 192.168.128.143; that's where the trainer must land.
-          nodeSelector:
-            kubernetes.io/hostname: 192.168.128.143
+$(yaml_node_selector 10 "${NODE_SELECTOR_KEY}" "${NODE_SELECTOR_VALUE}")
           securityContext: { runAsNonRoot: true, runAsUser: 65534, runAsGroup: 65534, fsGroup: 65534 }
-          imagePullSecrets: [{ name: harbor-mlops-regcred }]
+$(yaml_image_pull_secrets 10 "${IMAGE_PULL_SECRET}")
           volumes:
             - name: workspace
               emptyDir: {}
diff --git a/e2e/cases/c7_smoke_npu.sh b/e2e/cases/c7_smoke_npu.sh
index 267b1be..297020a 100755
--- a/e2e/cases/c7_smoke_npu.sh
+++ b/e2e/cases/c7_smoke_npu.sh
@@ -1,32 +1,26 @@
 #!/usr/bin/env bash
 # C7 — same smoke but on the NPU cluster against torch2.6-cann8.5-arm64:v0.1.0.
-# The published runtime YAML requests HAMI vNPU (huawei.com/Ascend910B4 +
-# Ascend910B4-memory + schedulerName: hami-scheduler). The dev NPU cluster from
-# my_dev_env_new.md exposes the standard Huawei k8s-device-plugin
-# (huawei.com/Ascend910, no HAMI), so we patch the runtime in-place before apply.
+# The published runtime YAML requests the documented HAMI vNPU resources.
 set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 source "${HERE}/../lib.sh"
 
+require_env NPU_NAMESPACE "namespace for NPU e2e resources"
 NS="${NPU_NAMESPACE}"
 ASSETS="${E2E_ROOT}/../docs/en/training_guides/assets/training-runtimes"
 
-log "C7: applying torch2.6-cann8.5-arm64 TrainingRuntime to ns/${NS} (non-HAMI patch + docker.io → ${NPU_DH_MIRROR})"
-sed -e "s/namespace: kubeflow-admin-cpaas-io/namespace: ${NS}/" \
-    -e '/schedulerName: hami-scheduler/d' \
-    -e 's@huawei.com/Ascend910B4: "1"@huawei.com/Ascend910: "1"@' \
-    -e '/huawei.com\/Ascend910B4-memory:/d' \
-    "${ASSETS}/torch2.6-cann8.5-arm64-trainingruntime.yaml" \
+if [ -n "${NPU_DH_MIRROR}" ]; then
+  log "C7: applying torch2.6-cann8.5-arm64 TrainingRuntime to ns/${NS} (Docker Hub mirror=${NPU_DH_MIRROR})"
+else
+  log "C7: applying torch2.6-cann8.5-arm64 TrainingRuntime to ns/${NS}"
+fi
+set_metadata_namespace "${NS}" < "${ASSETS}/torch2.6-cann8.5-arm64-trainingruntime.yaml" \
   | mirror_dockerhub "${NPU_DH_MIRROR}" \
   | retry_apply npu_kc
 
 log "C7: submitting TrainJob from trainjob-smoke.yaml (runtimeRef=torch2.6-cann8.5-arm64)"
-# Override CPU request to 200m — kubeos2 runs at ~99% CPU when shared with the dev workloads
-# in this cluster, but has 4 free NPUs. The smoke probe just runs a matmul, 200m is plenty.
-TJ_NAME=$(sed -e "s/namespace: kubeflow-admin-cpaas-io/namespace: ${NS}/" \
-              -e 's/name: torch2.6-cu126-amd64/name: torch2.6-cann8.5-arm64/' \
-              "${ASSETS}/trainjob-smoke.yaml" \
-          | yq '.spec.trainer.resourcesPerNode = {"requests":{"cpu":"50m","memory":"512Mi"},"limits":{"cpu":"500m","memory":"4Gi","huawei.com/Ascend910":"1"}}' \
+TJ_NAME=$(set_metadata_namespace "${NS}" < "${ASSETS}/trainjob-smoke.yaml" \
+          | sed -e 's/name: torch2.6-cu126-amd64/name: torch2.6-cann8.5-arm64/' \
           | retry_create npu_kc -o jsonpath='{.metadata.name}')
 log "C7: trainjob=${TJ_NAME}"
 
diff --git a/e2e/cases/c8_trainer_v2_mindspeed_npu.sh b/e2e/cases/c8_trainer_v2_mindspeed_npu.sh
index a369a21..c4a8c67 100755
--- a/e2e/cases/c8_trainer_v2_mindspeed_npu.sh
+++ b/e2e/cases/c8_trainer_v2_mindspeed_npu.sh
@@ -12,12 +12,19 @@ set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 source "${HERE}/../lib.sh"
 
+require_env NPU_NAMESPACE "namespace for NPU e2e resources"
+require_env NPU_RESOURCE_NAME "extended resource name for one NPU, for example huawei.com/Ascend910B4"
 NS="${NPU_NAMESPACE}"
 RUNTIME="c8-mindspeed-llm-qwen3-npu-runtime"
 IMAGE="${C8_IMAGE:-docker.io/alaudadockerhub/alauda-workbench-jupyter-pytorch-cann-py312-ubi9:v0.1.7}"
+IMAGE_PULL_SECRET="${C8_IMAGE_PULL_SECRET:-${E2E_IMAGE_PULL_SECRET:-}}"
+NPU_RESOURCE_VALUE="${NPU_RESOURCE_VALUE:-1}"
+NPU_MEMORY_RESOURCE_NAME="${NPU_MEMORY_RESOURCE_NAME:-}"
+NPU_MEMORY_RESOURCE_VALUE="${NPU_MEMORY_RESOURCE_VALUE:-8192}"
+NPU_RUNTIME_CLASS="${NPU_RUNTIME_CLASS:-}"
 
 log "C8: applying TrainingRuntime ${RUNTIME} (env-smoke variant, image=${IMAGE})"
-cat <<YAML | retry_apply npu_kc
+cat <<YAML | mirror_dockerhub "${NPU_DH_MIRROR}" | retry_apply npu_kc
 apiVersion: trainer.kubeflow.org/v1alpha1
 kind: TrainingRuntime
 metadata:
@@ -42,10 +49,8 @@ spec:
             backoffLimit: 0
             template:
               spec:
-                # Notebook publishes \`schedulerName: hami-scheduler\` +
-                # \`huawei.com/Ascend910B4\`; the dev NPU cluster uses the default
-                # scheduler and bare \`huawei.com/Ascend910\` (see my_dev_env_new.md).
-                runtimeClassName: ascend
+$(yaml_scalar_field 16 runtimeClassName "${NPU_RUNTIME_CLASS}")
+$(yaml_image_pull_secrets 16 "${IMAGE_PULL_SECRET}")
                 securityContext:
                   runAsNonRoot: true
                   runAsUser: 1001
@@ -98,7 +103,8 @@ spec:
                       limits:
                         cpu: "2"
                         memory: 8Gi
-                        huawei.com/Ascend910: "1"
+$(yaml_resource_limit 24 "${NPU_RESOURCE_NAME}" "${NPU_RESOURCE_VALUE}")
+$(yaml_resource_limit 24 "${NPU_MEMORY_RESOURCE_NAME}" "${NPU_MEMORY_RESOURCE_VALUE}")
                     securityContext:
                       allowPrivilegeEscalation: false
                       capabilities: { drop: [ALL] }
diff --git a/e2e/cases/c9_qwen3_finetune_verify.sh b/e2e/cases/c9_qwen3_finetune_verify.sh
index 5ba9768..35e39f6 100755
--- a/e2e/cases/c9_qwen3_finetune_verify.sh
+++ b/e2e/cases/c9_qwen3_finetune_verify.sh
@@ -14,9 +14,16 @@ set -euo pipefail
 HERE="$(cd "$(dirname "$0")" && pwd)"
 source "${HERE}/../lib.sh"
 
+require_env NPU_NAMESPACE "namespace for NPU e2e resources"
+require_env NPU_RESOURCE_NAME "extended resource name for one NPU, for example huawei.com/Ascend910B4"
 NS="${NPU_NAMESPACE}"
 JOB_NAME="c9-pytorch-cann-smoke-$(printf '%05x' $$)"
 IMAGE="${C9_IMAGE:-docker.io/alaudadockerhub/alauda-workbench-jupyter-pytorch-cann-py312-ubi9:v0.1.7}"
+IMAGE_PULL_SECRET="${C9_IMAGE_PULL_SECRET:-${E2E_IMAGE_PULL_SECRET:-}}"
+NPU_RESOURCE_VALUE="${NPU_RESOURCE_VALUE:-1}"
+NPU_MEMORY_RESOURCE_NAME="${NPU_MEMORY_RESOURCE_NAME:-}"
+NPU_MEMORY_RESOURCE_VALUE="${NPU_MEMORY_RESOURCE_VALUE:-8192}"
+NPU_RUNTIME_CLASS="${NPU_RUNTIME_CLASS:-}"
 
 cleanup() {
   npu_kc -n "${NS}" delete job "${JOB_NAME}" --ignore-not-found --wait=false || true
@@ -24,7 +31,7 @@ cleanup() {
 trap cleanup EXIT
 
 log "C9: submitting Job ${JOB_NAME} (image=${IMAGE})"
-cat <<YAML | retry_create npu_kc >/dev/null
+cat <<YAML | mirror_dockerhub "${NPU_DH_MIRROR}" | retry_create npu_kc >/dev/null
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -39,7 +46,8 @@ spec:
       labels: { e2e.alauda.io/case: c9 }
     spec:
       restartPolicy: Never
-      runtimeClassName: ascend
+$(yaml_scalar_field 6 runtimeClassName "${NPU_RUNTIME_CLASS}")
+$(yaml_image_pull_secrets 6 "${IMAGE_PULL_SECRET}")
       securityContext: { runAsNonRoot: true, runAsUser: 1001, runAsGroup: 0, fsGroup: 1000 }
       containers:
         - name: probe
@@ -55,7 +63,8 @@ spec:
             limits:
               cpu: "2"
               memory: 8Gi
-              huawei.com/Ascend910: "1"
+$(yaml_resource_limit 14 "${NPU_RESOURCE_NAME}" "${NPU_RESOURCE_VALUE}")
+$(yaml_resource_limit 14 "${NPU_MEMORY_RESOURCE_NAME}" "${NPU_MEMORY_RESOURCE_VALUE}")
           command: [bash, -lc]
           args:
             - |
diff --git a/e2e/lib.sh b/e2e/lib.sh
index cbd2ab5..2a4729a 100644
--- a/e2e/lib.sh
+++ b/e2e/lib.sh
@@ -8,28 +8,108 @@ E2E_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 LOG_DIR="${E2E_ROOT}/logs"
 mkdir -p "${LOG_DIR}"
 
-GPU_CONTEXT="${GPU_CONTEXT:-g1-c1-x86-admin@g1-c1-x86}"
-GPU_NAMESPACE="${GPU_NAMESPACE:-mlops-demo-e2e}"
-NPU_KUBECONFIG="${NPU_KUBECONFIG:-${HOME:-/tmp}/.kube/npu-env.yaml}"
-NPU_NAMESPACE="${NPU_NAMESPACE:-mlops-demo-ai-test}"
-# Docker Hub mirror prefixes — both clusters firewall registry-1.docker.io.
-# GPU cluster: docker-mirrors.alauda.cn proxies docker.io.
-# NPU cluster: docker.1ms.run proxies docker.io (per my_dev_env_new.md).
-GPU_DH_MIRROR="${GPU_DH_MIRROR:-docker-mirrors.alauda.cn}"
-NPU_DH_MIRROR="${NPU_DH_MIRROR:-docker.1ms.run}"
+E2E_SKIP_RC="${E2E_SKIP_RC:-77}"
+
+# Required per case: GPU_NAMESPACE or NPU_NAMESPACE.
+# Optional kube target: GPU_CONTEXT/GPU_KUBECONFIG/NPU_CONTEXT/NPU_KUBECONFIG.
+# Optional Docker Hub mirrors: GPU_DH_MIRROR/NPU_DH_MIRROR.
+# Optional private registry access: E2E_IMAGE_PULL_SECRET.
+# Optional scheduling/storage: E2E_GPU_NODE_SELECTOR_KEY/VALUE, E2E_RWX_STORAGE_CLASS.
+GPU_CONTEXT="${GPU_CONTEXT:-}"
+GPU_KUBECONFIG="${GPU_KUBECONFIG:-}"
+GPU_NAMESPACE="${GPU_NAMESPACE:-}"
+NPU_CONTEXT="${NPU_CONTEXT:-}"
+NPU_KUBECONFIG="${NPU_KUBECONFIG:-}"
+NPU_NAMESPACE="${NPU_NAMESPACE:-}"
+GPU_DH_MIRROR="${GPU_DH_MIRROR:-}"
+NPU_DH_MIRROR="${NPU_DH_MIRROR:-}"
 
 # Rewrite docker.io references to a mirror that the cluster can actually reach.
 # Args: mirror_host. Reads stdin, writes patched YAML to stdout.
 mirror_dockerhub() {
   local m="$1"
+  if [ -z "${m}" ]; then
+    cat
+    return 0
+  fi
   sed -e "s@docker.io/@${m}/@g" -e "s@image: alaudadockerhub/@image: ${m}/alaudadockerhub/@g"
 }
 
-gpu_kc() { kubectl --context "${GPU_CONTEXT}" "$@"; }
-npu_kc() { KUBECONFIG="${NPU_KUBECONFIG}" kubectl "$@"; }
+set_metadata_namespace() {
+  local ns="$1"
+  awk -v ns="${ns}" '
+    /^  namespace: / && !done { print "  namespace: " ns; done=1; next }
+    { print }
+  '
+}
 
 log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*"; }
 
+require_env() {
+  local name="$1" hint="${2:-}"
+  if [ -z "${!name:-}" ]; then
+    if [ -n "${hint}" ]; then
+      log "missing required env ${name}: ${hint}"
+    else
+      log "missing required env ${name}"
+    fi
+    exit "${E2E_SKIP_RC}"
+  fi
+}
+
+_kubectl_with_env() {
+  local kubeconfig="$1" context="$2"
+  shift 2
+  if [ -n "${kubeconfig}" ] && [ -n "${context}" ]; then
+    KUBECONFIG="${kubeconfig}" kubectl --context "${context}" "$@"
+  elif [ -n "${kubeconfig}" ]; then
+    KUBECONFIG="${kubeconfig}" kubectl "$@"
+  elif [ -n "${context}" ]; then
+    kubectl --context "${context}" "$@"
+  else
+    kubectl "$@"
+  fi
+}
+
+gpu_kc() { _kubectl_with_env "${GPU_KUBECONFIG}" "${GPU_CONTEXT}" "$@"; }
+npu_kc() { _kubectl_with_env "${NPU_KUBECONFIG}" "${NPU_CONTEXT}" "$@"; }
+
+yaml_scalar_field() {
+  local indent="$1" name="$2" value="${3:-}"
+  [ -z "${value}" ] && return 0
+  printf '%*s%s: %s\n' "${indent}" '' "${name}" "${value}"
+}
+
+yaml_image_pull_secrets() {
+  local indent="$1" secret="${2:-${E2E_IMAGE_PULL_SECRET:-}}"
+  [ -z "${secret}" ] && return 0
+  printf '%*simagePullSecrets:\n' "${indent}" ''
+  printf '%*s- name: %s\n' "$((indent + 2))" '' "${secret}"
+}
+
+yaml_node_selector() {
+  local indent="$1" key="${2:-}" value="${3:-}"
+  [ -z "${key}" ] && return 0
+  if [ -z "${value}" ]; then
+    log "node selector key ${key} requires a value"
+    exit "${E2E_SKIP_RC}"
+  fi
+  printf '%*snodeSelector:\n' "${indent}" ''
+  printf '%*s%s: %s\n' "$((indent + 2))" '' "${key}" "${value}"
+}
+
+yaml_storage_class() {
+  local indent="$1" storage_class="${2:-}"
+  [ -z "${storage_class}" ] && return 0
+  printf '%*sstorageClassName: %s\n' "${indent}" '' "${storage_class}"
+}
+
+yaml_resource_limit() {
+  local indent="$1" name="${2:-}" value="${3:-1}"
+  [ -z "${name}" ] && return 0
+  printf '%*s%s: "%s"\n' "${indent}" '' "${name}" "${value}"
+}
+
 # Reap a background log-follower without blocking. Used after a polling loop
 # that walks a pod to a terminal phase — if the pod never gets there, the
 # `kubectl logs -f` stays alive forever; without this `wait` would inherit
diff --git a/e2e/run_all.sh b/e2e/run_all.sh
index dadb8b4..b4097d8 100755
--- a/e2e/run_all.sh
+++ b/e2e/run_all.sh
@@ -54,7 +54,7 @@ for entry in "${CASES[@]}"; do
     pass=$((pass+1))
   else
     rc=$?
-    if [ "${rc}" -eq 77 ]; then
+    if [ "${rc}" -eq "${E2E_SKIP_RC}" ]; then
       log "    SKIP ${id} ($((SECONDS-start))s) — tail:"
       tail -n 20 "${log_file}" | sed 's/^/        /'
       skip=$((skip+1))

From 6c3a8cb6720af9fab0c32f28b96d2d7c23961e85 Mon Sep 17 00:00:00 2001
From: Wu Yi <typhoonzero1986@gmail.com>
Date: Thu, 11 Jun 2026 03:15:38 +0000
Subject: [PATCH 2/2] =?UTF-8?q?test(e2e):=20validate=20E2E=5FSKIP=5FRC=20a?=
 =?UTF-8?q?s=20an=20integer=200=E2=80=93255?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

E2E_SKIP_RC is overridable via env in this harness, but was used both as a
process exit status (truncated mod 256) and as the RHS of a numeric `-eq`
compare in run_all.sh. A non-numeric or out-of-range override makes the two
diverge, so a skipped case is misreported as FAIL (e.g. 300 -> exit 44, then
`[ 44 -eq 300 ]` is false). Coerce any invalid override back to the 77 default
right after assignment.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 e2e/lib.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/e2e/lib.sh b/e2e/lib.sh
index 2a4729a..16a8d42 100644
--- a/e2e/lib.sh
+++ b/e2e/lib.sh
@@ -9,6 +9,14 @@ LOG_DIR="${E2E_ROOT}/logs"
 mkdir -p "${LOG_DIR}"
 
 E2E_SKIP_RC="${E2E_SKIP_RC:-77}"
+# Must be a valid process exit status (0–255), otherwise `exit "${E2E_SKIP_RC}"`
+# gets truncated mod 256 while run_all.sh still compares against the raw value —
+# skips would then be misreported as FAIL. Reject non-numeric / out-of-range
+# overrides back to the default.
+case "${E2E_SKIP_RC}" in
+  *[!0-9]*) E2E_SKIP_RC=77 ;;
+  *) [ "${E2E_SKIP_RC}" -ge 0 ] && [ "${E2E_SKIP_RC}" -le 255 ] || E2E_SKIP_RC=77 ;;
+esac
 
 # Required per case: GPU_NAMESPACE or NPU_NAMESPACE.
 # Optional kube target: GPU_CONTEXT/GPU_KUBECONFIG/NPU_CONTEXT/NPU_KUBECONFIG.