diff --git a/setup/docker/kind.yaml b/setup/docker/kind.yaml index cc9d71c7..696eaa63 100644 --- a/setup/docker/kind.yaml +++ b/setup/docker/kind.yaml @@ -24,6 +24,7 @@ containerdConfigPatches: nodes: - role: control-plane + image: kindest/node:v1.33.1 kubeadmConfigPatches: - | kind: InitConfiguration diff --git a/setup/docker/opsfile.yml b/setup/docker/opsfile.yml index 4850a1d0..528c7078 100644 --- a/setup/docker/opsfile.yml +++ b/setup/docker/opsfile.yml @@ -17,6 +17,7 @@ version: '3' vars: + CLUSTER: nuvolaris INGRESS: "https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml" env: @@ -24,21 +25,20 @@ env: sh: echo "$OPS_TMP/kind" WORKSPACE: sh: echo "$OPS_HOME/workspace" - KUBECONFIG: - sh: |- + KUBECONFIG: + sh: |- if test -e "$OPS_TMP/kubeconfig" then echo "$OPS_TMP/kubeconfig" else echo ~/.kube/config fi tasks: - status: silent: true desc: status of the Apache OpenServerless cluster in Docker cmds: - | - if kind get clusters | rg nuvolaris + if kind get clusters | rg {{.CLUSTER}} then echo Cluster Apache OpenServerless up and running else echo Cluster Apache OpenServerless not found fi @@ -53,7 +53,7 @@ tasks: echo "=== Kind Version" kind version echo "=== Kind Clusters" - kind get clusters + kind get clusters if test -e $KUBECONFIG then echo "=== Nodes" kubectl get nodes @@ -67,6 +67,7 @@ tasks: - > kind create cluster --kubeconfig="$OPS_TMP/kubeconfig" +<<<<<<< HEAD --wait=1m --name=nuvolaris --config=_kind.yaml - | @@ -80,11 +81,16 @@ tasks: EOF' docker exec nuvolaris-control-plane systemctl restart containerd +======= + --wait=1m --name={{.CLUSTER}} + --config=_kind.yaml + +>>>>>>> 2afc1a6 (offline images) ingress: silent: true cmds: - - > - kubectl --kubeconfig="$OPS_TMP/kubeconfig" + - > + kubectl --kubeconfig="$OPS_TMP/kubeconfig" apply -f ingress-deploy.yaml check-space: @@ -101,11 +107,11 @@ tasks: - config SKIP_PRELOAD_IMAGES cmds: - | - for image in "$IMAGES_STREAMER" "$IMAGES_OPERATOR" "$IMAGES_CONTROLLER" + for image in "$IMAGES_STREAMER" "$IMAGES_OPERATOR" "$IMAGES_CONTROLLER" do docker pull "$image" --platform "linux/{{ARCH}}" - if ! kind load docker-image "$image" --name nuvolaris - then + if ! kind load docker-image "$image" --name {{.CLUSTER}} + then echo "*** Cannot preload images. You can disable preload with ops -config SKIP_PRELOAD_IMAGES=1" echo "*** Workaround on Docker Desktop: disable 'Use containerd for pulling and storing images' on General" echo "*** WARNING! If you skip preload, it can take a lot of time before the system starts. Be patient." @@ -121,7 +127,7 @@ tasks: | rg apache/openserverless-runtime \ | while read -r image; do docker pull "$image" --platform "linux/{{ARCH}}" - kind load docker-image "$image" --name nuvolaris + kind load docker-image "$image" --name "{{.CLUSTER}}" docker rmi "$image" done @@ -133,7 +139,7 @@ tasks: - echo "Cleaning all docker images and containers" - docker ps -qa | while read a ; do docker rm -f "$a" ; done - docker images -qa | while read a ; do docker rmi -f "$a" ; done - - docker system prune -af + - docker system prune -af - docker volume prune -af - docker system df @@ -141,18 +147,20 @@ tasks: silent: true desc: create the Apache OpenServerless cluster in Docker cmds: + - ops util freeze docker-load - task: cluster + - ops util freeze kind-load - task: ingress - - cp "$OPS_TMP/kubeconfig" "$OPS_TMP/kind.kubeconfig" + - cp "$OPS_TMP/kubeconfig" "$OPS_TMP/kind-{{.CLUSTER}}.kubeconfig" status: - - kind get clusters | rg nuvolaris + - kind get clusters | rg {{.CLUSTER}} delete: silent: true desc: destroy the Apache OpenServerless cluster in Docker cmds: - - kind delete clusters nuvolaris - - rm -f "$OPS_TMP/kubeconfig" "$OPS_TMP/kind.kubeconfig" + - kind delete clusters {{.CLUSTER}} + - rm -f "$OPS_TMP/kubeconfig" "$OPS_TMP/kind-{{.CLUSTER}}.kubeconfig" kubeconfig: desc: export kubeconfig diff --git a/setup/kubernetes/operator.yaml b/setup/kubernetes/operator.yaml index c2a4a9fc..121296b5 100644 --- a/setup/kubernetes/operator.yaml +++ b/setup/kubernetes/operator.yaml @@ -19,8 +19,8 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: nuvolaris-operator - namespace: nuvolaris -spec: + namespace: nuvolaris +spec: replicas: 1 selector: matchLabels: @@ -67,18 +67,18 @@ spec: - key: "nuvolaris-role" operator: "Equal" value: core - effect: "NoSchedule" + effect: "NoSchedule" securityContext: runAsUser: 1001 - fsGroup: 1001 + fsGroup: 1001 volumes: - name: deploy persistentVolumeClaim: - claimName: nuvolaris-operator-deploy-pvc + claimName: nuvolaris-operator-deploy-pvc containers: - name: nuvolaris-operator image: ${IMAGES_OPERATOR} - imagePullPolicy: Always + imagePullPolicy: IfNotPresent command: ["./run.sh"] args: ["--verbose"] securityContext: @@ -87,4 +87,4 @@ spec: - ALL volumeMounts: - name: deploy - mountPath: "/home/nuvolaris/deploy" \ No newline at end of file + mountPath: "/home/nuvolaris/deploy" \ No newline at end of file diff --git a/update/opsfile.yml b/update/opsfile.yml index 2dfdde77..24d7165a 100644 --- a/update/opsfile.yml +++ b/update/opsfile.yml @@ -33,7 +33,6 @@ env: fi tasks: - hi: echo hi cli: desc: update the cli diff --git a/util/freeze/opsfile.yml b/util/freeze/opsfile.yml new file mode 100644 index 00000000..d23a31ef --- /dev/null +++ b/util/freeze/opsfile.yml @@ -0,0 +1,175 @@ +version: 3 + +vars: + CLUSTER: nuvolaris + KIND_CLUSTER: '{{.CLUSTER}}-control-plane' + IMAGES: + sh: > + echo ~/.ops/{{OS}}-{{ARCH}}/images + KIND_IMAGE: + sh: > + awk '/image:/{print $2}' $OPS_ROOT/setup/docker/kind.yaml | tee _docker.lst + +tasks: + + docker-list: + silent: true + desc: list docker images + cmds: + - docker inspect {{.KIND_CLUSTER}} 2>/dev/null | jq -r '.[0].Config.Image' + + kind-list: + silent: true + env: + CLU: "{{.KIND_CLUSTER}}" + desc: list images in kindes + cmds: + - awk '/image:/ {print $2}' $OPS_ROOT/setup/docker/ingress-deploy.yaml | tee _kind.lst + - > + docker exec -it "$CLU" crictl images 2>/dev/null + | awk 'NR > 1 && !/^(registry.k8s.io\/(kube-|coredns|etc|pause|ingress-nginx)|docker.io\/kindest\/)/ {print $1 ":" $2 }' + | tee _kind.lst + + docker-save: + silent: true + desc: save images in docker + cmds: + - docker inspect {{.KIND_CLUSTER}} | jq -r '.[0].Config.Image' >_docker.lst + - mkdir -p {{.IMAGES}}/docker + - | + cat _docker.lst | while read img + do + file={{.IMAGES}}/docker/$(base64 -e "$img") + if test -e "$file" + then echo "already saved $img.{{ARCH}} " + else echo ">>> Saving: $img" + if docker pull "$img" + then echo "OK: save $file" + else echo "*** FAIL: pull $img" + fi + if docker save "$img" -o "$file" + then echo "OK: save $file" + else echo "*** FAIL: save $file" + fi + fi + done + + kind-save: + silent: false + desc: list images in kindes + env: + CLU: "{{.KIND_CLUSTER}}" + cmds: + - awk '/image:/ {print $2}' $OPS_ROOT/setup/docker/ingress-deploy.yaml > _kind.lst + - > + docker exec -it "$CLU" crictl images 2>/dev/null + | awk 'NR > 1 && !/^(registry.k8s.io\/(kube-|coredns|etc|pause|ingress-nginx)|docker.io\/kindest\/)/ {print $1 ":" $2 }' + >> _kind.lst + - mkdir -p {{.IMAGES}}/kind + - | + cat _kind.lst | while read img + do + file={{.IMAGES}}/kind/"$(base64 -e "$img")" + if test -e "$file" + then echo "already saved $img" + else echo ">>> Saving: $img" + if ! docker pull "$img" + then echo "*** FAIL pull $img" ; continue + fi + if ! docker save "$img" -o "$file" + then echo "*** FAIL save $file" ; continue + fi + echo "OK: save $file" + fi + done + + image-save: + desc: save images + cmds: + - kind get clusters | grep {{.CLUSTER}} || die "please setup an ops cluster first" + - task: docker-save + - task: kind-save + + image-list: + silent: true + desc: list images + cmds: + - rm -f _images.lst + - | + for dir in docker kind + do for file in {{.IMAGES}}/$dir/* + do if test -e "$file" + then + b64=$(basename "$file") + img=$(base64 -d "${b64}") + echo $dir/$img + echo $img >>_images.lst + fi + done + done + + image-clean: + prompt: "are you sure you wnat to remote images fron disk" + silent: true + desc: clean images + cmds: + - rm -f -v {{.IMAGES}}/docker/* {{.IMAGES}}/kind/* + + docker-load: + silent: true + desc: load disk images in docker + cmds: + - | + for file in {{.IMAGES}}/docker/* + do + b64=$(basename "$file") + img=$(base64 -d "${b64#*-}") + echo Docker: loading "$img" + docker load -i "$file" + #docker tag $(awk '{print $4}' _image) $img + docker tag ${img#*@} ${img%@*} + done + status: + - test -z "$(ls {{.IMAGES}}/docker/* 2>/dev/null)" + + kind-load: + silent: true + desc: load disk images in kind + cmds: + - | + N=0 + for file in {{.IMAGES}}/kind/* + do + N=$(($N+1)) + printf "\rKind: loading images: [$(seq 1 $N | awk '{printf "#"}')]" + #echo $file + b64="$(basename "$file")" + img="$(base64 -d "${b64#*-}")" + kind load image-archive "$file" -n {{.CLUSTER}} --nodes {{.CLUSTER}}-control-plane + done + status: + - test -z "$(ls {{.IMAGES}}/kind/* 2>/dev/null)" + + test-clean: + ignore_errors: true + cmds: + - task: docker-clean + - kind delete cluster --name=test + + test-compare: + - task: docker-list + - task: kind-list + - task: image-list + - cat _docker.lst _kind.lst | sort >_loaded.lst + - sort <_images.lst >_saved.lst + - echo "Comparing differences loaded and saved" + - diff _loaded.lst _saved.lst + + test: + - task: test-clean + - task: docker-load + - kind create cluster --name=test --image={{.KIND_IMAGE}} + - task: kind-load + - task: test-compare + - task: test-clean + diff --git a/util/opsfile.yml b/util/opsfile.yml index 3a0f947f..e938cd5d 100644 --- a/util/opsfile.yml +++ b/util/opsfile.yml @@ -48,7 +48,7 @@ tasks: - echo "{{OS}}-{{ARCH}}" apihost: - desc: current apihost`` + desc: current apihost silent: true cmds: - ops -wsk property get | awk '/whisk API host/{print $4}' @@ -306,3 +306,6 @@ tasks: fi + freeze: + desc: freeze util +