Skip to content

ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters#1868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
stehessel wants to merge 1 commit into
base:master
Choose a base branch
from
Draft

ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters #1868

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/PR.yaml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ jobs:
with:
flavor: gke-default
name: infra-pr-${{ github.event.pull_request.number }}
args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd
args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd,has-infra-server=true
lifespan: ${{ github.actor == 'dependabot[bot]' && '1h' || '24h' }}
wait: true
token: ${{ secrets.INFRA_TOKEN }}
Expand Down
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -175,7 +175,7 @@ push:

.PHONY: argo-workflow-lint
argo-workflow-lint:
@argo lint ./chart/infra-server/static/workflow*.yaml
@argo lint --offline ./chart/infra-server/static/workflow*.yaml ./chart/infra-server/templates/workflowtemplates/*.yaml

.PHONY: shellcheck
shellcheck:
Expand Down
5 changes: 5 additions & 0 deletions chart/infra-server/static/flavors.yaml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -194,6 +194,11 @@
value: false
kind: optional

- name: has-infra-server
description: Whether this cluster runs an infra-server instance. Enables pre-destroy cleanup of managed clusters.
value: false
kind: optional

artifacts:
- name: kubeconfig
description: Kube config for connecting to this cluster
Expand Down
22 changes: 22 additions & 0 deletions chart/infra-server/static/workflow-gke-default.yaml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: gke-default-
namespace: default
spec:
entrypoint: start
onExit: stop
Expand All@@ -20,6 +21,8 @@ spec:
value: ""
- name: set-ssd-storage-default
value: ""
- name: has-infra-server
value: "false"

volumes:
- name: credentials
Expand All@@ -36,6 +39,23 @@ spec:

- name: stop
steps:
# Stops all Argo workflows on a target cluster before it is destroyed,
# giving each workflow's onExit handler time to clean up cloud resources.
# Only relevant for GKE clusters that run the infra server.
- - name: cleanup-infra-clusters
templateRef:
name: cleanup-infra-clusters
template: cleanup
Comment thread
coderabbitai[bot] marked this conversation as resolved.
arguments:
artifacts:
- name: kubeconfig
from: '{{ "{{" }}workflow.outputs.artifacts.global-kubeconfig{{ "}}" }}'
optional: true
when: '{{ "{{" }}workflow.parameters.has-infra-server{{ "}}" }} == true'
continueOn:
failed: true
error: true

- - name: destroy
template: destroy
arguments:
Expand All@@ -48,8 +68,10 @@ spec:
outputs:
artifacts:
- name: kubeconfig
globalName: global-kubeconfig
path: /outputs/kubeconfig
mode: 0644
optional: true
archive:
none: {}
- name: connect
Expand Down
62 changes: 62 additions & 0 deletions chart/infra-server/templates/workflowtemplates/cleanup-infra-clusters.yaml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
---
apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
name: cleanup-infra-clusters
namespace: default
spec:
templates:
# Stops all infra-managed Argo workflows on a target cluster before it
# is destroyed, giving each workflow's onExit handler time to clean up
# cloud resources.
- name: cleanup
inputs:
artifacts:
- name: kubeconfig
path: /tmp/kubeconfig
optional: true
activeDeadlineSeconds: 3600
script:
image: quay.io/argoproj/argocli:latest
command: [bash]
source: |
set -uo pipefail

if [ ! -f /tmp/kubeconfig ]; then
echo "No kubeconfig artifact available. Skipping cleanup."
exit 0
fi

export KUBECONFIG=/tmp/kubeconfig
export ARGO_NAMESPACE=default

get_active_workflows() {
argo list --status Running,Pending -l infra.stackrox.com/cluster-id -o name
}

ACTIVE=$(get_active_workflows)
if [ -z "$ACTIVE" ]; then
echo "No active workflows found. Skipping cleanup."
exit 0
fi

echo "Stopping all active workflows to trigger their destroy phases."
for wf in $ACTIVE; do
echo "Stopping workflow: ${wf}"
argo stop "$wf" || true
done

DEADLINE=$((SECONDS + 3000))
while [ $SECONDS -lt $DEADLINE ]; do
REMAINING=$(get_active_workflows)
if [ -z "$REMAINING" ]; then
echo "All managed workflows completed."
exit 0
fi
COUNT=$(echo "$REMAINING" | wc -l | tr -d ' ')
echo "Waiting for ${COUNT} workflow(s) to complete..."
sleep 30
done

echo "Timed out waiting for managed workflows. Proceeding with cluster destroy."
Comment thread
stehessel marked this conversation as resolved.
exit 1
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters by stehessel · Pull Request #1868 · stackrox/infra · GitHub
Skip to content
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters#1868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Draft

stehessel
wants to merge
1
commit into

base:master
Choose a base branch

from

 Draft


ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters
#1868
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.github/workflows/PR.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ jobs:
 with:
 flavor: gke-default
 name: infra-pr-${{ github.event.pull_request.number }}
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd,has-infra-server=true
 lifespan: ${{ github.actor == 'dependabot[bot]' && '1h' || '24h' }}
 wait: true
 token: ${{ secrets.INFRA_TOKEN }}
Expand Down

2 changes: 1 addition & 1 deletion
Makefile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,7 +175,7 @@ push:

.PHONY: argo-workflow-lint
argo-workflow-lint:
	@argo lint ./chart/infra-server/static/workflow*.yaml
	@argo lint --offline ./chart/infra-server/static/workflow*.yaml ./chart/infra-server/templates/workflowtemplates/*.yaml

.PHONY: shellcheck
shellcheck:
Expand Down

5 changes: 5 additions & 0 deletions
chart/infra-server/static/flavors.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -194,6 +194,11 @@
 value: false
 kind: optional

 - name: has-infra-server
 description: Whether this cluster runs an infra-server instance. Enables pre-destroy cleanup of managed clusters.
 value: false
 kind: optional

 artifacts:
 - name: kubeconfig
 description: Kube config for connecting to this cluster
Expand Down

22 changes: 22 additions & 0 deletions
chart/infra-server/static/workflow-gke-default.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
 generateName: gke-default-
 namespace: default
spec:
 entrypoint: start
 onExit: stop
Expand All@@ -20,6 +21,8 @@ spec:
 value: ""
 - name: set-ssd-storage-default
 value: ""
 - name: has-infra-server
 value: "false"

 volumes:
 - name: credentials
Expand All@@ -36,6 +39,23 @@ spec:

 - name: stop
 steps:
 # Stops all Argo workflows on a target cluster before it is destroyed,
 # giving each workflow's onExit handler time to clean up cloud resources.
 # Only relevant for GKE clusters that run the infra server.
 - - name: cleanup-infra-clusters
 templateRef:
 name: cleanup-infra-clusters
 template: cleanup
Comment thread
coderabbitai[bot] marked this conversation as resolved.

 arguments:
 artifacts:
 - name: kubeconfig
 from: '{{ "{{" }}workflow.outputs.artifacts.global-kubeconfig{{ "}}" }}'
 optional: true
 when: '{{ "{{" }}workflow.parameters.has-infra-server{{ "}}" }} == true'
 continueOn:
 failed: true
 error: true

 - - name: destroy
 template: destroy
 arguments:
Expand All@@ -48,8 +68,10 @@ spec:
 outputs:
 artifacts:
 - name: kubeconfig
 globalName: global-kubeconfig
 path: /outputs/kubeconfig
 mode: 0644
 optional: true
 archive:
 none: {}
 - name: connect
Expand Down

62 changes: 62 additions & 0 deletions
chart/infra-server/templates/workflowtemplates/cleanup-infra-clusters.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
---
apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
 name: cleanup-infra-clusters
 namespace: default
spec:
 templates:
 # Stops all infra-managed Argo workflows on a target cluster before it
 # is destroyed, giving each workflow's onExit handler time to clean up
 # cloud resources.
 - name: cleanup
 inputs:
 artifacts:
 - name: kubeconfig
 path: /tmp/kubeconfig
 optional: true
 activeDeadlineSeconds: 3600
 script:
 image: quay.io/argoproj/argocli:latest
 command: [bash]
 source: |
 set -uo pipefail

 if [ ! -f /tmp/kubeconfig ]; then
 echo "No kubeconfig artifact available. Skipping cleanup."
 exit 0
 fi

 export KUBECONFIG=/tmp/kubeconfig
 export ARGO_NAMESPACE=default

 get_active_workflows() {
 argo list --status Running,Pending -l infra.stackrox.com/cluster-id -o name
 }

 ACTIVE=$(get_active_workflows)
 if [ -z "$ACTIVE" ]; then
 echo "No active workflows found. Skipping cleanup."
 exit 0
 fi

 echo "Stopping all active workflows to trigger their destroy phases."
 for wf in $ACTIVE; do
 echo "Stopping workflow: ${wf}"
 argo stop "$wf" || true
 done

 DEADLINE=$((SECONDS + 3000))
 while [ $SECONDS -lt $DEADLINE ]; do
 REMAINING=$(get_active_workflows)
 if [ -z "$REMAINING" ]; then
 echo "All managed workflows completed."
 exit 0
 fi
 COUNT=$(echo "$REMAINING" | wc -l | tr -d ' ')
 echo "Waiting for ${COUNT} workflow(s) to complete..."
 sleep 30
 done

 echo "Timed out waiting for managed workflows. Proceeding with cluster destroy."
Comment thread
stehessel marked this conversation as resolved.

 exit 1
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters by stehessel · Pull Request #1868 · stackrox/infra · GitHub
Skip to content
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters#1868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Draft

stehessel
wants to merge
1
commit into

base:master
Choose a base branch

from

 Draft


ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters
#1868
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.github/workflows/PR.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ jobs:
 with:
 flavor: gke-default
 name: infra-pr-${{ github.event.pull_request.number }}
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd,has-infra-server=true
 lifespan: ${{ github.actor == 'dependabot[bot]' && '1h' || '24h' }}
 wait: true
 token: ${{ secrets.INFRA_TOKEN }}
Expand Down

2 changes: 1 addition & 1 deletion
Makefile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,7 +175,7 @@ push:

.PHONY: argo-workflow-lint
argo-workflow-lint:
	@argo lint ./chart/infra-server/static/workflow*.yaml
	@argo lint --offline ./chart/infra-server/static/workflow*.yaml ./chart/infra-server/templates/workflowtemplates/*.yaml

.PHONY: shellcheck
shellcheck:
Expand Down

5 changes: 5 additions & 0 deletions
chart/infra-server/static/flavors.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -194,6 +194,11 @@
 value: false
 kind: optional

 - name: has-infra-server
 description: Whether this cluster runs an infra-server instance. Enables pre-destroy cleanup of managed clusters.
 value: false
 kind: optional

 artifacts:
 - name: kubeconfig
 description: Kube config for connecting to this cluster
Expand Down

22 changes: 22 additions & 0 deletions
chart/infra-server/static/workflow-gke-default.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
 generateName: gke-default-
 namespace: default
spec:
 entrypoint: start
 onExit: stop
Expand All@@ -20,6 +21,8 @@ spec:
 value: ""
 - name: set-ssd-storage-default
 value: ""
 - name: has-infra-server
 value: "false"

 volumes:
 - name: credentials
Expand All@@ -36,6 +39,23 @@ spec:

 - name: stop
 steps:
 # Stops all Argo workflows on a target cluster before it is destroyed,
 # giving each workflow's onExit handler time to clean up cloud resources.
 # Only relevant for GKE clusters that run the infra server.
 - - name: cleanup-infra-clusters
 templateRef:
 name: cleanup-infra-clusters
 template: cleanup
Comment thread
coderabbitai[bot] marked this conversation as resolved.

 arguments:
 artifacts:
 - name: kubeconfig
 from: '{{ "{{" }}workflow.outputs.artifacts.global-kubeconfig{{ "}}" }}'
 optional: true
 when: '{{ "{{" }}workflow.parameters.has-infra-server{{ "}}" }} == true'
 continueOn:
 failed: true
 error: true

 - - name: destroy
 template: destroy
 arguments:
Expand All@@ -48,8 +68,10 @@ spec:
 outputs:
 artifacts:
 - name: kubeconfig
 globalName: global-kubeconfig
 path: /outputs/kubeconfig
 mode: 0644
 optional: true
 archive:
 none: {}
 - name: connect
Expand Down

62 changes: 62 additions & 0 deletions
chart/infra-server/templates/workflowtemplates/cleanup-infra-clusters.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
---
apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
 name: cleanup-infra-clusters
 namespace: default
spec:
 templates:
 # Stops all infra-managed Argo workflows on a target cluster before it
 # is destroyed, giving each workflow's onExit handler time to clean up
 # cloud resources.
 - name: cleanup
 inputs:
 artifacts:
 - name: kubeconfig
 path: /tmp/kubeconfig
 optional: true
 activeDeadlineSeconds: 3600
 script:
 image: quay.io/argoproj/argocli:latest
 command: [bash]
 source: |
 set -uo pipefail

 if [ ! -f /tmp/kubeconfig ]; then
 echo "No kubeconfig artifact available. Skipping cleanup."
 exit 0
 fi

 export KUBECONFIG=/tmp/kubeconfig
 export ARGO_NAMESPACE=default

 get_active_workflows() {
 argo list --status Running,Pending -l infra.stackrox.com/cluster-id -o name
 }

 ACTIVE=$(get_active_workflows)
 if [ -z "$ACTIVE" ]; then
 echo "No active workflows found. Skipping cleanup."
 exit 0
 fi

 echo "Stopping all active workflows to trigger their destroy phases."
 for wf in $ACTIVE; do
 echo "Stopping workflow: ${wf}"
 argo stop "$wf" || true
 done

 DEADLINE=$((SECONDS + 3000))
 while [ $SECONDS -lt $DEADLINE ]; do
 REMAINING=$(get_active_workflows)
 if [ -z "$REMAINING" ]; then
 echo "All managed workflows completed."
 exit 0
 fi
 COUNT=$(echo "$REMAINING" | wc -l | tr -d ' ')
 echo "Waiting for ${COUNT} workflow(s) to complete..."
 sleep 30
 done

 echo "Timed out waiting for managed workflows. Proceeding with cluster destroy."
Comment thread
stehessel marked this conversation as resolved.

 exit 1
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters by stehessel · Pull Request #1868 · stackrox/infra · GitHub
Skip to content
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters#1868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Draft

stehessel
wants to merge
1
commit into

base:master
Choose a base branch

from

 Draft


ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters
#1868
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.github/workflows/PR.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ jobs:
 with:
 flavor: gke-default
 name: infra-pr-${{ github.event.pull_request.number }}
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd,has-infra-server=true
 lifespan: ${{ github.actor == 'dependabot[bot]' && '1h' || '24h' }}
 wait: true
 token: ${{ secrets.INFRA_TOKEN }}
Expand Down

2 changes: 1 addition & 1 deletion
Makefile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,7 +175,7 @@ push:

.PHONY: argo-workflow-lint
argo-workflow-lint:
	@argo lint ./chart/infra-server/static/workflow*.yaml
	@argo lint --offline ./chart/infra-server/static/workflow*.yaml ./chart/infra-server/templates/workflowtemplates/*.yaml

.PHONY: shellcheck
shellcheck:
Expand Down

5 changes: 5 additions & 0 deletions
chart/infra-server/static/flavors.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -194,6 +194,11 @@
 value: false
 kind: optional

 - name: has-infra-server
 description: Whether this cluster runs an infra-server instance. Enables pre-destroy cleanup of managed clusters.
 value: false
 kind: optional

 artifacts:
 - name: kubeconfig
 description: Kube config for connecting to this cluster
Expand Down

22 changes: 22 additions & 0 deletions
chart/infra-server/static/workflow-gke-default.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
 generateName: gke-default-
 namespace: default
spec:
 entrypoint: start
 onExit: stop
Expand All@@ -20,6 +21,8 @@ spec:
 value: ""
 - name: set-ssd-storage-default
 value: ""
 - name: has-infra-server
 value: "false"

 volumes:
 - name: credentials
Expand All@@ -36,6 +39,23 @@ spec:

 - name: stop
 steps:
 # Stops all Argo workflows on a target cluster before it is destroyed,
 # giving each workflow's onExit handler time to clean up cloud resources.
 # Only relevant for GKE clusters that run the infra server.
 - - name: cleanup-infra-clusters
 templateRef:
 name: cleanup-infra-clusters
 template: cleanup
Comment thread
coderabbitai[bot] marked this conversation as resolved.

 arguments:
 artifacts:
 - name: kubeconfig
 from: '{{ "{{" }}workflow.outputs.artifacts.global-kubeconfig{{ "}}" }}'
 optional: true
 when: '{{ "{{" }}workflow.parameters.has-infra-server{{ "}}" }} == true'
 continueOn:
 failed: true
 error: true

 - - name: destroy
 template: destroy
 arguments:
Expand All@@ -48,8 +68,10 @@ spec:
 outputs:
 artifacts:
 - name: kubeconfig
 globalName: global-kubeconfig
 path: /outputs/kubeconfig
 mode: 0644
 optional: true
 archive:
 none: {}
 - name: connect
Expand Down

62 changes: 62 additions & 0 deletions
chart/infra-server/templates/workflowtemplates/cleanup-infra-clusters.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
---
apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
 name: cleanup-infra-clusters
 namespace: default
spec:
 templates:
 # Stops all infra-managed Argo workflows on a target cluster before it
 # is destroyed, giving each workflow's onExit handler time to clean up
 # cloud resources.
 - name: cleanup
 inputs:
 artifacts:
 - name: kubeconfig
 path: /tmp/kubeconfig
 optional: true
 activeDeadlineSeconds: 3600
 script:
 image: quay.io/argoproj/argocli:latest
 command: [bash]
 source: |
 set -uo pipefail

 if [ ! -f /tmp/kubeconfig ]; then
 echo "No kubeconfig artifact available. Skipping cleanup."
 exit 0
 fi

 export KUBECONFIG=/tmp/kubeconfig
 export ARGO_NAMESPACE=default

 get_active_workflows() {
 argo list --status Running,Pending -l infra.stackrox.com/cluster-id -o name
 }

 ACTIVE=$(get_active_workflows)
 if [ -z "$ACTIVE" ]; then
 echo "No active workflows found. Skipping cleanup."
 exit 0
 fi

 echo "Stopping all active workflows to trigger their destroy phases."
 for wf in $ACTIVE; do
 echo "Stopping workflow: ${wf}"
 argo stop "$wf" || true
 done

 DEADLINE=$((SECONDS + 3000))
 while [ $SECONDS -lt $DEADLINE ]; do
 REMAINING=$(get_active_workflows)
 if [ -z "$REMAINING" ]; then
 echo "All managed workflows completed."
 exit 0
 fi
 COUNT=$(echo "$REMAINING" | wc -l | tr -d ' ')
 echo "Waiting for ${COUNT} workflow(s) to complete..."
 sleep 30
 done

 echo "Timed out waiting for managed workflows. Proceeding with cluster destroy."
Comment thread
stehessel marked this conversation as resolved.

 exit 1
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters by stehessel · Pull Request #1868 · stackrox/infra · GitHub
Skip to content
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters#1868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Draft

stehessel
wants to merge
1
commit into

base:master
Choose a base branch

from

 Draft


ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters
#1868
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.github/workflows/PR.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ jobs:
 with:
 flavor: gke-default
 name: infra-pr-${{ github.event.pull_request.number }}
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd,has-infra-server=true
 lifespan: ${{ github.actor == 'dependabot[bot]' && '1h' || '24h' }}
 wait: true
 token: ${{ secrets.INFRA_TOKEN }}
Expand Down

2 changes: 1 addition & 1 deletion
Makefile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,7 +175,7 @@ push:

.PHONY: argo-workflow-lint
argo-workflow-lint:
	@argo lint ./chart/infra-server/static/workflow*.yaml
	@argo lint --offline ./chart/infra-server/static/workflow*.yaml ./chart/infra-server/templates/workflowtemplates/*.yaml

.PHONY: shellcheck
shellcheck:
Expand Down

5 changes: 5 additions & 0 deletions
chart/infra-server/static/flavors.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -194,6 +194,11 @@
 value: false
 kind: optional

 - name: has-infra-server
 description: Whether this cluster runs an infra-server instance. Enables pre-destroy cleanup of managed clusters.
 value: false
 kind: optional

 artifacts:
 - name: kubeconfig
 description: Kube config for connecting to this cluster
Expand Down

22 changes: 22 additions & 0 deletions
chart/infra-server/static/workflow-gke-default.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
 generateName: gke-default-
 namespace: default
spec:
 entrypoint: start
 onExit: stop
Expand All@@ -20,6 +21,8 @@ spec:
 value: ""
 - name: set-ssd-storage-default
 value: ""
 - name: has-infra-server
 value: "false"

 volumes:
 - name: credentials
Expand All@@ -36,6 +39,23 @@ spec:

 - name: stop
 steps:
 # Stops all Argo workflows on a target cluster before it is destroyed,
 # giving each workflow's onExit handler time to clean up cloud resources.
 # Only relevant for GKE clusters that run the infra server.
 - - name: cleanup-infra-clusters
 templateRef:
 name: cleanup-infra-clusters
 template: cleanup
Comment thread
coderabbitai[bot] marked this conversation as resolved.

 arguments:
 artifacts:
 - name: kubeconfig
 from: '{{ "{{" }}workflow.outputs.artifacts.global-kubeconfig{{ "}}" }}'
 optional: true
 when: '{{ "{{" }}workflow.parameters.has-infra-server{{ "}}" }} == true'
 continueOn:
 failed: true
 error: true

 - - name: destroy
 template: destroy
 arguments:
Expand All@@ -48,8 +68,10 @@ spec:
 outputs:
 artifacts:
 - name: kubeconfig
 globalName: global-kubeconfig
 path: /outputs/kubeconfig
 mode: 0644
 optional: true
 archive:
 none: {}
 - name: connect
Expand Down

62 changes: 62 additions & 0 deletions
chart/infra-server/templates/workflowtemplates/cleanup-infra-clusters.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
---
apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
 name: cleanup-infra-clusters
 namespace: default
spec:
 templates:
 # Stops all infra-managed Argo workflows on a target cluster before it
 # is destroyed, giving each workflow's onExit handler time to clean up
 # cloud resources.
 - name: cleanup
 inputs:
 artifacts:
 - name: kubeconfig
 path: /tmp/kubeconfig
 optional: true
 activeDeadlineSeconds: 3600
 script:
 image: quay.io/argoproj/argocli:latest
 command: [bash]
 source: |
 set -uo pipefail

 if [ ! -f /tmp/kubeconfig ]; then
 echo "No kubeconfig artifact available. Skipping cleanup."
 exit 0
 fi

 export KUBECONFIG=/tmp/kubeconfig
 export ARGO_NAMESPACE=default

 get_active_workflows() {
 argo list --status Running,Pending -l infra.stackrox.com/cluster-id -o name
 }

 ACTIVE=$(get_active_workflows)
 if [ -z "$ACTIVE" ]; then
 echo "No active workflows found. Skipping cleanup."
 exit 0
 fi

 echo "Stopping all active workflows to trigger their destroy phases."
 for wf in $ACTIVE; do
 echo "Stopping workflow: ${wf}"
 argo stop "$wf" || true
 done

 DEADLINE=$((SECONDS + 3000))
 while [ $SECONDS -lt $DEADLINE ]; do
 REMAINING=$(get_active_workflows)
 if [ -z "$REMAINING" ]; then
 echo "All managed workflows completed."
 exit 0
 fi
 COUNT=$(echo "$REMAINING" | wc -l | tr -d ' ')
 echo "Waiting for ${COUNT} workflow(s) to complete..."
 sleep 30
 done

 echo "Timed out waiting for managed workflows. Proceeding with cluster destroy."
Comment thread
stehessel marked this conversation as resolved.

 exit 1
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters by stehessel · Pull Request #1868 · stackrox/infra · GitHub
Skip to content
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters#1868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Draft

stehessel
wants to merge
1
commit into

base:master
Choose a base branch

from

 Draft


ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters
#1868
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.github/workflows/PR.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ jobs:
 with:
 flavor: gke-default
 name: infra-pr-${{ github.event.pull_request.number }}
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd,has-infra-server=true
 lifespan: ${{ github.actor == 'dependabot[bot]' && '1h' || '24h' }}
 wait: true
 token: ${{ secrets.INFRA_TOKEN }}
Expand Down

2 changes: 1 addition & 1 deletion
Makefile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,7 +175,7 @@ push:

.PHONY: argo-workflow-lint
argo-workflow-lint:
	@argo lint ./chart/infra-server/static/workflow*.yaml
	@argo lint --offline ./chart/infra-server/static/workflow*.yaml ./chart/infra-server/templates/workflowtemplates/*.yaml

.PHONY: shellcheck
shellcheck:
Expand Down

5 changes: 5 additions & 0 deletions
chart/infra-server/static/flavors.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -194,6 +194,11 @@
 value: false
 kind: optional

 - name: has-infra-server
 description: Whether this cluster runs an infra-server instance. Enables pre-destroy cleanup of managed clusters.
 value: false
 kind: optional

 artifacts:
 - name: kubeconfig
 description: Kube config for connecting to this cluster
Expand Down

22 changes: 22 additions & 0 deletions
chart/infra-server/static/workflow-gke-default.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
 generateName: gke-default-
 namespace: default
spec:
 entrypoint: start
 onExit: stop
Expand All@@ -20,6 +21,8 @@ spec:
 value: ""
 - name: set-ssd-storage-default
 value: ""
 - name: has-infra-server
 value: "false"

 volumes:
 - name: credentials
Expand All@@ -36,6 +39,23 @@ spec:

 - name: stop
 steps:
 # Stops all Argo workflows on a target cluster before it is destroyed,
 # giving each workflow's onExit handler time to clean up cloud resources.
 # Only relevant for GKE clusters that run the infra server.
 - - name: cleanup-infra-clusters
 templateRef:
 name: cleanup-infra-clusters
 template: cleanup
Comment thread
coderabbitai[bot] marked this conversation as resolved.

 arguments:
 artifacts:
 - name: kubeconfig
 from: '{{ "{{" }}workflow.outputs.artifacts.global-kubeconfig{{ "}}" }}'
 optional: true
 when: '{{ "{{" }}workflow.parameters.has-infra-server{{ "}}" }} == true'
 continueOn:
 failed: true
 error: true

 - - name: destroy
 template: destroy
 arguments:
Expand All@@ -48,8 +68,10 @@ spec:
 outputs:
 artifacts:
 - name: kubeconfig
 globalName: global-kubeconfig
 path: /outputs/kubeconfig
 mode: 0644
 optional: true
 archive:
 none: {}
 - name: connect
Expand Down

62 changes: 62 additions & 0 deletions
chart/infra-server/templates/workflowtemplates/cleanup-infra-clusters.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
---
apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
 name: cleanup-infra-clusters
 namespace: default
spec:
 templates:
 # Stops all infra-managed Argo workflows on a target cluster before it
 # is destroyed, giving each workflow's onExit handler time to clean up
 # cloud resources.
 - name: cleanup
 inputs:
 artifacts:
 - name: kubeconfig
 path: /tmp/kubeconfig
 optional: true
 activeDeadlineSeconds: 3600
 script:
 image: quay.io/argoproj/argocli:latest
 command: [bash]
 source: |
 set -uo pipefail

 if [ ! -f /tmp/kubeconfig ]; then
 echo "No kubeconfig artifact available. Skipping cleanup."
 exit 0
 fi

 export KUBECONFIG=/tmp/kubeconfig
 export ARGO_NAMESPACE=default

 get_active_workflows() {
 argo list --status Running,Pending -l infra.stackrox.com/cluster-id -o name
 }

 ACTIVE=$(get_active_workflows)
 if [ -z "$ACTIVE" ]; then
 echo "No active workflows found. Skipping cleanup."
 exit 0
 fi

 echo "Stopping all active workflows to trigger their destroy phases."
 for wf in $ACTIVE; do
 echo "Stopping workflow: ${wf}"
 argo stop "$wf" || true
 done

 DEADLINE=$((SECONDS + 3000))
 while [ $SECONDS -lt $DEADLINE ]; do
 REMAINING=$(get_active_workflows)
 if [ -z "$REMAINING" ]; then
 echo "All managed workflows completed."
 exit 0
 fi
 COUNT=$(echo "$REMAINING" | wc -l | tr -d ' ')
 echo "Waiting for ${COUNT} workflow(s) to complete..."
 sleep 30
 done

 echo "Timed out waiting for managed workflows. Proceeding with cluster destroy."
Comment thread
stehessel marked this conversation as resolved.

 exit 1
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters by stehessel · Pull Request #1868 · stackrox/infra · GitHub
Skip to content
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters#1868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Draft

stehessel
wants to merge
1
commit into

base:master
Choose a base branch

from

 Draft


ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters
#1868
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.github/workflows/PR.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ jobs:
 with:
 flavor: gke-default
 name: infra-pr-${{ github.event.pull_request.number }}
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd,has-infra-server=true
 lifespan: ${{ github.actor == 'dependabot[bot]' && '1h' || '24h' }}
 wait: true
 token: ${{ secrets.INFRA_TOKEN }}
Expand Down

2 changes: 1 addition & 1 deletion
Makefile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,7 +175,7 @@ push:

.PHONY: argo-workflow-lint
argo-workflow-lint:
	@argo lint ./chart/infra-server/static/workflow*.yaml
	@argo lint --offline ./chart/infra-server/static/workflow*.yaml ./chart/infra-server/templates/workflowtemplates/*.yaml

.PHONY: shellcheck
shellcheck:
Expand Down

5 changes: 5 additions & 0 deletions
chart/infra-server/static/flavors.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -194,6 +194,11 @@
 value: false
 kind: optional

 - name: has-infra-server
 description: Whether this cluster runs an infra-server instance. Enables pre-destroy cleanup of managed clusters.
 value: false
 kind: optional

 artifacts:
 - name: kubeconfig
 description: Kube config for connecting to this cluster
Expand Down

22 changes: 22 additions & 0 deletions
chart/infra-server/static/workflow-gke-default.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
 generateName: gke-default-
 namespace: default
spec:
 entrypoint: start
 onExit: stop
Expand All@@ -20,6 +21,8 @@ spec:
 value: ""
 - name: set-ssd-storage-default
 value: ""
 - name: has-infra-server
 value: "false"

 volumes:
 - name: credentials
Expand All@@ -36,6 +39,23 @@ spec:

 - name: stop
 steps:
 # Stops all Argo workflows on a target cluster before it is destroyed,
 # giving each workflow's onExit handler time to clean up cloud resources.
 # Only relevant for GKE clusters that run the infra server.
 - - name: cleanup-infra-clusters
 templateRef:
 name: cleanup-infra-clusters
 template: cleanup
Comment thread
coderabbitai[bot] marked this conversation as resolved.

 arguments:
 artifacts:
 - name: kubeconfig
 from: '{{ "{{" }}workflow.outputs.artifacts.global-kubeconfig{{ "}}" }}'
 optional: true
 when: '{{ "{{" }}workflow.parameters.has-infra-server{{ "}}" }} == true'
 continueOn:
 failed: true
 error: true

 - - name: destroy
 template: destroy
 arguments:
Expand All@@ -48,8 +68,10 @@ spec:
 outputs:
 artifacts:
 - name: kubeconfig
 globalName: global-kubeconfig
 path: /outputs/kubeconfig
 mode: 0644
 optional: true
 archive:
 none: {}
 - name: connect
Expand Down

62 changes: 62 additions & 0 deletions
chart/infra-server/templates/workflowtemplates/cleanup-infra-clusters.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
---
apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
 name: cleanup-infra-clusters
 namespace: default
spec:
 templates:
 # Stops all infra-managed Argo workflows on a target cluster before it
 # is destroyed, giving each workflow's onExit handler time to clean up
 # cloud resources.
 - name: cleanup
 inputs:
 artifacts:
 - name: kubeconfig
 path: /tmp/kubeconfig
 optional: true
 activeDeadlineSeconds: 3600
 script:
 image: quay.io/argoproj/argocli:latest
 command: [bash]
 source: |
 set -uo pipefail

 if [ ! -f /tmp/kubeconfig ]; then
 echo "No kubeconfig artifact available. Skipping cleanup."
 exit 0
 fi

 export KUBECONFIG=/tmp/kubeconfig
 export ARGO_NAMESPACE=default

 get_active_workflows() {
 argo list --status Running,Pending -l infra.stackrox.com/cluster-id -o name
 }

 ACTIVE=$(get_active_workflows)
 if [ -z "$ACTIVE" ]; then
 echo "No active workflows found. Skipping cleanup."
 exit 0
 fi

 echo "Stopping all active workflows to trigger their destroy phases."
 for wf in $ACTIVE; do
 echo "Stopping workflow: ${wf}"
 argo stop "$wf" || true
 done

 DEADLINE=$((SECONDS + 3000))
 while [ $SECONDS -lt $DEADLINE ]; do
 REMAINING=$(get_active_workflows)
 if [ -z "$REMAINING" ]; then
 echo "All managed workflows completed."
 exit 0
 fi
 COUNT=$(echo "$REMAINING" | wc -l | tr -d ' ')
 echo "Waiting for ${COUNT} workflow(s) to complete..."
 sleep 30
 done

 echo "Timed out waiting for managed workflows. Proceeding with cluster destroy."
Comment thread
stehessel marked this conversation as resolved.

 exit 1
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters by stehessel · Pull Request #1868 · stackrox/infra · GitHub
Skip to content
ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters#1868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Draft

stehessel
wants to merge
1
commit into

base:master
Choose a base branch

from

 Draft


ROX-35269: clean up managed clusters before destroying infra-pr GKE clusters
#1868
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
.github/workflows/PR.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@ jobs:
 with:
 flavor: gke-default
 name: infra-pr-${{ github.event.pull_request.number }}
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd
 args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd,has-infra-server=true
 lifespan: ${{ github.actor == 'dependabot[bot]' && '1h' || '24h' }}
 wait: true
 token: ${{ secrets.INFRA_TOKEN }}
Expand Down

2 changes: 1 addition & 1 deletion
Makefile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,7 +175,7 @@ push:

.PHONY: argo-workflow-lint
argo-workflow-lint:
	@argo lint ./chart/infra-server/static/workflow*.yaml
	@argo lint --offline ./chart/infra-server/static/workflow*.yaml ./chart/infra-server/templates/workflowtemplates/*.yaml

.PHONY: shellcheck
shellcheck:
Expand Down

5 changes: 5 additions & 0 deletions
chart/infra-server/static/flavors.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -194,6 +194,11 @@
 value: false
 kind: optional

 - name: has-infra-server
 description: Whether this cluster runs an infra-server instance. Enables pre-destroy cleanup of managed clusters.
 value: false
 kind: optional

 artifacts:
 - name: kubeconfig
 description: Kube config for connecting to this cluster
Expand Down

22 changes: 22 additions & 0 deletions
chart/infra-server/static/workflow-gke-default.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
 generateName: gke-default-
 namespace: default
spec:
 entrypoint: start
 onExit: stop
Expand All@@ -20,6 +21,8 @@ spec:
 value: ""
 - name: set-ssd-storage-default
 value: ""
 - name: has-infra-server
 value: "false"

 volumes:
 - name: credentials
Expand All@@ -36,6 +39,23 @@ spec:

 - name: stop
 steps:
 # Stops all Argo workflows on a target cluster before it is destroyed,
 # giving each workflow's onExit handler time to clean up cloud resources.
 # Only relevant for GKE clusters that run the infra server.
 - - name: cleanup-infra-clusters
 templateRef:
 name: cleanup-infra-clusters
 template: cleanup
Comment thread
coderabbitai[bot] marked this conversation as resolved.

 arguments:
 artifacts:
 - name: kubeconfig
 from: '{{ "{{" }}workflow.outputs.artifacts.global-kubeconfig{{ "}}" }}'
 optional: true
 when: '{{ "{{" }}workflow.parameters.has-infra-server{{ "}}" }} == true'
 continueOn:
 failed: true
 error: true

 - - name: destroy
 template: destroy
 arguments:
Expand All@@ -48,8 +68,10 @@ spec:
 outputs:
 artifacts:
 - name: kubeconfig
 globalName: global-kubeconfig
 path: /outputs/kubeconfig
 mode: 0644
 optional: true
 archive:
 none: {}
 - name: connect
Expand Down

62 changes: 62 additions & 0 deletions
chart/infra-server/templates/workflowtemplates/cleanup-infra-clusters.yaml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
---
apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
 name: cleanup-infra-clusters
 namespace: default
spec:
 templates:
 # Stops all infra-managed Argo workflows on a target cluster before it
 # is destroyed, giving each workflow's onExit handler time to clean up
 # cloud resources.
 - name: cleanup
 inputs:
 artifacts:
 - name: kubeconfig
 path: /tmp/kubeconfig
 optional: true
 activeDeadlineSeconds: 3600
 script:
 image: quay.io/argoproj/argocli:latest
 command: [bash]
 source: |
 set -uo pipefail

 if [ ! -f /tmp/kubeconfig ]; then
 echo "No kubeconfig artifact available. Skipping cleanup."
 exit 0
 fi

 export KUBECONFIG=/tmp/kubeconfig
 export ARGO_NAMESPACE=default

 get_active_workflows() {
 argo list --status Running,Pending -l infra.stackrox.com/cluster-id -o name
 }

 ACTIVE=$(get_active_workflows)
 if [ -z "$ACTIVE" ]; then
 echo "No active workflows found. Skipping cleanup."
 exit 0
 fi

 echo "Stopping all active workflows to trigger their destroy phases."
 for wf in $ACTIVE; do
 echo "Stopping workflow: ${wf}"
 argo stop "$wf" || true
 done

 DEADLINE=$((SECONDS + 3000))
 while [ $SECONDS -lt $DEADLINE ]; do
 REMAINING=$(get_active_workflows)
 if [ -z "$REMAINING" ]; then
 echo "All managed workflows completed."
 exit 0
 fi
 COUNT=$(echo "$REMAINING" | wc -l | tr -d ' ')
 echo "Waiting for ${COUNT} workflow(s) to complete..."
 sleep 30
 done

 echo "Timed out waiting for managed workflows. Proceeding with cluster destroy."
Comment thread
stehessel marked this conversation as resolved.

 exit 1
Loading