From 198e87ce1d43d952c37d7baa38a85b5132898ca1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Nov 2025 13:36:47 +0000 Subject: [PATCH 1/8] Add AIOps workflows and documentation This commit adds four new GitHub Actions workflows for AI operations quality assurance: - aiops-link-check.yml: Validates links in documentation and AI files - aiops-index-drift.yml: Ensures index files reference all their leaves - aiops-frontmatter.yml: Validates YAML front matter in prompts, chatmodes, and instructions - aiops-secrets-scan.yml: Scans for secrets and PII using Gitleaks Also includes: - New model-migration-playbook.md documenting LLM migration process - Updated AGENTS.md with test status table for tracking agent test coverage - Updated .coderabbit.yml with AI asset path instructions for better code review These workflows support the AIOps initiative by automating quality checks for AI-related assets across the organisation. --- .coderabbit.yml | 11 ++++++++ .github/workflows/aiops-frontmatter.yml | 29 +++++++++++++++++++++ .github/workflows/aiops-index-drift.yml | 33 ++++++++++++++++++++++++ .github/workflows/aiops-link-check.yml | 20 ++++++++++++++ .github/workflows/aiops-secrets-scan.yml | 17 ++++++++++++ AGENTS.md | 8 ++++++ docs/model-migration-playbook.md | 30 +++++++++++++++++++++ 7 files changed, 148 insertions(+) create mode 100644 .github/workflows/aiops-frontmatter.yml create mode 100644 .github/workflows/aiops-index-drift.yml create mode 100644 .github/workflows/aiops-link-check.yml create mode 100644 .github/workflows/aiops-secrets-scan.yml create mode 100644 docs/model-migration-playbook.md diff --git a/.coderabbit.yml b/.coderabbit.yml index 3cbaeb0ce0..828c3968f4 100644 --- a/.coderabbit.yml +++ b/.coderabbit.yml @@ -28,6 +28,17 @@ reviews: # === Path-Specific Review Instructions === path_instructions: + # 0. AI Assets (General) + - path: ".github/prompts/**" + instructions: | + Prefer concise, actionable reviews. Respect documented style precedence. Link suggested fixes. + - path: ".github/chatmodes/**" + instructions: | + Prefer concise, actionable reviews. Respect documented style precedence. Link suggested fixes. + - path: ".github/agents/**" + instructions: | + Prefer concise, actionable reviews. Respect documented style precedence. Link suggested fixes. + # 1. Copilot & AI Files - path: ".github/custom-instructions.md" instructions: | diff --git a/.github/workflows/aiops-frontmatter.yml b/.github/workflows/aiops-frontmatter.yml new file mode 100644 index 0000000000..197caa3f40 --- /dev/null +++ b/.github/workflows/aiops-frontmatter.yml @@ -0,0 +1,29 @@ +name: AIOps — Front Matter + +on: + pull_request: + paths: + - ".github/prompts/**" + - ".github/chatmodes/**" + - ".github/instructions/**" + +jobs: + frontmatter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Validate front matter presence and minimal schema + shell: bash + run: | + set -euo pipefail + fail=0 + while IFS= read -r -d '' file; do + # Must start with '---' + head -1 "$file" | grep -q "^---" || { echo "::error file=$file::missing YAML front matter"; fail=1; continue; } + # Extract front matter block + fm=$(awk '/^---/{f=!f;next} f' "$file") + echo "$fm" | grep -q "^title:" || { echo "::error file=$file::missing title"; fail=1; } + echo "$fm" | grep -q "^description:" || { echo "::error file=$file::missing description"; fail=1; } + echo "$fm" | grep -q "^applyTo:" || { echo "::error file=$file::missing applyTo"; fail=1; } + done < <(find .github/prompts .github/chatmodes .github/instructions -type f -name "*.md" -print0) + exit $fail diff --git a/.github/workflows/aiops-index-drift.yml b/.github/workflows/aiops-index-drift.yml new file mode 100644 index 0000000000..1958a4d180 --- /dev/null +++ b/.github/workflows/aiops-index-drift.yml @@ -0,0 +1,33 @@ +name: AIOps — Index Drift + +on: + pull_request: + paths: + - ".github/prompts/**" + - ".github/chatmodes/**" + - ".github/collections/**" + - ".github/instructions/**" + +jobs: + drift: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check plural indexes include all leaves + shell: bash + run: | + set -euo pipefail + fail=0 + check_dir() { + local dir="$1"; local index="$2"; local pattern="$3" + leaves=$(find "$dir" -maxdepth 1 -type f -name "$pattern" -printf "%f\n" | sort) + [[ -f "$index" ]] || { echo "::error file=$index::missing index file"; fail=1; return; } + for f in $leaves; do + grep -q "$f" "$index" || { echo "::error file=$index::missing entry for $f"; fail=1; } + done + } + check_dir ".github/prompts" ".github/prompts/prompts.md" "*.prompt.md" + check_dir ".github/chatmodes" ".github/chatmodes/chatmodes.md" "*.chatmode.md" + check_dir ".github/collections" ".github/collections/collections.md" "*.md" + check_dir ".github/instructions" ".github/instructions/instructions.md" "*.md" + exit $fail diff --git a/.github/workflows/aiops-link-check.yml b/.github/workflows/aiops-link-check.yml new file mode 100644 index 0000000000..7b323d78f9 --- /dev/null +++ b/.github/workflows/aiops-link-check.yml @@ -0,0 +1,20 @@ +name: AIOps — Link Check + +on: + pull_request: + paths: + - ".github/**" + - "AGENTS.md" + - "CLAUDE.md" + - "GEMINI.md" + - "docs/**" + +jobs: + lychee: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run lychee link checker + uses: lycheeverse/lychee-action@v1 + with: + args: --no-progress --exclude-mail --max-concurrency 8 --retry-wait-time 2 --accept 200,206 . diff --git a/.github/workflows/aiops-secrets-scan.yml b/.github/workflows/aiops-secrets-scan.yml new file mode 100644 index 0000000000..bd57d51519 --- /dev/null +++ b/.github/workflows/aiops-secrets-scan.yml @@ -0,0 +1,17 @@ +name: AIOps — Secrets/PII Scan + +on: + pull_request: + paths: + - ".github/**" + - "docs/**" + +jobs: + gitleaks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Gitleaks scan + uses: gitleaks/gitleaks-action@v2 + with: + args: detect -v --source . --report-format sarif --report-path gitleaks.sarif diff --git a/AGENTS.md b/AGENTS.md index 15fadcea43..fe6bbc0e3d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,6 +15,14 @@ - Each agent must have both a code file (`.js`, `.py`, etc.) and a spec (`.md`) following the template. - All contributors must follow the org [Coding Standards](.github/instructions/coding-standards.instructions.md). +## Agent Test Status + +| Agent | Tests | Notes | +|-------|-------|-------| +| _TBD_ | ⏳ | Awaiting test implementation | + +> **Note:** As agents are developed and tested, this table will be updated with their status. ✅ indicates passing tests, ❌ indicates failing tests, and ⏳ indicates tests pending implementation. + ## Global Principles & Agent Rules | Principle / Rule | Guidance / Details | diff --git a/docs/model-migration-playbook.md b/docs/model-migration-playbook.md new file mode 100644 index 0000000000..57d50b1518 --- /dev/null +++ b/docs/model-migration-playbook.md @@ -0,0 +1,30 @@ +--- +title: 'Model Migration Playbook' +description: 'Concise process for changing/adding LLMs across prompts, chatmodes, and agents' +version: '1.0' +last_updated: '2025-11-12' +author: 'LightSpeed' +tags: ['aiops', 'migration', 'llm', 'agents', 'process'] +type: 'playbook' +--- + +# Model Migration Playbook + +## Purpose +Concise process for changing/adding LLMs (e.g., Gemini → Claude → ChatGPT) across prompts/chatmodes/agents. + +## Steps +1) Propose change and scope impact. +2) Update affected prompts/chatmodes with test cases. +3) Run agent tests; compare outputs against baselines. +4) Update docs (`AGENTS.md`, model guides); seek sign‑off. +5) Rollout on `develop`; monitor; then promote to `main`. + +## Sign‑off +- Docs, Agents owners. + +## Related Resources +- [AGENTS.md](../AGENTS.md) +- [CLAUDE.md](../CLAUDE.md) +- [AI Model Comparison Guide](../ai-model-comparison.md) +- [Model Selection Guide](../model-guide.md) From f17f4f1f691e3767aa1317ff8832ae7d6fe928e0 Mon Sep 17 00:00:00 2001 From: Ash Shaw Date: Wed, 12 Nov 2025 21:13:46 +0700 Subject: [PATCH 2/8] Update docs/model-migration-playbook.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Ash Shaw --- docs/model-migration-playbook.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/model-migration-playbook.md b/docs/model-migration-playbook.md index 57d50b1518..407b02a254 100644 --- a/docs/model-migration-playbook.md +++ b/docs/model-migration-playbook.md @@ -21,7 +21,8 @@ Concise process for changing/adding LLMs (e.g., Gemini → Claude → ChatGPT) a 5) Rollout on `develop`; monitor; then promote to `main`. ## Sign‑off -- Docs, Agents owners. +- @lightspeedwp/docs-team (Docs team) +- @lightspeedwp/agents-team (Agents team) ## Related Resources - [AGENTS.md](../AGENTS.md) From 1e9cb4a49c3fdc8b7449e38fbdd1006ecaa7a7a6 Mon Sep 17 00:00:00 2001 From: Ash Shaw Date: Wed, 12 Nov 2025 21:13:59 +0700 Subject: [PATCH 3/8] Update .github/workflows/aiops-frontmatter.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Ash Shaw --- .github/workflows/aiops-frontmatter.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/aiops-frontmatter.yml b/.github/workflows/aiops-frontmatter.yml index 197caa3f40..f8e469d357 100644 --- a/.github/workflows/aiops-frontmatter.yml +++ b/.github/workflows/aiops-frontmatter.yml @@ -24,6 +24,9 @@ jobs: fm=$(awk '/^---/{f=!f;next} f' "$file") echo "$fm" | grep -q "^title:" || { echo "::error file=$file::missing title"; fail=1; } echo "$fm" | grep -q "^description:" || { echo "::error file=$file::missing description"; fail=1; } - echo "$fm" | grep -q "^applyTo:" || { echo "::error file=$file::missing applyTo"; fail=1; } + # Only require applyTo for instructions files + if [[ "$file" == .github/instructions/* ]]; then + echo "$fm" | grep -q "^applyTo:" || { echo "::error file=$file::missing applyTo"; fail=1; } + fi done < <(find .github/prompts .github/chatmodes .github/instructions -type f -name "*.md" -print0) exit $fail From 813f4a49aa48047a3b75a9132c35940160a68c71 Mon Sep 17 00:00:00 2001 From: Ash Shaw Date: Wed, 12 Nov 2025 21:14:08 +0700 Subject: [PATCH 4/8] Update .github/workflows/aiops-secrets-scan.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Ash Shaw --- .github/workflows/aiops-secrets-scan.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/aiops-secrets-scan.yml b/.github/workflows/aiops-secrets-scan.yml index bd57d51519..a4e876a28e 100644 --- a/.github/workflows/aiops-secrets-scan.yml +++ b/.github/workflows/aiops-secrets-scan.yml @@ -15,3 +15,7 @@ jobs: uses: gitleaks/gitleaks-action@v2 with: args: detect -v --source . --report-format sarif --report-path gitleaks.sarif + - name: Upload SARIF to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: gitleaks.sarif From 2c5e38429660f8b418bd47d5fc3d73b109782661 Mon Sep 17 00:00:00 2001 From: Ash Shaw Date: Wed, 12 Nov 2025 21:14:16 +0700 Subject: [PATCH 5/8] Update .github/workflows/aiops-frontmatter.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Ash Shaw --- .github/workflows/aiops-frontmatter.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/aiops-frontmatter.yml b/.github/workflows/aiops-frontmatter.yml index f8e469d357..49b1591607 100644 --- a/.github/workflows/aiops-frontmatter.yml +++ b/.github/workflows/aiops-frontmatter.yml @@ -17,6 +17,8 @@ jobs: run: | set -euo pipefail fail=0 + # Use null-delimited output from 'find ... -print0' to safely handle filenames with spaces. + # If you change the find command, ensure -print0 is preserved to match the '-d ''' in the read loop. while IFS= read -r -d '' file; do # Must start with '---' head -1 "$file" | grep -q "^---" || { echo "::error file=$file::missing YAML front matter"; fail=1; continue; } From 03b97d31fe57ca586fb4735b4d66da68617fac8a Mon Sep 17 00:00:00 2001 From: Ash Shaw Date: Wed, 12 Nov 2025 21:14:25 +0700 Subject: [PATCH 6/8] Update .github/workflows/aiops-index-drift.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Ash Shaw --- .github/workflows/aiops-index-drift.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aiops-index-drift.yml b/.github/workflows/aiops-index-drift.yml index 1958a4d180..db2b2eba8e 100644 --- a/.github/workflows/aiops-index-drift.yml +++ b/.github/workflows/aiops-index-drift.yml @@ -20,7 +20,7 @@ jobs: fail=0 check_dir() { local dir="$1"; local index="$2"; local pattern="$3" - leaves=$(find "$dir" -maxdepth 1 -type f -name "$pattern" -printf "%f\n" | sort) + leaves=$(find "$dir" -maxdepth 1 -type f -name "$pattern" -exec basename {} \; | sort) [[ -f "$index" ]] || { echo "::error file=$index::missing index file"; fail=1; return; } for f in $leaves; do grep -q "$f" "$index" || { echo "::error file=$index::missing entry for $f"; fail=1; } From 7f84f9eb55bab07fab429fd58a21a7b66a7c8f18 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 13 Nov 2025 07:44:10 +0000 Subject: [PATCH 7/8] Initial plan From 1c23a34484e11170c33c26b9cb15844a48a9eeaa Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 13 Nov 2025 07:44:29 +0000 Subject: [PATCH 8/8] Initial plan