ci: Add automatic flaky test detector

.github/FLAKY_CI_FAILURE_TEMPLATE.md

@@ -0,0 +1,24 @@
+---
+title: '[Flaky CI]: {{ env.JOB_NAME }}'
+labels: Tests
+---
+
+### Flakiness Type
+
+Other / Unknown
+
+### Name of Job
+
+{{ env.JOB_NAME }}
+
+### Name of Test
+
+_Not available - check the run link for details_
+
+### Link to Test Run
+
+{{ env.RUN_LINK }}
+
+---
+
+_This issue was automatically created._

.github/workflows/build.yml

@@ -1168,7 +1168,85 @@ jobs:
  # Always run this, even if a dependent job failed
  if: always()
  runs-on: ubuntu-24.04
+ permissions:
+ issues: write
  steps:
+ - name: Check out current commit
+ if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure')
+ uses: actions/checkout@v6
+ with:
+ sparse-checkout: .github
+
+ - name: Create issues for failed jobs
+ if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure')
+ uses: actions/github-script@v7
+ with:
+ script: |
+ const fs = require('fs');
+
+ // Fetch actual job details from the API to get descriptive names
+ const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ run_id: context.runId,
+ per_page: 100
+ });
+
+ const failedJobs = jobs.filter(job => job.conclusion === 'failure');
+
+ if (failedJobs.length === 0) {
+ console.log('No failed jobs found');
+ return;
+ }
+
+ // Read and parse template
+ const template = fs.readFileSync('.github/FLAKY_CI_FAILURE_TEMPLATE.md', 'utf8');
+ const [, frontmatter, bodyTemplate] = template.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+
+ // Get existing open issues with Tests label
+ const existing = await github.paginate(github.rest.issues.listForRepo, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ state: 'open',
+ labels: 'Tests',
+ per_page: 100
+ });
+
+ for (const job of failedJobs) {
+ const jobName = job.name;
+ const jobUrl = job.html_url;
+
+ // Replace template variables
+ const vars = {
+ 'JOB_NAME': jobName,
+ 'RUN_LINK': jobUrl
+ };
+
+ let title = frontmatter.match(/title:\s*'(.*)'/)[1];
+ let issueBody = bodyTemplate;
+ for (const [key, value] of Object.entries(vars)) {
+ const pattern = new RegExp(`\\{\\{\\s*env\\.${key}\\s*\\}\\}`, 'g');
+ title = title.replace(pattern, value);
+ issueBody = issueBody.replace(pattern, value);
+ }
+
+ const existingIssue = existing.find(i => i.title === title);
+
+ if (existingIssue) {
+ console.log(`Issue already exists for ${jobName}: #${existingIssue.number}`);
+ continue;
+ }
+
+ const newIssue = await github.rest.issues.create({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ title: title,
+ body: issueBody.trim(),
+ labels: ['Tests']
+ });
+ console.log(`Created issue #${newIssue.data.number} for ${jobName}`);
+ }
+
  - name: Check for failures
  if: cancelled() || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
  run: |