diff --git a/.github/scripts/keepalive_loop.js b/.github/scripts/keepalive_loop.js index 87854e7c..a5afb996 100644 --- a/.github/scripts/keepalive_loop.js +++ b/.github/scripts/keepalive_loop.js @@ -229,10 +229,14 @@ function classifyFailureDetails({ action, runResult, summaryReason, agentExitCod const message = [agentSummary, summaryReason, runResult].filter(Boolean).join(' '); const errorInfo = classifyError({ message, code: agentExitCode }); let category = errorInfo.category; + const isGateCancelled = summaryReason.startsWith('gate-cancelled'); if (runFailed && (runResult === 'cancelled' || runResult === 'skipped')) { category = ERROR_CATEGORIES.transient; } + if (!runFailed && isGateCancelled) { + category = ERROR_CATEGORIES.transient; + } let type = ''; if (runFailed) { @@ -572,12 +576,20 @@ async function classifyGateFailure({ github, context, pr, core }) { async function resolveGateConclusion({ github, context, pr, eventName, payload, core }) { + const run = await resolveGateRun({ github, context, pr, eventName, payload, core }); + return run.conclusion; +} + +async function resolveGateRun({ github, context, pr, eventName, payload, core }) { if (eventName === 'workflow_run') { - return normalise(payload?.workflow_run?.conclusion); + return { + conclusion: normalise(payload?.workflow_run?.conclusion), + runId: payload?.workflow_run?.id ? Number(payload.workflow_run.id) : 0, + }; } if (!pr) { - return ''; + return { conclusion: '', runId: 0 }; } try { @@ -592,23 +604,166 @@ async function resolveGateConclusion({ github, context, pr, eventName, payload, if (Array.isArray(data?.workflow_runs)) { const match = data.workflow_runs.find((run) => run.head_sha === pr.head.sha); if (match) { - return normalise(match.conclusion); + return { + conclusion: normalise(match.conclusion), + runId: Number(match.id) || 0, + }; } const latest = data.workflow_runs[0]; if (latest) { - return normalise(latest.conclusion); + return { + conclusion: normalise(latest.conclusion), + runId: Number(latest.id) || 0, + }; } } } catch (error) { if (core) core.info(`Failed to resolve Gate conclusion: ${error.message}`); } - return ''; + return { conclusion: '', runId: 0 }; +} + +function extractCheckRunId(job) { + const directId = Number(job?.check_run_id); + if (Number.isFinite(directId) && directId > 0) { + return directId; + } + const url = normalise(job?.check_run_url ?? job?.check_run?.url); + const match = url.match(/\/check-runs\/(\d+)/i); + if (match) { + return Number(match[1]) || 0; + } + return 0; +} + +const RATE_LIMIT_PATTERNS = [ + /rate limit/i, + /rate[-\s]limit/i, + /rate[-\s]limited/i, + /secondary rate limit/i, + /abuse detection/i, + /too many requests/i, + /api rate/i, + /exceeded.*rate limit/i, +]; + +function hasRateLimitSignal(text) { + const candidate = normalise(text); + if (!candidate) { + return false; + } + return RATE_LIMIT_PATTERNS.some((pattern) => pattern.test(candidate)); +} + +function annotationsContainRateLimit(annotations = []) { + for (const annotation of annotations) { + const combined = [ + annotation?.message, + annotation?.title, + annotation?.raw_details, + ] + .filter(Boolean) + .join(' '); + if (hasRateLimitSignal(combined)) { + return true; + } + } + return false; +} + +function extractRateLimitLogText(data) { + if (!data) { + return ''; + } + const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data); + if (buffer.length >= 2 && buffer[0] === 0x1f && buffer[1] === 0x8b) { + try { + const zlib = require('zlib'); + return zlib.gunzipSync(buffer).toString('utf8'); + } catch (error) { + return buffer.toString('utf8'); + } + } + return buffer.toString('utf8'); } -async function evaluateKeepaliveLoop({ github, context, core, payload: overridePayload }) { +function logContainsRateLimit(data) { + const text = extractRateLimitLogText(data); + if (!text) { + return false; + } + const sample = text.length > 500000 ? `${text.slice(0, 250000)}\n${text.slice(-250000)}` : text; + return hasRateLimitSignal(sample); +} + +async function detectRateLimitCancellation({ github, context, runId, core }) { + const targetRunId = Number(runId) || 0; + if (!targetRunId || !github?.rest?.actions?.listJobsForWorkflowRun) { + return false; + } + const canCheckAnnotations = Boolean(github?.rest?.checks?.listAnnotations); + const canCheckLogs = Boolean(github?.rest?.actions?.downloadJobLogsForWorkflowRun); + if (!canCheckAnnotations && !canCheckLogs) { + if (core) core.info('Rate limit detection skipped; no annotations or logs API available.'); + return false; + } + + try { + const { data } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: targetRunId, + per_page: 100, + }); + const jobs = Array.isArray(data?.jobs) ? data.jobs : []; + for (const job of jobs) { + if (canCheckAnnotations) { + const checkRunId = extractCheckRunId(job); + if (checkRunId) { + const params = { + owner: context.repo.owner, + repo: context.repo.repo, + check_run_id: checkRunId, + per_page: 100, + }; + const annotations = github.paginate + ? await github.paginate(github.rest.checks.listAnnotations, params) + : (await github.rest.checks.listAnnotations(params))?.data; + if (annotationsContainRateLimit(annotations)) { + return true; + } + } + } + + if (canCheckLogs) { + const jobId = Number(job?.id) || 0; + if (jobId) { + try { + const logs = await github.rest.actions.downloadJobLogsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + job_id: jobId, + }); + if (logContainsRateLimit(logs?.data)) { + return true; + } + } catch (error) { + if (core) core.info(`Failed to inspect Gate job logs for rate limits: ${error.message}`); + } + } + } + } + } catch (error) { + if (core) core.info(`Failed to inspect Gate cancellation signals for rate limits: ${error.message}`); + } + + return false; +} + +async function evaluateKeepaliveLoop({ github, context, core, payload: overridePayload, overridePrNumber, forceRetry }) { const payload = overridePayload || context.payload || {}; - const prNumber = await resolvePrNumber({ github, context, core, payload }); + let prNumber = overridePrNumber || await resolvePrNumber({ github, context, core, payload }); if (!prNumber) { return { prNumber: 0, @@ -623,7 +778,7 @@ async function evaluateKeepaliveLoop({ github, context, core, payload: overrideP pull_number: prNumber, }); - const gateConclusion = await resolveGateConclusion({ + const gateRun = await resolveGateRun({ github, context, pr, @@ -631,7 +786,9 @@ async function evaluateKeepaliveLoop({ github, context, core, payload: overrideP payload, core, }); + const gateConclusion = gateRun.conclusion; const gateNormalized = normalise(gateConclusion).toLowerCase(); + let gateRateLimit = false; const config = parseConfig(pr.body || ''); const labels = Array.isArray(pr.labels) ? pr.labels.map((label) => normalise(label.name).toLowerCase()) : []; @@ -697,14 +854,37 @@ async function evaluateKeepaliveLoop({ github, context, core, payload: overrideP action = 'stop'; reason = isProductive ? 'max-iterations' : 'max-iterations-unproductive'; } else if (gateNormalized !== 'success') { - // Gate failed - check if we should route to fix mode or wait - const gateFailure = await classifyGateFailure({ github, context, pr, core }); - if (gateFailure.shouldFixMode && gateNormalized === 'failure') { - action = 'fix'; - reason = `fix-${gateFailure.failureType}`; + if (gateNormalized === 'cancelled') { + gateRateLimit = await detectRateLimitCancellation({ + github, + context, + runId: gateRun.runId, + core, + }); + // forceRetry bypasses defer/wait for cancelled gates + if (forceRetry && tasksRemaining) { + action = 'run'; + reason = 'force-retry-cancelled'; + if (core) core.info(`Force retry enabled: bypassing cancelled gate (rate_limit=${gateRateLimit})`); + } else { + action = gateRateLimit ? 'defer' : 'wait'; + reason = gateRateLimit ? 'gate-cancelled-rate-limit' : 'gate-cancelled'; + } } else { - action = 'wait'; - reason = gateNormalized ? 'gate-not-success' : 'gate-pending'; + // Gate failed - check if we should route to fix mode or wait + const gateFailure = await classifyGateFailure({ github, context, pr, core }); + if (gateFailure.shouldFixMode && gateNormalized === 'failure') { + action = 'fix'; + reason = `fix-${gateFailure.failureType}`; + } else if (forceRetry && tasksRemaining) { + // forceRetry can also bypass non-success gates (user explicitly wants to retry) + action = 'run'; + reason = 'force-retry-gate'; + if (core) core.info(`Force retry enabled: bypassing gate conclusion '${gateNormalized}'`); + } else { + action = 'wait'; + reason = gateNormalized ? 'gate-not-success' : 'gate-pending'; + } } } else if (tasksRemaining) { action = 'run'; @@ -737,6 +917,7 @@ async function evaluateKeepaliveLoop({ github, context, core, payload: overrideP keepaliveEnabled, stateCommentId: stateResult.commentId || 0, state, + forceRetry: Boolean(forceRetry), }; } @@ -769,6 +950,11 @@ async function updateKeepaliveLoopSummary({ github, context, core, inputs }) { const agentSummary = normalise(inputs.agent_summary ?? inputs.agentSummary ?? inputs.codex_summary ?? inputs.codexSummary); const runUrl = normalise(inputs.run_url ?? inputs.runUrl); + // LLM task analysis details + const llmProvider = normalise(inputs.llm_provider ?? inputs.llmProvider); + const llmConfidence = toNumber(inputs.llm_confidence ?? inputs.llmConfidence, 0); + const llmAnalysisRun = toBool(inputs.llm_analysis_run ?? inputs.llmAnalysisRun, false); + const { state: previousState, commentId } = await loadKeepaliveState({ github, context, @@ -791,6 +977,7 @@ async function updateKeepaliveLoopSummary({ github, context, core, inputs }) { const isNeutralStop = reason === 'no-checklists' || reason === 'keepalive-disabled'; let stop = action === 'stop' && !isSuccessStop && !isNeutralStop; let summaryReason = reason || action || 'unknown'; + const baseReason = summaryReason; const transientDetails = classifyFailureDetails({ action, runResult, @@ -804,6 +991,16 @@ async function updateKeepaliveLoopSummary({ github, context, core, inputs }) { runResult && runResult !== 'success' && transientDetails.category === ERROR_CATEGORIES.transient; + const waitLikeAction = action === 'wait' || action === 'defer'; + const waitIsTransientReason = [ + 'gate-pending', + 'missing-agent-label', + 'gate-cancelled', + 'gate-cancelled-rate-limit', + ].includes(baseReason); + const isTransientWait = + waitLikeAction && + (transientDetails.category === ERROR_CATEGORIES.transient || waitIsTransientReason); // Task reconciliation: detect when agent made changes but didn't update checkboxes const previousTasks = previousState?.tasks || {}; @@ -856,12 +1053,12 @@ async function updateKeepaliveLoopSummary({ github, context, core, inputs }) { summaryReason = `${summaryReason}-repeat`; } } - } else if (action === 'wait') { + } else if (waitLikeAction) { // Wait states are NOT failures - they're transient conditions // Don't increment failure counter for: gate-pending, gate-not-success, missing-agent-label // These are expected states that will resolve on their own // Check if this is a transient error (from error classification) - if (transientDetails.category === ERROR_CATEGORIES.transient) { + if (isTransientWait) { failure = {}; summaryReason = `${summaryReason}-transient`; } else if (failure.reason && !failure.reason.startsWith('gate-') && failure.reason !== 'missing-agent-label') { @@ -915,6 +1112,22 @@ async function updateKeepaliveLoopSummary({ github, context, core, inputs }) { ? `**${maxIterations}+${extendedCount}** 🚀 extended` : `${nextIteration}/${maxIterations || '∞'}`; + const dispositionLabel = (() => { + if (action === 'defer') { + return 'deferred (transient)'; + } + if (action === 'wait') { + return isTransientWait ? 'skipped (transient)' : 'skipped (failure)'; + } + if (action === 'skip') { + return 'skipped'; + } + return ''; + })(); + const actionReason = waitLikeAction + ? (baseReason || summaryReason) + : (summaryReason || baseReason); + const summaryLines = [ '', `## 🤖 Keepalive Loop Status`, @@ -931,7 +1144,8 @@ async function updateKeepaliveLoopSummary({ github, context, core, inputs }) { : formatProgressBar(nextIteration, maxIterations) : 'n/a (unbounded)' } |`, - `| Action | ${action || 'unknown'} (${summaryReason || 'n/a'}) |`, + `| Action | ${action || 'unknown'} (${actionReason || 'n/a'}) |`, + ...(dispositionLabel ? [`| Disposition | ${dispositionLabel} |`] : []), ...(runFailed ? [`| Agent status | ❌ AGENT FAILED |`] : []), `| Gate | ${gateConclusion || 'unknown'} |`, `| Tasks | ${tasksComplete}/${tasksTotal} complete |`, @@ -1002,6 +1216,29 @@ async function updateKeepaliveLoopSummary({ github, context, core, inputs }) { } } + // LLM analysis details - show which provider was used for task completion detection + if (llmAnalysisRun && llmProvider) { + const providerIcon = llmProvider === 'github-models' ? '✅' : + llmProvider === 'openai' ? 'âš ī¸' : + llmProvider === 'regex-fallback' ? 'đŸ”ļ' : 'â„šī¸'; + const providerLabel = llmProvider === 'github-models' ? 'GitHub Models (primary)' : + llmProvider === 'openai' ? 'OpenAI (fallback)' : + llmProvider === 'regex-fallback' ? 'Regex (fallback)' : llmProvider; + const confidencePercent = Math.round(llmConfidence * 100); + summaryLines.push( + '', + '### 🧠 Task Analysis', + `| Provider | ${providerIcon} ${providerLabel} |`, + `| Confidence | ${confidencePercent}% |`, + ); + if (llmProvider !== 'github-models') { + summaryLines.push( + '', + `> âš ī¸ Primary provider (GitHub Models) was unavailable; used ${providerLabel} instead.`, + ); + } + } + if (isTransientFailure) { summaryLines.push( '', @@ -1010,6 +1247,14 @@ async function updateKeepaliveLoopSummary({ github, context, core, inputs }) { ); } + if (action === 'defer') { + summaryLines.push( + '', + '### âŗ Deferred', + 'Keepalive deferred due to a transient Gate cancellation (likely rate limits). It will retry later.', + ); + } + // Show failure tracking prominently if there are failures if (failure.count > 0) { summaryLines.push( @@ -1465,12 +1710,13 @@ async function analyzeTaskCompletion({ github, context, prNumber, baseSha, headS * @param {number} params.prNumber - PR number * @param {string} params.baseSha - Base SHA (before agent work) * @param {string} params.headSha - Head SHA (after agent work) + * @param {string[]} [params.llmCompletedTasks] - Tasks marked complete by LLM analysis * @param {object} [params.core] - Optional core for logging * @returns {Promise<{updated: boolean, tasksChecked: number, details: string}>} */ -async function autoReconcileTasks({ github, context, prNumber, baseSha, headSha, core }) { +async function autoReconcileTasks({ github, context, prNumber, baseSha, headSha, llmCompletedTasks, core }) { const log = (msg) => core?.info?.(msg) || console.log(msg); - + // Get current PR body let pr; try { @@ -1493,13 +1739,39 @@ async function autoReconcileTasks({ github, context, prNumber, baseSha, headSha, return { updated: false, tasksChecked: 0, details: 'No tasks found in PR body' }; } - // Analyze what tasks may have been completed + // Build high-confidence matches from multiple sources + let highConfidence = []; + + // Source 1: LLM analysis (highest priority if available) + if (llmCompletedTasks && Array.isArray(llmCompletedTasks) && llmCompletedTasks.length > 0) { + log(`LLM analysis found ${llmCompletedTasks.length} completed task(s)`); + for (const task of llmCompletedTasks) { + highConfidence.push({ + task, + reason: 'LLM session analysis', + confidence: 'high', + source: 'llm', + }); + } + } + + // Source 2: Commit/file analysis (fallback or supplementary) const analysis = await analyzeTaskCompletion({ github, context, prNumber, baseSha, headSha, taskText, core }); - // Only auto-check high-confidence matches - const highConfidence = analysis.matches.filter(m => m.confidence === 'high'); + // Add commit-based matches that aren't already covered by LLM + const llmTasksLower = new Set((llmCompletedTasks || []).map(t => t.toLowerCase())); + const commitMatches = analysis.matches + .filter(m => m.confidence === 'high') + .filter(m => !llmTasksLower.has(m.task.toLowerCase())); + + if (commitMatches.length > 0) { + log(`Commit analysis found ${commitMatches.length} additional task(s)`); + for (const match of commitMatches) { + highConfidence.push({ ...match, source: 'commit' }); + } + } if (highConfidence.length === 0) { log('No high-confidence task matches to auto-check'); @@ -1549,14 +1821,26 @@ async function autoReconcileTasks({ github, context, prNumber, baseSha, headSha, return { updated: false, tasksChecked: 0, - details: `Failed to update PR: ${error.message}` + details: `Failed to update PR: ${error.message}`, + sources: { llm: 0, commit: 0 }, }; } + // Count matches by source for reporting + const llmCount = highConfidence.filter(m => m.source === 'llm').length; + const commitCount = highConfidence.filter(m => m.source === 'commit').length; + + // Build detailed description + const sourceDesc = []; + if (llmCount > 0) sourceDesc.push(`${llmCount} from LLM analysis`); + if (commitCount > 0) sourceDesc.push(`${commitCount} from commit analysis`); + const sourceInfo = sourceDesc.length > 0 ? ` (${sourceDesc.join(', ')})` : ''; + return { updated: true, tasksChecked: checkedCount, - details: `Auto-checked ${checkedCount} task(s): ${highConfidence.map(m => m.task.slice(0, 30) + '...').join(', ')}` + details: `Auto-checked ${checkedCount} task(s)${sourceInfo}: ${highConfidence.map(m => m.task.slice(0, 30) + '...').join(', ')}`, + sources: { llm: llmCount, commit: commitCount }, }; } diff --git a/.github/workflows/agents-keepalive-loop.yml b/.github/workflows/agents-keepalive-loop.yml index f070017f..f228686a 100644 --- a/.github/workflows/agents-keepalive-loop.yml +++ b/.github/workflows/agents-keepalive-loop.yml @@ -438,6 +438,18 @@ jobs: const beforeSha = '${{ needs.evaluate.outputs.head_sha }}'; const headSha = '${{ needs.run-codex.outputs.commit-sha }}'; + // Parse LLM completed tasks if available + let llmCompletedTasks = []; + const llmTasksJson = '${{ needs.run-codex.outputs.llm-completed-tasks || '[]' }}'; + try { + llmCompletedTasks = JSON.parse(llmTasksJson); + if (llmCompletedTasks.length > 0) { + core.info(`LLM analysis found ${llmCompletedTasks.length} completed task(s)`); + } + } catch (e) { + core.debug(`Failed to parse LLM tasks: ${e.message}`); + } + if (!prNumber || !beforeSha || !headSha) { core.info('Missing required inputs for task reconciliation'); return; @@ -447,12 +459,12 @@ jobs: core.info(`Comparing ${beforeSha.slice(0, 7)} → ${headSha.slice(0, 7)}`); const result = await autoReconcileTasks({ - github, context, prNumber, baseSha: beforeSha, headSha, core + github, context, prNumber, baseSha: beforeSha, headSha, llmCompletedTasks, core }); if (result.updated) { core.info(`✅ ${result.details}`); - core.notice(`Auto-checked ${result.tasksChecked} task(s) based on commit analysis`); + core.notice(`Auto-checked ${result.tasksChecked} task(s) based on analysis`); } else { core.info(`â„šī¸ ${result.details}`); } @@ -488,5 +500,9 @@ jobs: agent_commit_sha: '${{ needs.run-codex.outputs.commit-sha }}', agent_files_changed: '${{ needs.run-codex.outputs.files-changed }}', agent_summary: process.env.CODEX_SUMMARY || '', + // LLM task analysis provider info + llm_provider: '${{ needs.run-codex.outputs.llm-provider || '' }}', + llm_confidence: '${{ needs.run-codex.outputs.llm-confidence || '' }}', + llm_analysis_run: '${{ needs.run-codex.outputs.llm-analysis-run }}' === 'true', }; await updateKeepaliveLoopSummary({ github, context, core, inputs });