Skip to content

[codex] Suppress legacy verifier model metadata warnings #1877

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stranske merged 1 commit into from
Apr 26, 2026
Merged

[codex] Suppress legacy verifier model metadata warnings #1877

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions .github/scripts/__tests__/terminal-disposition-coverage.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ const {
normalizeArtifactSelectionSummary,
normalizeExpectedSource,
normalizeUnsupportedCodexModels,
normalizeVerifierModelMetadataContract,
readArtifactSelectionReport,
readNdjsonFiles,
summarizeVerifierModelCompatibility,
Expand Down Expand Up @@ -250,6 +251,10 @@ test('summarizes verifier model compatibility with configurable unsupported mode
'bad-model',
'gpt-5.2-codex',
]);
assert.equal(
normalizeVerifierModelMetadataContract().required_after,
'2026-04-26T04:25:00Z'
);

const summary = summarizeVerifierModelCompatibility(
[
Expand Down Expand Up @@ -317,6 +322,83 @@ test('warns when Codex verifier terminal records omit model metadata', () => {
assert.doesNotMatch(markdown, /\| pull-request:1873 \| verified-pass \| evaluate/);
});

test('suppresses pre-contract verifier terminal records missing model metadata', () => {
const report = summarizeTerminalDispositionCoverage(
[
{
schema: 'workflows-terminal-disposition/v1',
artifact_family: 'verifier-terminal-disposition',
source_type: 'pull-request',
source_id: '1872',
pr_number: 1872,
run_id: '24948023778',
disposition: 'verifier-error',
},
],
{
input_file_count: 1,
artifact_selection_report: {
schema: 'workflows-weekly-metrics-artifact-selection/v1',
status: 'pass',
selected_artifacts: [
{
id: 6644772052,
name: 'verifier-terminal-disposition-24948023778',
family: 'verifier-terminal-disposition',
created_at: '2026-04-26T04:18:01Z',
},
],
},
}
);
const markdown = formatTerminalDispositionCoverageMarkdown(report);

assert.equal(report.status, 'pass');
assert.equal(report.verifier_model_compatibility.status, 'pass');
assert.equal(report.verifier_model_compatibility.missing_model_record_count, 0);
assert.equal(report.verifier_model_compatibility.legacy_missing_model_record_count, 1);
assert.deepEqual(report.enforcement.blockers, []);
assert.match(markdown, /Legacy missing verifier model metadata records: 1/);
assert.match(markdown, /2026-04-26T04:18:01Z/);
});

test('still warns for post-contract verifier terminal records missing model metadata', () => {
const report = summarizeTerminalDispositionCoverage(
[
{
schema: 'workflows-terminal-disposition/v1',
artifact_family: 'verifier-terminal-disposition',
source_type: 'pull-request',
source_id: '1877',
pr_number: 1877,
run_id: '24950000000',
disposition: 'verifier-error',
},
],
{
input_file_count: 1,
artifact_selection_report: {
schema: 'workflows-weekly-metrics-artifact-selection/v1',
status: 'pass',
selected_artifacts: [
{
id: 6645000000,
name: 'verifier-terminal-disposition-24950000000',
family: 'verifier-terminal-disposition',
created_at: '2026-04-26T06:00:00Z',
},
],
},
}
);

assert.equal(report.status, 'warning');
assert.equal(report.verifier_model_compatibility.status, 'warning');
assert.equal(report.verifier_model_compatibility.missing_model_record_count, 1);
assert.equal(report.verifier_model_compatibility.legacy_missing_model_record_count, 0);
assert.deepEqual(report.enforcement.blockers, ['unsupported-verifier-model']);
});

test('reads ndjson files and counts parse errors', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'terminal-coverage-'));
const file = path.join(dir, 'records.ndjson');
Expand Down
136 changes: 126 additions & 10 deletions .github/scripts/terminal_disposition_coverage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ const TERMINAL_ARTIFACT_FAMILIES = new Set([
'review-thread-terminal-disposition',
]);
const DEFAULT_UNSUPPORTED_CODEX_MODELS = ['gpt-5.2-codex'];
const DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = '2026-04-26T04:25:00Z';
const DEFAULT_ENFORCEMENT_MODE = 'warning-only';
const HARD_BLOCK_MODE = 'hard-block';

Expand All @@ -28,6 +29,13 @@ function cleanInt(value) {
return Number.isFinite(parsed) ? parsed : null;
}

function parseTimestampMs(value) {
const text = cleanString(value);
if (!text) return null;
const parsed = Date.parse(text);
return Number.isFinite(parsed) ? parsed : null;
}

function normalizeBoolean(value) {
const text = cleanString(value).toLowerCase();
return ['1', 'true', 'yes', 'y', 'approved', 'approve', 'on'].includes(text);
Expand Down Expand Up @@ -80,15 +88,76 @@ function normalizeUnsupportedCodexModels(value) {
.sort((a, b) => a.localeCompare(b));
}

function normalizeVerifierModelMetadataContract(value) {
const raw = value ??
process.env.TERMINAL_DISPOSITION_VERIFIER_MODEL_METADATA_REQUIRED_AFTER ??
DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER;
const text = cleanString(raw);
if (['', '0', 'false', 'none', 'off', 'disabled'].includes(text.toLowerCase())) {
return {
required_after: '',
required_after_epoch_ms: null,
suppress_pre_contract_missing_metadata: false,
};
}
const epochMs = parseTimestampMs(text);
return {
required_after: text,
required_after_epoch_ms: epochMs,
suppress_pre_contract_missing_metadata: epochMs !== null,
Comment on lines +104 to +107

Copilot AI Apr 26, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the required-after timestamp is invalid/unparseable, parseTimestampMs() returns null but required_after is still set to the raw string, and the markdown renderer will print it even though suppression is effectively disabled. Consider treating an invalid timestamp the same as “disabled” (clear required_after and set suppress_pre_contract_missing_metadata to false), or fall back to the default timestamp, to avoid misleading output.

Suggested change
return {
required_after: text,
required_after_epoch_ms: epochMs,
suppress_pre_contract_missing_metadata: epochMs !== null,
if (epochMs === null) {
return {
required_after: '',
required_after_epoch_ms: null,
suppress_pre_contract_missing_metadata: false,
};
}
return {
required_after: text,
required_after_epoch_ms: epochMs,
suppress_pre_contract_missing_metadata: true,

Copilot uses AI. Check for mistakes.
};
}

function runIdFromArtifactName(name) {
const match = cleanString(name).match(/-(\d+)$/);
return match ? match[1] : '';
}

function artifactMetadataByRunId(artifactSelection) {
const byRunId = new Map();
const artifacts = artifactSelection?.selected_terminal_artifacts || [];
for (const artifact of artifacts) {
if (artifact?.family !== 'verifier-terminal-disposition') continue;
const runId = runIdFromArtifactName(artifact.name);
if (!runId) continue;
byRunId.set(runId, {
artifact_name: cleanString(artifact.name),
created_at: cleanString(artifact.created_at),
updated_at: cleanString(artifact.updated_at),
});
}
return byRunId;
}

function isPreContractVerifierModelRecord(record, artifactMetadata, contract) {
if (!contract.suppress_pre_contract_missing_metadata) return false;
const requiredAfter = contract.required_after_epoch_ms;
const candidates = [
record.created_at,
record.timestamp,
artifactMetadata?.created_at,
artifactMetadata?.updated_at,
];
return candidates.some((value) => {
const epochMs = parseTimestampMs(value);
return epochMs !== null && epochMs < requiredAfter;
});
}

function summarizeVerifierModelCompatibility(records = [], options = {}) {
const unsupportedModels = normalizeUnsupportedCodexModels(
options.unsupported_codex_models ?? options.unsupportedCodexModels
);
const modelMetadataContract = normalizeVerifierModelMetadataContract(
options.model_metadata_required_after ?? options.modelMetadataRequiredAfter
);
const artifactMetadata = artifactMetadataByRunId(options.artifact_selection);
const unsupportedSet = new Set(unsupportedModels);
const selectedModels = {};
const modelSelectionReasons = {};
const unsupportedRecords = [];
const missingModelRecords = [];
const legacyMissingModelRecords = [];
let verifierRecordCount = 0;

for (const raw of records) {
Expand All @@ -107,13 +176,21 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) {
if (model) selectedModels[model] = (selectedModels[model] || 0) + 1;
if (reason) modelSelectionReasons[reason] = (modelSelectionReasons[reason] || 0) + 1;
if (!model && requiresCodexModel) {
missingModelRecords.push({
const runId = cleanString(record.run_id);
const metadata = artifactMetadata.get(runId);
const missingRecord = {
source_key: record.source_key,
pr_number: record.pr_number || null,
run_id: cleanString(record.run_id),
run_id: runId,
disposition: record.disposition,
verifier_mode: verifierMode || 'unknown',
});
};
if (metadata?.created_at) missingRecord.artifact_created_at = metadata.created_at;
if (isPreContractVerifierModelRecord(record, metadata, modelMetadataContract)) {
legacyMissingModelRecords.push(missingRecord);
} else {
missingModelRecords.push(missingRecord);
}
}
if (model && unsupportedSet.has(model)) {
unsupportedRecords.push({
Expand All @@ -130,10 +207,12 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) {
return {
schema: 'workflows-verifier-model-compatibility/v1',
status: unsupportedRecords.length > 0 || missingModelRecords.length > 0 ? 'warning' : 'pass',
model_metadata_contract: modelMetadataContract,
verifier_record_count: verifierRecordCount,
unsupported_models: unsupportedModels,
unsupported_record_count: unsupportedRecords.length,
missing_model_record_count: missingModelRecords.length,
legacy_missing_model_record_count: legacyMissingModelRecords.length,
selected_models: Object.fromEntries(
Object.entries(selectedModels).sort((a, b) => a[0].localeCompare(b[0]))
),
Expand All @@ -146,6 +225,9 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) {
missing_model_records: missingModelRecords.sort((a, b) =>
`${a.source_key}:${a.run_id}`.localeCompare(`${b.source_key}:${b.run_id}`)
),
legacy_missing_model_records: legacyMissingModelRecords.sort((a, b) =>
`${a.source_key}:${a.run_id}`.localeCompare(`${b.source_key}:${b.run_id}`)
),
};
}

Expand Down Expand Up @@ -199,7 +281,10 @@ function summarizeTerminalDispositionCoverage(records = [], options = {}) {
const terminalRecords = records
.filter(isTerminalDispositionRecord)
.map((record) => normalizeTerminalDisposition(record));
const verifierModelCompatibility = summarizeVerifierModelCompatibility(terminalRecords, options);
const verifierModelCompatibility = summarizeVerifierModelCompatibility(terminalRecords, {
...options,
artifact_selection: artifactSelection,
});
const scannedRecordCount = records.length;
const nonTerminalRecordCount = scannedRecordCount - terminalRecords.length;
const inputFiles = Array.isArray(options.input_files) ? options.input_files.map(cleanString).filter(Boolean) : [];
Expand Down Expand Up @@ -418,11 +503,18 @@ function normalizeArtifactSelectionSummary(report) {

const selectedArtifacts = Array.isArray(report.selected_artifacts)
? report.selected_artifacts
.map((artifact) => ({
id: artifact.id ?? artifact.artifact_id ?? artifact.artifactId ?? null,
name: cleanString(artifact.name),
family: artifactFamilyFromSelection(artifact),
}))
.map((artifact) => {
const normalized = {
id: artifact.id ?? artifact.artifact_id ?? artifact.artifactId ?? null,
name: cleanString(artifact.name),
family: artifactFamilyFromSelection(artifact),
};
const createdAt = cleanString(artifact.created_at ?? artifact.createdAt);
const updatedAt = cleanString(artifact.updated_at ?? artifact.updatedAt);
if (createdAt) normalized.created_at = createdAt;
if (updatedAt) normalized.updated_at = updatedAt;
return normalized;
})
.filter((artifact) => TERMINAL_ARTIFACT_FAMILIES.has(artifact.family))
: [];
const terminalFamilyStatuses = normalizeTerminalPriorityFamilyStatuses(report, selectedArtifacts);
Expand Down Expand Up @@ -499,8 +591,14 @@ function formatTerminalDispositionCoverageMarkdown(report) {
lines.push(
`- Verifier model compatibility: ${modelCompatibility.status}`,
`- Unsupported verifier model records: ${modelCompatibility.unsupported_record_count}`,
`- Missing verifier model metadata records: ${modelCompatibility.missing_model_record_count}`
`- Missing verifier model metadata records: ${modelCompatibility.missing_model_record_count}`,
`- Legacy missing verifier model metadata records: ${modelCompatibility.legacy_missing_model_record_count || 0}`
);
if (modelCompatibility.model_metadata_contract?.required_after) {
lines.push(
`- Verifier model metadata required after: ${modelCompatibility.model_metadata_contract.required_after}`
);
}
if (modelCompatibility.unsupported_models?.length > 0) {
lines.push(`- Unsupported verifier models: ${modelCompatibility.unsupported_models.join(', ')}`);
}
Expand Down Expand Up @@ -577,6 +675,23 @@ function formatTerminalDispositionCoverageMarkdown(report) {
}
}

const legacyMissingModelRecords = modelCompatibility?.legacy_missing_model_records || [];
if (legacyMissingModelRecords.length > 0) {
lines.push(
'',
'| Legacy missing verifier model source | Disposition | Mode | PR | Run | Artifact created |',
'|--------------------------------------|-------------|------|----|-----|------------------|'
);
for (const record of legacyMissingModelRecords) {
const pr = record.pr_number ? `#${record.pr_number}` : 'n/a';
const run = record.run_id || 'n/a';
const createdAt = record.artifact_created_at || 'n/a';
lines.push(
`| ${record.source_key} | ${record.disposition} | ${record.verifier_mode} | ${pr} | ${run} | ${createdAt} |`
);
}
}

if (report.status === 'no-data') {
lines.push('', '_No terminal disposition records were found in the metrics input._');
} else if (report.terminal_record_count === 0 && (report.non_terminal_record_count || 0) > 0) {
Expand Down Expand Up @@ -764,6 +879,7 @@ module.exports = {
normalizeArtifactSelectionSummary,
normalizeTerminalPriorityFamilyStatuses,
normalizeUnsupportedCodexModels,
normalizeVerifierModelMetadataContract,
readNdjsonFiles,
readArtifactSelectionReport,
summarizeVerifierModelCompatibility,
Expand Down
Loading
Loading