Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -523,11 +523,11 @@ test('keeps Local explicitly usable without routing default work away from an un
},
);

await assert.rejects(manager.enable(remoteTarget('offline')), /stopped responding/);
await assert.rejects(manager.enable(remoteTarget('offline')), /did not become ready/);
manager.setDefaultProfile('offline');
await assert.rejects(
manager.handleBotIncomingMessage({ text: 'default' } as BotIncomingMessage),
/stopped responding/,
/did not become ready/,
);

assert.equal(local.botMessages, 0);
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
ELECTION_DEADLINE_MS_ENV_VAR,
connectOrSpawnRuntimeHostWithDependencies,
electionDeadlineMsFromEnvironment,
} from '../client/connect-or-spawn.js';
import {
INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
RUNTIME_HOST_PROTOCOL_VERSION,
} from '../protocol/index.js';

test('treats an unset or blank override as unconfigured', () => {
assert.equal(electionDeadlineMsFromEnvironment(undefined), undefined);
assert.equal(electionDeadlineMsFromEnvironment(''), undefined);
assert.equal(electionDeadlineMsFromEnvironment(' '), undefined);
});

test('parses a valid millisecond override', () => {
assert.equal(electionDeadlineMsFromEnvironment('90000'), 90_000);
assert.equal(electionDeadlineMsFromEnvironment(' 5000 '), 5_000);
});

test('fails closed on an invalid override instead of silently ignoring it', () => {
for (const invalid of ['abc', '0', '-100', '120001', '1.5']) {
assert.throws(() => electionDeadlineMsFromEnvironment(invalid), RangeError);
}
assert.throws(
() => electionDeadlineMsFromEnvironment('abc'),
new RegExp(`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer`, 'u'),
);
});

test('an invalid environment override fails the election before touching storage', async () => {
await assert.rejects(
connectOrSpawnRuntimeHostWithDependencies(
{
rootPath: '/nonexistent-maka-3474-root',
protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION },
compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
candidateEntrypoint: 'candidate-entry.js',
},
{
launchCandidate: () => ({ spawned: Promise.reject(new Error('must not spawn')) }),
random: Math.random,
env: { [ELECTION_DEADLINE_MS_ENV_VAR]: 'not-a-number' },
},
),
(error: unknown) =>
error instanceof RangeError && /MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u.test(error.message),
);
});
13 changes: 12 additions & 1 deletion packages/runtime-host/src/__tests__/startup-error.test.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,18 @@ test('presents migration blockers with a permanent previous-release recovery pat
test('keeps an unresponsive Host retryable', () => {
const error = runtimeHostStartupError('host_unresponsive');
assert.equal(error instanceof RuntimeHostPermanentReconnectError, false);
assert.match(error.message, /stopped responding/u);
assert.match(error.message, /did not become ready before the startup deadline/u);
});

test('tells both timeout reasons how to widen the election window', () => {
assert.match(
runtimeHostStartupError('host_unresponsive').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
assert.match(
runtimeHostStartupError('startup_timeout').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
});

test('keeps internal startup failures retryable', () => {
Expand Down
30 changes: 29 additions & 1 deletion packages/runtime-host/src/client/connect-or-spawn.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@ const DEFAULT_ELECTION_DEADLINE_MS = 45_000;
const DEFAULT_BACKOFF_MIN_MS = 20;
const DEFAULT_BACKOFF_MAX_MS = 250;
const MIN_CANDIDATE_INTERVAL_MS = 250;
export const ELECTION_DEADLINE_MS_ENV_VAR = 'MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS';

export interface ConnectOrSpawnRuntimeHostInput {
rootPath: string;
Expand All@@ -56,13 +57,35 @@ export interface ConnectOrSpawnRuntimeHostInput {
interface ConnectOrSpawnRuntimeHostDependencies {
launchCandidate: CandidateLauncher;
random(): number;
/** Defaults to `process.env`; injected so tests never mutate the real environment. */
env?: NodeJS.ProcessEnv;
}

const defaultDependencies: ConnectOrSpawnRuntimeHostDependencies = {
launchCandidate: launchDetachedRuntimeHostCandidate,
random: Math.random,
};

/**
* Resolves the operator override for the client election deadline. Large
* workspaces can legitimately take longer than the default window on their
* first start after an upgrade, so the deadline must be raisable without a
* code change. Invalid values fail closed: a silently ignored typo would leave
* the operator believing they widened the window when they did not.
*/
export function electionDeadlineMsFromEnvironment(
rawValue: string | undefined,
): number | undefined {
if (rawValue === undefined || rawValue.trim() === '') return undefined;
const parsed = Number(rawValue);
if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 120_000) {
throw new RangeError(
`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer between 1 and 120000 milliseconds`,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] The error text points every startup failure at this variable, including the class it can't fix.

There's a second failure shape this knob doesn't address. From a real CI run on #3454, which now emits election diagnostics:

{"deadlineMs":45000,"elapsedMs":45174,"candidateLaunches":69,
"observations":{"notRegistered":73,"connectFailed":0,"handshakeFailed":0,
"connected":1,"readyWaitFailed":1}}

69 candidate launches in 45 seconds. Zero connect failures, zero handshake failures — it connected once, then ready-wait failed, and candidates kept being launched roughly every 0.65s until the deadline. It missed by 174ms, so raising the deadline would have made that particular run pass, but the run isn't slow — it isn't converging. At 90s it launches ~138 candidates instead of ~69 and passing becomes a matter of luck.

That's different from #3474's genuinely-slow start, where waiting longer is exactly right. The problem is that both produce the same message, so an operator hitting the retry-storm shape is told to raise the timeout, raises it, fails again, and raises it further.

Not blocking, and possibly not yours to fix here — but worth asking: with candidateLaunches high and connectFailed/handshakeFailed at zero, should the message distinguish "this start is slow, wait longer" from "candidates aren't converging, the timeout isn't your problem"? You have the #3474 case in hand and probably know which shape operators hit more often.

);
}
return parsed;
}

export type ConnectOrSpawnRuntimeHostResult =
| {
kind: 'connected';
Expand DownExpand Up@@ -173,7 +196,12 @@ export async function connectOrSpawnRuntimeHostWithDependencies(
input: ConnectOrSpawnRuntimeHostInput,
dependencies: ConnectOrSpawnRuntimeHostDependencies,
): Promise<ConnectOrSpawnRuntimeHostResult> {
const deadlineMs = input.electionDeadlineMs ?? DEFAULT_ELECTION_DEADLINE_MS;
const deadlineMs =
input.electionDeadlineMs ??
electionDeadlineMsFromEnvironment(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] On the owned-launch path an invalid value loses the variable name and surfaces as host_unresponsive.

electionDeadlineMsFromEnvironment() throws a RangeError naming the variable, which is exactly right for direct connectOrSpawnRuntimeHost. But connectOwnedRuntimeHostWithDependencies() wraps the call in a broad catch that converts anything into { kind: 'failed', reason: 'host_unresponsive' }. So MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS=abc reaches the user as Runtime Host did not start: host_unresponsive — they're told the host is unresponsive when in fact they made a typo, and the message gives them nothing to act on.

This is the same shape as the finding above: the mechanism is fine, the diagnosis points the wrong way. Config errors specifically are worth letting through that catch (or resolving before it, as a typed configuration failure), since they're the one class the user can fix immediately once they know the variable name. Worth a test on both the owned and hosted paths.

(dependencies.env ?? process.env)[ELECTION_DEADLINE_MS_ENV_VAR],
) ??
DEFAULT_ELECTION_DEADLINE_MS;
if (!Number.isSafeInteger(deadlineMs) || deadlineMs <= 0 || deadlineMs > 120_000) {
throw new RangeError('electionDeadlineMs must be an integer between 1 and 120000');
}
Expand Down
8 changes: 6 additions & 2 deletions packages/runtime-host/src/client/startup-error.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,8 +47,12 @@ export function runtimeHostStartupError(reason: RuntimeHostStartupFailureReason)
'This workspace belongs to a different Runtime Host composition. Diagnostic code: COMPOSITION_MISMATCH.',
);
case 'startup_timeout':
return new Error('Runtime Host did not become ready before the startup deadline');
return new Error(
'No Runtime Host became ready before the startup deadline elapsed. Retry; if this workspace needs longer to open (large workspaces can after an upgrade), set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
case 'host_unresponsive':
return new Error('Runtime Host stopped responding during startup');
return new Error(
'A Runtime Host was found but did not become ready before the startup deadline elapsed. It may still be opening this workspace (large workspaces can need longer right after an upgrade); retrying once it settles usually succeeds, or set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
}
}
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
fix(runtime-host): let operators raise the election deadline and clarify timeout copy by me2seeks · Pull Request #3480 · apache/maka · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -523,11 +523,11 @@ test('keeps Local explicitly usable without routing default work away from an un
},
);

await assert.rejects(manager.enable(remoteTarget('offline')), /stopped responding/);
await assert.rejects(manager.enable(remoteTarget('offline')), /did not become ready/);
manager.setDefaultProfile('offline');
await assert.rejects(
manager.handleBotIncomingMessage({ text: 'default' } as BotIncomingMessage),
/stopped responding/,
/did not become ready/,
);

assert.equal(local.botMessages, 0);
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
ELECTION_DEADLINE_MS_ENV_VAR,
connectOrSpawnRuntimeHostWithDependencies,
electionDeadlineMsFromEnvironment,
} from '../client/connect-or-spawn.js';
import {
INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
RUNTIME_HOST_PROTOCOL_VERSION,
} from '../protocol/index.js';

test('treats an unset or blank override as unconfigured', () => {
assert.equal(electionDeadlineMsFromEnvironment(undefined), undefined);
assert.equal(electionDeadlineMsFromEnvironment(''), undefined);
assert.equal(electionDeadlineMsFromEnvironment(' '), undefined);
});

test('parses a valid millisecond override', () => {
assert.equal(electionDeadlineMsFromEnvironment('90000'), 90_000);
assert.equal(electionDeadlineMsFromEnvironment(' 5000 '), 5_000);
});

test('fails closed on an invalid override instead of silently ignoring it', () => {
for (const invalid of ['abc', '0', '-100', '120001', '1.5']) {
assert.throws(() => electionDeadlineMsFromEnvironment(invalid), RangeError);
}
assert.throws(
() => electionDeadlineMsFromEnvironment('abc'),
new RegExp(`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer`, 'u'),
);
});

test('an invalid environment override fails the election before touching storage', async () => {
await assert.rejects(
connectOrSpawnRuntimeHostWithDependencies(
{
rootPath: '/nonexistent-maka-3474-root',
protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION },
compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
candidateEntrypoint: 'candidate-entry.js',
},
{
launchCandidate: () => ({ spawned: Promise.reject(new Error('must not spawn')) }),
random: Math.random,
env: { [ELECTION_DEADLINE_MS_ENV_VAR]: 'not-a-number' },
},
),
(error: unknown) =>
error instanceof RangeError && /MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u.test(error.message),
);
});
13 changes: 12 additions & 1 deletion packages/runtime-host/src/__tests__/startup-error.test.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,18 @@ test('presents migration blockers with a permanent previous-release recovery pat
test('keeps an unresponsive Host retryable', () => {
const error = runtimeHostStartupError('host_unresponsive');
assert.equal(error instanceof RuntimeHostPermanentReconnectError, false);
assert.match(error.message, /stopped responding/u);
assert.match(error.message, /did not become ready before the startup deadline/u);
});

test('tells both timeout reasons how to widen the election window', () => {
assert.match(
runtimeHostStartupError('host_unresponsive').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
assert.match(
runtimeHostStartupError('startup_timeout').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
});

test('keeps internal startup failures retryable', () => {
Expand Down
30 changes: 29 additions & 1 deletion packages/runtime-host/src/client/connect-or-spawn.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@ const DEFAULT_ELECTION_DEADLINE_MS = 45_000;
const DEFAULT_BACKOFF_MIN_MS = 20;
const DEFAULT_BACKOFF_MAX_MS = 250;
const MIN_CANDIDATE_INTERVAL_MS = 250;
export const ELECTION_DEADLINE_MS_ENV_VAR = 'MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS';

export interface ConnectOrSpawnRuntimeHostInput {
rootPath: string;
Expand All@@ -56,13 +57,35 @@ export interface ConnectOrSpawnRuntimeHostInput {
interface ConnectOrSpawnRuntimeHostDependencies {
launchCandidate: CandidateLauncher;
random(): number;
/** Defaults to `process.env`; injected so tests never mutate the real environment. */
env?: NodeJS.ProcessEnv;
}

const defaultDependencies: ConnectOrSpawnRuntimeHostDependencies = {
launchCandidate: launchDetachedRuntimeHostCandidate,
random: Math.random,
};

/**
* Resolves the operator override for the client election deadline. Large
* workspaces can legitimately take longer than the default window on their
* first start after an upgrade, so the deadline must be raisable without a
* code change. Invalid values fail closed: a silently ignored typo would leave
* the operator believing they widened the window when they did not.
*/
export function electionDeadlineMsFromEnvironment(
rawValue: string | undefined,
): number | undefined {
if (rawValue === undefined || rawValue.trim() === '') return undefined;
const parsed = Number(rawValue);
if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 120_000) {
throw new RangeError(
`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer between 1 and 120000 milliseconds`,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] The error text points every startup failure at this variable, including the class it can't fix.

There's a second failure shape this knob doesn't address. From a real CI run on #3454, which now emits election diagnostics:

{"deadlineMs":45000,"elapsedMs":45174,"candidateLaunches":69,
"observations":{"notRegistered":73,"connectFailed":0,"handshakeFailed":0,
"connected":1,"readyWaitFailed":1}}

69 candidate launches in 45 seconds. Zero connect failures, zero handshake failures — it connected once, then ready-wait failed, and candidates kept being launched roughly every 0.65s until the deadline. It missed by 174ms, so raising the deadline would have made that particular run pass, but the run isn't slow — it isn't converging. At 90s it launches ~138 candidates instead of ~69 and passing becomes a matter of luck.

That's different from #3474's genuinely-slow start, where waiting longer is exactly right. The problem is that both produce the same message, so an operator hitting the retry-storm shape is told to raise the timeout, raises it, fails again, and raises it further.

Not blocking, and possibly not yours to fix here — but worth asking: with candidateLaunches high and connectFailed/handshakeFailed at zero, should the message distinguish "this start is slow, wait longer" from "candidates aren't converging, the timeout isn't your problem"? You have the #3474 case in hand and probably know which shape operators hit more often.

);
}
return parsed;
}

export type ConnectOrSpawnRuntimeHostResult =
| {
kind: 'connected';
Expand DownExpand Up@@ -173,7 +196,12 @@ export async function connectOrSpawnRuntimeHostWithDependencies(
input: ConnectOrSpawnRuntimeHostInput,
dependencies: ConnectOrSpawnRuntimeHostDependencies,
): Promise<ConnectOrSpawnRuntimeHostResult> {
const deadlineMs = input.electionDeadlineMs ?? DEFAULT_ELECTION_DEADLINE_MS;
const deadlineMs =
input.electionDeadlineMs ??
electionDeadlineMsFromEnvironment(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] On the owned-launch path an invalid value loses the variable name and surfaces as host_unresponsive.

electionDeadlineMsFromEnvironment() throws a RangeError naming the variable, which is exactly right for direct connectOrSpawnRuntimeHost. But connectOwnedRuntimeHostWithDependencies() wraps the call in a broad catch that converts anything into { kind: 'failed', reason: 'host_unresponsive' }. So MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS=abc reaches the user as Runtime Host did not start: host_unresponsive — they're told the host is unresponsive when in fact they made a typo, and the message gives them nothing to act on.

This is the same shape as the finding above: the mechanism is fine, the diagnosis points the wrong way. Config errors specifically are worth letting through that catch (or resolving before it, as a typed configuration failure), since they're the one class the user can fix immediately once they know the variable name. Worth a test on both the owned and hosted paths.

(dependencies.env ?? process.env)[ELECTION_DEADLINE_MS_ENV_VAR],
) ??
DEFAULT_ELECTION_DEADLINE_MS;
if (!Number.isSafeInteger(deadlineMs) || deadlineMs <= 0 || deadlineMs > 120_000) {
throw new RangeError('electionDeadlineMs must be an integer between 1 and 120000');
}
Expand Down
8 changes: 6 additions & 2 deletions packages/runtime-host/src/client/startup-error.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,8 +47,12 @@ export function runtimeHostStartupError(reason: RuntimeHostStartupFailureReason)
'This workspace belongs to a different Runtime Host composition. Diagnostic code: COMPOSITION_MISMATCH.',
);
case 'startup_timeout':
return new Error('Runtime Host did not become ready before the startup deadline');
return new Error(
'No Runtime Host became ready before the startup deadline elapsed. Retry; if this workspace needs longer to open (large workspaces can after an upgrade), set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
case 'host_unresponsive':
return new Error('Runtime Host stopped responding during startup');
return new Error(
'A Runtime Host was found but did not become ready before the startup deadline elapsed. It may still be opening this workspace (large workspaces can need longer right after an upgrade); retrying once it settles usually succeeds, or set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
}
}
, 'i'); if (__m === '*' || __re.test(location.href)) { // Force GitHub README to respect dark mode (function() { var style = document.createElement('style'); style.textContent = ' .markdown-body { color-scheme: dark light; } .markdown-body pre { background: #161b22 !important; } .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; } .markdown-body table th, .markdown-body table td { border-color: #30363d !important; } .markdown-body img { background: #0d1117; } .markdown-body blockquote { border-left-color: #8b949e; } .markdown-body hr { border-color: #30363d; } '; document.head.appendChild(style); })(); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' fix(runtime-host): let operators raise the election deadline and clarify timeout copy by me2seeks · Pull Request #3480 · apache/maka · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -523,11 +523,11 @@ test('keeps Local explicitly usable without routing default work away from an un
},
);

await assert.rejects(manager.enable(remoteTarget('offline')), /stopped responding/);
await assert.rejects(manager.enable(remoteTarget('offline')), /did not become ready/);
manager.setDefaultProfile('offline');
await assert.rejects(
manager.handleBotIncomingMessage({ text: 'default' } as BotIncomingMessage),
/stopped responding/,
/did not become ready/,
);

assert.equal(local.botMessages, 0);
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
ELECTION_DEADLINE_MS_ENV_VAR,
connectOrSpawnRuntimeHostWithDependencies,
electionDeadlineMsFromEnvironment,
} from '../client/connect-or-spawn.js';
import {
INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
RUNTIME_HOST_PROTOCOL_VERSION,
} from '../protocol/index.js';

test('treats an unset or blank override as unconfigured', () => {
assert.equal(electionDeadlineMsFromEnvironment(undefined), undefined);
assert.equal(electionDeadlineMsFromEnvironment(''), undefined);
assert.equal(electionDeadlineMsFromEnvironment(' '), undefined);
});

test('parses a valid millisecond override', () => {
assert.equal(electionDeadlineMsFromEnvironment('90000'), 90_000);
assert.equal(electionDeadlineMsFromEnvironment(' 5000 '), 5_000);
});

test('fails closed on an invalid override instead of silently ignoring it', () => {
for (const invalid of ['abc', '0', '-100', '120001', '1.5']) {
assert.throws(() => electionDeadlineMsFromEnvironment(invalid), RangeError);
}
assert.throws(
() => electionDeadlineMsFromEnvironment('abc'),
new RegExp(`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer`, 'u'),
);
});

test('an invalid environment override fails the election before touching storage', async () => {
await assert.rejects(
connectOrSpawnRuntimeHostWithDependencies(
{
rootPath: '/nonexistent-maka-3474-root',
protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION },
compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
candidateEntrypoint: 'candidate-entry.js',
},
{
launchCandidate: () => ({ spawned: Promise.reject(new Error('must not spawn')) }),
random: Math.random,
env: { [ELECTION_DEADLINE_MS_ENV_VAR]: 'not-a-number' },
},
),
(error: unknown) =>
error instanceof RangeError && /MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u.test(error.message),
);
});
13 changes: 12 additions & 1 deletion packages/runtime-host/src/__tests__/startup-error.test.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,18 @@ test('presents migration blockers with a permanent previous-release recovery pat
test('keeps an unresponsive Host retryable', () => {
const error = runtimeHostStartupError('host_unresponsive');
assert.equal(error instanceof RuntimeHostPermanentReconnectError, false);
assert.match(error.message, /stopped responding/u);
assert.match(error.message, /did not become ready before the startup deadline/u);
});

test('tells both timeout reasons how to widen the election window', () => {
assert.match(
runtimeHostStartupError('host_unresponsive').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
assert.match(
runtimeHostStartupError('startup_timeout').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
});

test('keeps internal startup failures retryable', () => {
Expand Down
30 changes: 29 additions & 1 deletion packages/runtime-host/src/client/connect-or-spawn.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@ const DEFAULT_ELECTION_DEADLINE_MS = 45_000;
const DEFAULT_BACKOFF_MIN_MS = 20;
const DEFAULT_BACKOFF_MAX_MS = 250;
const MIN_CANDIDATE_INTERVAL_MS = 250;
export const ELECTION_DEADLINE_MS_ENV_VAR = 'MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS';

export interface ConnectOrSpawnRuntimeHostInput {
rootPath: string;
Expand All@@ -56,13 +57,35 @@ export interface ConnectOrSpawnRuntimeHostInput {
interface ConnectOrSpawnRuntimeHostDependencies {
launchCandidate: CandidateLauncher;
random(): number;
/** Defaults to `process.env`; injected so tests never mutate the real environment. */
env?: NodeJS.ProcessEnv;
}

const defaultDependencies: ConnectOrSpawnRuntimeHostDependencies = {
launchCandidate: launchDetachedRuntimeHostCandidate,
random: Math.random,
};

/**
* Resolves the operator override for the client election deadline. Large
* workspaces can legitimately take longer than the default window on their
* first start after an upgrade, so the deadline must be raisable without a
* code change. Invalid values fail closed: a silently ignored typo would leave
* the operator believing they widened the window when they did not.
*/
export function electionDeadlineMsFromEnvironment(
rawValue: string | undefined,
): number | undefined {
if (rawValue === undefined || rawValue.trim() === '') return undefined;
const parsed = Number(rawValue);
if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 120_000) {
throw new RangeError(
`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer between 1 and 120000 milliseconds`,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] The error text points every startup failure at this variable, including the class it can't fix.

There's a second failure shape this knob doesn't address. From a real CI run on #3454, which now emits election diagnostics:

{"deadlineMs":45000,"elapsedMs":45174,"candidateLaunches":69,
"observations":{"notRegistered":73,"connectFailed":0,"handshakeFailed":0,
"connected":1,"readyWaitFailed":1}}

69 candidate launches in 45 seconds. Zero connect failures, zero handshake failures — it connected once, then ready-wait failed, and candidates kept being launched roughly every 0.65s until the deadline. It missed by 174ms, so raising the deadline would have made that particular run pass, but the run isn't slow — it isn't converging. At 90s it launches ~138 candidates instead of ~69 and passing becomes a matter of luck.

That's different from #3474's genuinely-slow start, where waiting longer is exactly right. The problem is that both produce the same message, so an operator hitting the retry-storm shape is told to raise the timeout, raises it, fails again, and raises it further.

Not blocking, and possibly not yours to fix here — but worth asking: with candidateLaunches high and connectFailed/handshakeFailed at zero, should the message distinguish "this start is slow, wait longer" from "candidates aren't converging, the timeout isn't your problem"? You have the #3474 case in hand and probably know which shape operators hit more often.

);
}
return parsed;
}

export type ConnectOrSpawnRuntimeHostResult =
| {
kind: 'connected';
Expand DownExpand Up@@ -173,7 +196,12 @@ export async function connectOrSpawnRuntimeHostWithDependencies(
input: ConnectOrSpawnRuntimeHostInput,
dependencies: ConnectOrSpawnRuntimeHostDependencies,
): Promise<ConnectOrSpawnRuntimeHostResult> {
const deadlineMs = input.electionDeadlineMs ?? DEFAULT_ELECTION_DEADLINE_MS;
const deadlineMs =
input.electionDeadlineMs ??
electionDeadlineMsFromEnvironment(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] On the owned-launch path an invalid value loses the variable name and surfaces as host_unresponsive.

electionDeadlineMsFromEnvironment() throws a RangeError naming the variable, which is exactly right for direct connectOrSpawnRuntimeHost. But connectOwnedRuntimeHostWithDependencies() wraps the call in a broad catch that converts anything into { kind: 'failed', reason: 'host_unresponsive' }. So MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS=abc reaches the user as Runtime Host did not start: host_unresponsive — they're told the host is unresponsive when in fact they made a typo, and the message gives them nothing to act on.

This is the same shape as the finding above: the mechanism is fine, the diagnosis points the wrong way. Config errors specifically are worth letting through that catch (or resolving before it, as a typed configuration failure), since they're the one class the user can fix immediately once they know the variable name. Worth a test on both the owned and hosted paths.

(dependencies.env ?? process.env)[ELECTION_DEADLINE_MS_ENV_VAR],
) ??
DEFAULT_ELECTION_DEADLINE_MS;
if (!Number.isSafeInteger(deadlineMs) || deadlineMs <= 0 || deadlineMs > 120_000) {
throw new RangeError('electionDeadlineMs must be an integer between 1 and 120000');
}
Expand Down
8 changes: 6 additions & 2 deletions packages/runtime-host/src/client/startup-error.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,8 +47,12 @@ export function runtimeHostStartupError(reason: RuntimeHostStartupFailureReason)
'This workspace belongs to a different Runtime Host composition. Diagnostic code: COMPOSITION_MISMATCH.',
);
case 'startup_timeout':
return new Error('Runtime Host did not become ready before the startup deadline');
return new Error(
'No Runtime Host became ready before the startup deadline elapsed. Retry; if this workspace needs longer to open (large workspaces can after an upgrade), set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
case 'host_unresponsive':
return new Error('Runtime Host stopped responding during startup');
return new Error(
'A Runtime Host was found but did not become ready before the startup deadline elapsed. It may still be opening this workspace (large workspaces can need longer right after an upgrade); retrying once it settles usually succeeds, or set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
}
}
, 'i'); if (__m === '*' || __re.test(location.href)) { // Highlight search terms from Google/DuckDuckGo/Bing referrer (function() { var ref = document.referrer; var terms = []; if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) { var url = new URL(ref); var q = url.searchParams.get('q') || url.searchParams.get('p'); if (q) { terms = q.split(/\s+/).filter(function(t) { return t.length > 2; }); } } if (terms.length === 0) return; var style = document.createElement('style'); style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }'; document.head.appendChild(style); function highlight(node) { if (node.nodeType === 3) { // text node var text = node.textContent; var found = false; terms.forEach(function(term) { var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' fix(runtime-host): let operators raise the election deadline and clarify timeout copy by me2seeks · Pull Request #3480 · apache/maka · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -523,11 +523,11 @@ test('keeps Local explicitly usable without routing default work away from an un
},
);

await assert.rejects(manager.enable(remoteTarget('offline')), /stopped responding/);
await assert.rejects(manager.enable(remoteTarget('offline')), /did not become ready/);
manager.setDefaultProfile('offline');
await assert.rejects(
manager.handleBotIncomingMessage({ text: 'default' } as BotIncomingMessage),
/stopped responding/,
/did not become ready/,
);

assert.equal(local.botMessages, 0);
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
ELECTION_DEADLINE_MS_ENV_VAR,
connectOrSpawnRuntimeHostWithDependencies,
electionDeadlineMsFromEnvironment,
} from '../client/connect-or-spawn.js';
import {
INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
RUNTIME_HOST_PROTOCOL_VERSION,
} from '../protocol/index.js';

test('treats an unset or blank override as unconfigured', () => {
assert.equal(electionDeadlineMsFromEnvironment(undefined), undefined);
assert.equal(electionDeadlineMsFromEnvironment(''), undefined);
assert.equal(electionDeadlineMsFromEnvironment(' '), undefined);
});

test('parses a valid millisecond override', () => {
assert.equal(electionDeadlineMsFromEnvironment('90000'), 90_000);
assert.equal(electionDeadlineMsFromEnvironment(' 5000 '), 5_000);
});

test('fails closed on an invalid override instead of silently ignoring it', () => {
for (const invalid of ['abc', '0', '-100', '120001', '1.5']) {
assert.throws(() => electionDeadlineMsFromEnvironment(invalid), RangeError);
}
assert.throws(
() => electionDeadlineMsFromEnvironment('abc'),
new RegExp(`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer`, 'u'),
);
});

test('an invalid environment override fails the election before touching storage', async () => {
await assert.rejects(
connectOrSpawnRuntimeHostWithDependencies(
{
rootPath: '/nonexistent-maka-3474-root',
protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION },
compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
candidateEntrypoint: 'candidate-entry.js',
},
{
launchCandidate: () => ({ spawned: Promise.reject(new Error('must not spawn')) }),
random: Math.random,
env: { [ELECTION_DEADLINE_MS_ENV_VAR]: 'not-a-number' },
},
),
(error: unknown) =>
error instanceof RangeError && /MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u.test(error.message),
);
});
13 changes: 12 additions & 1 deletion packages/runtime-host/src/__tests__/startup-error.test.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,18 @@ test('presents migration blockers with a permanent previous-release recovery pat
test('keeps an unresponsive Host retryable', () => {
const error = runtimeHostStartupError('host_unresponsive');
assert.equal(error instanceof RuntimeHostPermanentReconnectError, false);
assert.match(error.message, /stopped responding/u);
assert.match(error.message, /did not become ready before the startup deadline/u);
});

test('tells both timeout reasons how to widen the election window', () => {
assert.match(
runtimeHostStartupError('host_unresponsive').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
assert.match(
runtimeHostStartupError('startup_timeout').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
});

test('keeps internal startup failures retryable', () => {
Expand Down
30 changes: 29 additions & 1 deletion packages/runtime-host/src/client/connect-or-spawn.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@ const DEFAULT_ELECTION_DEADLINE_MS = 45_000;
const DEFAULT_BACKOFF_MIN_MS = 20;
const DEFAULT_BACKOFF_MAX_MS = 250;
const MIN_CANDIDATE_INTERVAL_MS = 250;
export const ELECTION_DEADLINE_MS_ENV_VAR = 'MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS';

export interface ConnectOrSpawnRuntimeHostInput {
rootPath: string;
Expand All@@ -56,13 +57,35 @@ export interface ConnectOrSpawnRuntimeHostInput {
interface ConnectOrSpawnRuntimeHostDependencies {
launchCandidate: CandidateLauncher;
random(): number;
/** Defaults to `process.env`; injected so tests never mutate the real environment. */
env?: NodeJS.ProcessEnv;
}

const defaultDependencies: ConnectOrSpawnRuntimeHostDependencies = {
launchCandidate: launchDetachedRuntimeHostCandidate,
random: Math.random,
};

/**
* Resolves the operator override for the client election deadline. Large
* workspaces can legitimately take longer than the default window on their
* first start after an upgrade, so the deadline must be raisable without a
* code change. Invalid values fail closed: a silently ignored typo would leave
* the operator believing they widened the window when they did not.
*/
export function electionDeadlineMsFromEnvironment(
rawValue: string | undefined,
): number | undefined {
if (rawValue === undefined || rawValue.trim() === '') return undefined;
const parsed = Number(rawValue);
if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 120_000) {
throw new RangeError(
`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer between 1 and 120000 milliseconds`,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] The error text points every startup failure at this variable, including the class it can't fix.

There's a second failure shape this knob doesn't address. From a real CI run on #3454, which now emits election diagnostics:

{"deadlineMs":45000,"elapsedMs":45174,"candidateLaunches":69,
"observations":{"notRegistered":73,"connectFailed":0,"handshakeFailed":0,
"connected":1,"readyWaitFailed":1}}

69 candidate launches in 45 seconds. Zero connect failures, zero handshake failures — it connected once, then ready-wait failed, and candidates kept being launched roughly every 0.65s until the deadline. It missed by 174ms, so raising the deadline would have made that particular run pass, but the run isn't slow — it isn't converging. At 90s it launches ~138 candidates instead of ~69 and passing becomes a matter of luck.

That's different from #3474's genuinely-slow start, where waiting longer is exactly right. The problem is that both produce the same message, so an operator hitting the retry-storm shape is told to raise the timeout, raises it, fails again, and raises it further.

Not blocking, and possibly not yours to fix here — but worth asking: with candidateLaunches high and connectFailed/handshakeFailed at zero, should the message distinguish "this start is slow, wait longer" from "candidates aren't converging, the timeout isn't your problem"? You have the #3474 case in hand and probably know which shape operators hit more often.

);
}
return parsed;
}

export type ConnectOrSpawnRuntimeHostResult =
| {
kind: 'connected';
Expand DownExpand Up@@ -173,7 +196,12 @@ export async function connectOrSpawnRuntimeHostWithDependencies(
input: ConnectOrSpawnRuntimeHostInput,
dependencies: ConnectOrSpawnRuntimeHostDependencies,
): Promise<ConnectOrSpawnRuntimeHostResult> {
const deadlineMs = input.electionDeadlineMs ?? DEFAULT_ELECTION_DEADLINE_MS;
const deadlineMs =
input.electionDeadlineMs ??
electionDeadlineMsFromEnvironment(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] On the owned-launch path an invalid value loses the variable name and surfaces as host_unresponsive.

electionDeadlineMsFromEnvironment() throws a RangeError naming the variable, which is exactly right for direct connectOrSpawnRuntimeHost. But connectOwnedRuntimeHostWithDependencies() wraps the call in a broad catch that converts anything into { kind: 'failed', reason: 'host_unresponsive' }. So MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS=abc reaches the user as Runtime Host did not start: host_unresponsive — they're told the host is unresponsive when in fact they made a typo, and the message gives them nothing to act on.

This is the same shape as the finding above: the mechanism is fine, the diagnosis points the wrong way. Config errors specifically are worth letting through that catch (or resolving before it, as a typed configuration failure), since they're the one class the user can fix immediately once they know the variable name. Worth a test on both the owned and hosted paths.

(dependencies.env ?? process.env)[ELECTION_DEADLINE_MS_ENV_VAR],
) ??
DEFAULT_ELECTION_DEADLINE_MS;
if (!Number.isSafeInteger(deadlineMs) || deadlineMs <= 0 || deadlineMs > 120_000) {
throw new RangeError('electionDeadlineMs must be an integer between 1 and 120000');
}
Expand Down
8 changes: 6 additions & 2 deletions packages/runtime-host/src/client/startup-error.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,8 +47,12 @@ export function runtimeHostStartupError(reason: RuntimeHostStartupFailureReason)
'This workspace belongs to a different Runtime Host composition. Diagnostic code: COMPOSITION_MISMATCH.',
);
case 'startup_timeout':
return new Error('Runtime Host did not become ready before the startup deadline');
return new Error(
'No Runtime Host became ready before the startup deadline elapsed. Retry; if this workspace needs longer to open (large workspaces can after an upgrade), set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
case 'host_unresponsive':
return new Error('Runtime Host stopped responding during startup');
return new Error(
'A Runtime Host was found but did not become ready before the startup deadline elapsed. It may still be opening this workspace (large workspaces can need longer right after an upgrade); retrying once it settles usually succeeds, or set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
}
}
, 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ' fix(runtime-host): let operators raise the election deadline and clarify timeout copy by me2seeks · Pull Request #3480 · apache/maka · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -523,11 +523,11 @@ test('keeps Local explicitly usable without routing default work away from an un
},
);

await assert.rejects(manager.enable(remoteTarget('offline')), /stopped responding/);
await assert.rejects(manager.enable(remoteTarget('offline')), /did not become ready/);
manager.setDefaultProfile('offline');
await assert.rejects(
manager.handleBotIncomingMessage({ text: 'default' } as BotIncomingMessage),
/stopped responding/,
/did not become ready/,
);

assert.equal(local.botMessages, 0);
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
ELECTION_DEADLINE_MS_ENV_VAR,
connectOrSpawnRuntimeHostWithDependencies,
electionDeadlineMsFromEnvironment,
} from '../client/connect-or-spawn.js';
import {
INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
RUNTIME_HOST_PROTOCOL_VERSION,
} from '../protocol/index.js';

test('treats an unset or blank override as unconfigured', () => {
assert.equal(electionDeadlineMsFromEnvironment(undefined), undefined);
assert.equal(electionDeadlineMsFromEnvironment(''), undefined);
assert.equal(electionDeadlineMsFromEnvironment(' '), undefined);
});

test('parses a valid millisecond override', () => {
assert.equal(electionDeadlineMsFromEnvironment('90000'), 90_000);
assert.equal(electionDeadlineMsFromEnvironment(' 5000 '), 5_000);
});

test('fails closed on an invalid override instead of silently ignoring it', () => {
for (const invalid of ['abc', '0', '-100', '120001', '1.5']) {
assert.throws(() => electionDeadlineMsFromEnvironment(invalid), RangeError);
}
assert.throws(
() => electionDeadlineMsFromEnvironment('abc'),
new RegExp(`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer`, 'u'),
);
});

test('an invalid environment override fails the election before touching storage', async () => {
await assert.rejects(
connectOrSpawnRuntimeHostWithDependencies(
{
rootPath: '/nonexistent-maka-3474-root',
protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION },
compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
candidateEntrypoint: 'candidate-entry.js',
},
{
launchCandidate: () => ({ spawned: Promise.reject(new Error('must not spawn')) }),
random: Math.random,
env: { [ELECTION_DEADLINE_MS_ENV_VAR]: 'not-a-number' },
},
),
(error: unknown) =>
error instanceof RangeError && /MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u.test(error.message),
);
});
13 changes: 12 additions & 1 deletion packages/runtime-host/src/__tests__/startup-error.test.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,18 @@ test('presents migration blockers with a permanent previous-release recovery pat
test('keeps an unresponsive Host retryable', () => {
const error = runtimeHostStartupError('host_unresponsive');
assert.equal(error instanceof RuntimeHostPermanentReconnectError, false);
assert.match(error.message, /stopped responding/u);
assert.match(error.message, /did not become ready before the startup deadline/u);
});

test('tells both timeout reasons how to widen the election window', () => {
assert.match(
runtimeHostStartupError('host_unresponsive').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
assert.match(
runtimeHostStartupError('startup_timeout').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
});

test('keeps internal startup failures retryable', () => {
Expand Down
30 changes: 29 additions & 1 deletion packages/runtime-host/src/client/connect-or-spawn.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@ const DEFAULT_ELECTION_DEADLINE_MS = 45_000;
const DEFAULT_BACKOFF_MIN_MS = 20;
const DEFAULT_BACKOFF_MAX_MS = 250;
const MIN_CANDIDATE_INTERVAL_MS = 250;
export const ELECTION_DEADLINE_MS_ENV_VAR = 'MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS';

export interface ConnectOrSpawnRuntimeHostInput {
rootPath: string;
Expand All@@ -56,13 +57,35 @@ export interface ConnectOrSpawnRuntimeHostInput {
interface ConnectOrSpawnRuntimeHostDependencies {
launchCandidate: CandidateLauncher;
random(): number;
/** Defaults to `process.env`; injected so tests never mutate the real environment. */
env?: NodeJS.ProcessEnv;
}

const defaultDependencies: ConnectOrSpawnRuntimeHostDependencies = {
launchCandidate: launchDetachedRuntimeHostCandidate,
random: Math.random,
};

/**
* Resolves the operator override for the client election deadline. Large
* workspaces can legitimately take longer than the default window on their
* first start after an upgrade, so the deadline must be raisable without a
* code change. Invalid values fail closed: a silently ignored typo would leave
* the operator believing they widened the window when they did not.
*/
export function electionDeadlineMsFromEnvironment(
rawValue: string | undefined,
): number | undefined {
if (rawValue === undefined || rawValue.trim() === '') return undefined;
const parsed = Number(rawValue);
if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 120_000) {
throw new RangeError(
`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer between 1 and 120000 milliseconds`,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] The error text points every startup failure at this variable, including the class it can't fix.

There's a second failure shape this knob doesn't address. From a real CI run on #3454, which now emits election diagnostics:

{"deadlineMs":45000,"elapsedMs":45174,"candidateLaunches":69,
"observations":{"notRegistered":73,"connectFailed":0,"handshakeFailed":0,
"connected":1,"readyWaitFailed":1}}

69 candidate launches in 45 seconds. Zero connect failures, zero handshake failures — it connected once, then ready-wait failed, and candidates kept being launched roughly every 0.65s until the deadline. It missed by 174ms, so raising the deadline would have made that particular run pass, but the run isn't slow — it isn't converging. At 90s it launches ~138 candidates instead of ~69 and passing becomes a matter of luck.

That's different from #3474's genuinely-slow start, where waiting longer is exactly right. The problem is that both produce the same message, so an operator hitting the retry-storm shape is told to raise the timeout, raises it, fails again, and raises it further.

Not blocking, and possibly not yours to fix here — but worth asking: with candidateLaunches high and connectFailed/handshakeFailed at zero, should the message distinguish "this start is slow, wait longer" from "candidates aren't converging, the timeout isn't your problem"? You have the #3474 case in hand and probably know which shape operators hit more often.

);
}
return parsed;
}

export type ConnectOrSpawnRuntimeHostResult =
| {
kind: 'connected';
Expand DownExpand Up@@ -173,7 +196,12 @@ export async function connectOrSpawnRuntimeHostWithDependencies(
input: ConnectOrSpawnRuntimeHostInput,
dependencies: ConnectOrSpawnRuntimeHostDependencies,
): Promise<ConnectOrSpawnRuntimeHostResult> {
const deadlineMs = input.electionDeadlineMs ?? DEFAULT_ELECTION_DEADLINE_MS;
const deadlineMs =
input.electionDeadlineMs ??
electionDeadlineMsFromEnvironment(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] On the owned-launch path an invalid value loses the variable name and surfaces as host_unresponsive.

electionDeadlineMsFromEnvironment() throws a RangeError naming the variable, which is exactly right for direct connectOrSpawnRuntimeHost. But connectOwnedRuntimeHostWithDependencies() wraps the call in a broad catch that converts anything into { kind: 'failed', reason: 'host_unresponsive' }. So MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS=abc reaches the user as Runtime Host did not start: host_unresponsive — they're told the host is unresponsive when in fact they made a typo, and the message gives them nothing to act on.

This is the same shape as the finding above: the mechanism is fine, the diagnosis points the wrong way. Config errors specifically are worth letting through that catch (or resolving before it, as a typed configuration failure), since they're the one class the user can fix immediately once they know the variable name. Worth a test on both the owned and hosted paths.

(dependencies.env ?? process.env)[ELECTION_DEADLINE_MS_ENV_VAR],
) ??
DEFAULT_ELECTION_DEADLINE_MS;
if (!Number.isSafeInteger(deadlineMs) || deadlineMs <= 0 || deadlineMs > 120_000) {
throw new RangeError('electionDeadlineMs must be an integer between 1 and 120000');
}
Expand Down
8 changes: 6 additions & 2 deletions packages/runtime-host/src/client/startup-error.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,8 +47,12 @@ export function runtimeHostStartupError(reason: RuntimeHostStartupFailureReason)
'This workspace belongs to a different Runtime Host composition. Diagnostic code: COMPOSITION_MISMATCH.',
);
case 'startup_timeout':
return new Error('Runtime Host did not become ready before the startup deadline');
return new Error(
'No Runtime Host became ready before the startup deadline elapsed. Retry; if this workspace needs longer to open (large workspaces can after an upgrade), set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
case 'host_unresponsive':
return new Error('Runtime Host stopped responding during startup');
return new Error(
'A Runtime Host was found but did not become ready before the startup deadline elapsed. It may still be opening this workspace (large workspaces can need longer right after an upgrade); retrying once it settles usually succeeds, or set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
}
}
, 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' fix(runtime-host): let operators raise the election deadline and clarify timeout copy by me2seeks · Pull Request #3480 · apache/maka · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -523,11 +523,11 @@ test('keeps Local explicitly usable without routing default work away from an un
},
);

await assert.rejects(manager.enable(remoteTarget('offline')), /stopped responding/);
await assert.rejects(manager.enable(remoteTarget('offline')), /did not become ready/);
manager.setDefaultProfile('offline');
await assert.rejects(
manager.handleBotIncomingMessage({ text: 'default' } as BotIncomingMessage),
/stopped responding/,
/did not become ready/,
);

assert.equal(local.botMessages, 0);
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
ELECTION_DEADLINE_MS_ENV_VAR,
connectOrSpawnRuntimeHostWithDependencies,
electionDeadlineMsFromEnvironment,
} from '../client/connect-or-spawn.js';
import {
INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
RUNTIME_HOST_PROTOCOL_VERSION,
} from '../protocol/index.js';

test('treats an unset or blank override as unconfigured', () => {
assert.equal(electionDeadlineMsFromEnvironment(undefined), undefined);
assert.equal(electionDeadlineMsFromEnvironment(''), undefined);
assert.equal(electionDeadlineMsFromEnvironment(' '), undefined);
});

test('parses a valid millisecond override', () => {
assert.equal(electionDeadlineMsFromEnvironment('90000'), 90_000);
assert.equal(electionDeadlineMsFromEnvironment(' 5000 '), 5_000);
});

test('fails closed on an invalid override instead of silently ignoring it', () => {
for (const invalid of ['abc', '0', '-100', '120001', '1.5']) {
assert.throws(() => electionDeadlineMsFromEnvironment(invalid), RangeError);
}
assert.throws(
() => electionDeadlineMsFromEnvironment('abc'),
new RegExp(`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer`, 'u'),
);
});

test('an invalid environment override fails the election before touching storage', async () => {
await assert.rejects(
connectOrSpawnRuntimeHostWithDependencies(
{
rootPath: '/nonexistent-maka-3474-root',
protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION },
compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
candidateEntrypoint: 'candidate-entry.js',
},
{
launchCandidate: () => ({ spawned: Promise.reject(new Error('must not spawn')) }),
random: Math.random,
env: { [ELECTION_DEADLINE_MS_ENV_VAR]: 'not-a-number' },
},
),
(error: unknown) =>
error instanceof RangeError && /MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u.test(error.message),
);
});
13 changes: 12 additions & 1 deletion packages/runtime-host/src/__tests__/startup-error.test.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,18 @@ test('presents migration blockers with a permanent previous-release recovery pat
test('keeps an unresponsive Host retryable', () => {
const error = runtimeHostStartupError('host_unresponsive');
assert.equal(error instanceof RuntimeHostPermanentReconnectError, false);
assert.match(error.message, /stopped responding/u);
assert.match(error.message, /did not become ready before the startup deadline/u);
});

test('tells both timeout reasons how to widen the election window', () => {
assert.match(
runtimeHostStartupError('host_unresponsive').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
assert.match(
runtimeHostStartupError('startup_timeout').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
});

test('keeps internal startup failures retryable', () => {
Expand Down
30 changes: 29 additions & 1 deletion packages/runtime-host/src/client/connect-or-spawn.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@ const DEFAULT_ELECTION_DEADLINE_MS = 45_000;
const DEFAULT_BACKOFF_MIN_MS = 20;
const DEFAULT_BACKOFF_MAX_MS = 250;
const MIN_CANDIDATE_INTERVAL_MS = 250;
export const ELECTION_DEADLINE_MS_ENV_VAR = 'MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS';

export interface ConnectOrSpawnRuntimeHostInput {
rootPath: string;
Expand All@@ -56,13 +57,35 @@ export interface ConnectOrSpawnRuntimeHostInput {
interface ConnectOrSpawnRuntimeHostDependencies {
launchCandidate: CandidateLauncher;
random(): number;
/** Defaults to `process.env`; injected so tests never mutate the real environment. */
env?: NodeJS.ProcessEnv;
}

const defaultDependencies: ConnectOrSpawnRuntimeHostDependencies = {
launchCandidate: launchDetachedRuntimeHostCandidate,
random: Math.random,
};

/**
* Resolves the operator override for the client election deadline. Large
* workspaces can legitimately take longer than the default window on their
* first start after an upgrade, so the deadline must be raisable without a
* code change. Invalid values fail closed: a silently ignored typo would leave
* the operator believing they widened the window when they did not.
*/
export function electionDeadlineMsFromEnvironment(
rawValue: string | undefined,
): number | undefined {
if (rawValue === undefined || rawValue.trim() === '') return undefined;
const parsed = Number(rawValue);
if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 120_000) {
throw new RangeError(
`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer between 1 and 120000 milliseconds`,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] The error text points every startup failure at this variable, including the class it can't fix.

There's a second failure shape this knob doesn't address. From a real CI run on #3454, which now emits election diagnostics:

{"deadlineMs":45000,"elapsedMs":45174,"candidateLaunches":69,
"observations":{"notRegistered":73,"connectFailed":0,"handshakeFailed":0,
"connected":1,"readyWaitFailed":1}}

69 candidate launches in 45 seconds. Zero connect failures, zero handshake failures — it connected once, then ready-wait failed, and candidates kept being launched roughly every 0.65s until the deadline. It missed by 174ms, so raising the deadline would have made that particular run pass, but the run isn't slow — it isn't converging. At 90s it launches ~138 candidates instead of ~69 and passing becomes a matter of luck.

That's different from #3474's genuinely-slow start, where waiting longer is exactly right. The problem is that both produce the same message, so an operator hitting the retry-storm shape is told to raise the timeout, raises it, fails again, and raises it further.

Not blocking, and possibly not yours to fix here — but worth asking: with candidateLaunches high and connectFailed/handshakeFailed at zero, should the message distinguish "this start is slow, wait longer" from "candidates aren't converging, the timeout isn't your problem"? You have the #3474 case in hand and probably know which shape operators hit more often.

);
}
return parsed;
}

export type ConnectOrSpawnRuntimeHostResult =
| {
kind: 'connected';
Expand DownExpand Up@@ -173,7 +196,12 @@ export async function connectOrSpawnRuntimeHostWithDependencies(
input: ConnectOrSpawnRuntimeHostInput,
dependencies: ConnectOrSpawnRuntimeHostDependencies,
): Promise<ConnectOrSpawnRuntimeHostResult> {
const deadlineMs = input.electionDeadlineMs ?? DEFAULT_ELECTION_DEADLINE_MS;
const deadlineMs =
input.electionDeadlineMs ??
electionDeadlineMsFromEnvironment(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] On the owned-launch path an invalid value loses the variable name and surfaces as host_unresponsive.

electionDeadlineMsFromEnvironment() throws a RangeError naming the variable, which is exactly right for direct connectOrSpawnRuntimeHost. But connectOwnedRuntimeHostWithDependencies() wraps the call in a broad catch that converts anything into { kind: 'failed', reason: 'host_unresponsive' }. So MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS=abc reaches the user as Runtime Host did not start: host_unresponsive — they're told the host is unresponsive when in fact they made a typo, and the message gives them nothing to act on.

This is the same shape as the finding above: the mechanism is fine, the diagnosis points the wrong way. Config errors specifically are worth letting through that catch (or resolving before it, as a typed configuration failure), since they're the one class the user can fix immediately once they know the variable name. Worth a test on both the owned and hosted paths.

(dependencies.env ?? process.env)[ELECTION_DEADLINE_MS_ENV_VAR],
) ??
DEFAULT_ELECTION_DEADLINE_MS;
if (!Number.isSafeInteger(deadlineMs) || deadlineMs <= 0 || deadlineMs > 120_000) {
throw new RangeError('electionDeadlineMs must be an integer between 1 and 120000');
}
Expand Down
8 changes: 6 additions & 2 deletions packages/runtime-host/src/client/startup-error.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,8 +47,12 @@ export function runtimeHostStartupError(reason: RuntimeHostStartupFailureReason)
'This workspace belongs to a different Runtime Host composition. Diagnostic code: COMPOSITION_MISMATCH.',
);
case 'startup_timeout':
return new Error('Runtime Host did not become ready before the startup deadline');
return new Error(
'No Runtime Host became ready before the startup deadline elapsed. Retry; if this workspace needs longer to open (large workspaces can after an upgrade), set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
case 'host_unresponsive':
return new Error('Runtime Host stopped responding during startup');
return new Error(
'A Runtime Host was found but did not become ready before the startup deadline elapsed. It may still be opening this workspace (large workspaces can need longer right after an upgrade); retrying once it settles usually succeeds, or set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
}
}
, 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' fix(runtime-host): let operators raise the election deadline and clarify timeout copy by me2seeks · Pull Request #3480 · apache/maka · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -523,11 +523,11 @@ test('keeps Local explicitly usable without routing default work away from an un
},
);

await assert.rejects(manager.enable(remoteTarget('offline')), /stopped responding/);
await assert.rejects(manager.enable(remoteTarget('offline')), /did not become ready/);
manager.setDefaultProfile('offline');
await assert.rejects(
manager.handleBotIncomingMessage({ text: 'default' } as BotIncomingMessage),
/stopped responding/,
/did not become ready/,
);

assert.equal(local.botMessages, 0);
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
ELECTION_DEADLINE_MS_ENV_VAR,
connectOrSpawnRuntimeHostWithDependencies,
electionDeadlineMsFromEnvironment,
} from '../client/connect-or-spawn.js';
import {
INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
RUNTIME_HOST_PROTOCOL_VERSION,
} from '../protocol/index.js';

test('treats an unset or blank override as unconfigured', () => {
assert.equal(electionDeadlineMsFromEnvironment(undefined), undefined);
assert.equal(electionDeadlineMsFromEnvironment(''), undefined);
assert.equal(electionDeadlineMsFromEnvironment(' '), undefined);
});

test('parses a valid millisecond override', () => {
assert.equal(electionDeadlineMsFromEnvironment('90000'), 90_000);
assert.equal(electionDeadlineMsFromEnvironment(' 5000 '), 5_000);
});

test('fails closed on an invalid override instead of silently ignoring it', () => {
for (const invalid of ['abc', '0', '-100', '120001', '1.5']) {
assert.throws(() => electionDeadlineMsFromEnvironment(invalid), RangeError);
}
assert.throws(
() => electionDeadlineMsFromEnvironment('abc'),
new RegExp(`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer`, 'u'),
);
});

test('an invalid environment override fails the election before touching storage', async () => {
await assert.rejects(
connectOrSpawnRuntimeHostWithDependencies(
{
rootPath: '/nonexistent-maka-3474-root',
protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION },
compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
candidateEntrypoint: 'candidate-entry.js',
},
{
launchCandidate: () => ({ spawned: Promise.reject(new Error('must not spawn')) }),
random: Math.random,
env: { [ELECTION_DEADLINE_MS_ENV_VAR]: 'not-a-number' },
},
),
(error: unknown) =>
error instanceof RangeError && /MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u.test(error.message),
);
});
13 changes: 12 additions & 1 deletion packages/runtime-host/src/__tests__/startup-error.test.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,18 @@ test('presents migration blockers with a permanent previous-release recovery pat
test('keeps an unresponsive Host retryable', () => {
const error = runtimeHostStartupError('host_unresponsive');
assert.equal(error instanceof RuntimeHostPermanentReconnectError, false);
assert.match(error.message, /stopped responding/u);
assert.match(error.message, /did not become ready before the startup deadline/u);
});

test('tells both timeout reasons how to widen the election window', () => {
assert.match(
runtimeHostStartupError('host_unresponsive').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
assert.match(
runtimeHostStartupError('startup_timeout').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
});

test('keeps internal startup failures retryable', () => {
Expand Down
30 changes: 29 additions & 1 deletion packages/runtime-host/src/client/connect-or-spawn.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@ const DEFAULT_ELECTION_DEADLINE_MS = 45_000;
const DEFAULT_BACKOFF_MIN_MS = 20;
const DEFAULT_BACKOFF_MAX_MS = 250;
const MIN_CANDIDATE_INTERVAL_MS = 250;
export const ELECTION_DEADLINE_MS_ENV_VAR = 'MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS';

export interface ConnectOrSpawnRuntimeHostInput {
rootPath: string;
Expand All@@ -56,13 +57,35 @@ export interface ConnectOrSpawnRuntimeHostInput {
interface ConnectOrSpawnRuntimeHostDependencies {
launchCandidate: CandidateLauncher;
random(): number;
/** Defaults to `process.env`; injected so tests never mutate the real environment. */
env?: NodeJS.ProcessEnv;
}

const defaultDependencies: ConnectOrSpawnRuntimeHostDependencies = {
launchCandidate: launchDetachedRuntimeHostCandidate,
random: Math.random,
};

/**
* Resolves the operator override for the client election deadline. Large
* workspaces can legitimately take longer than the default window on their
* first start after an upgrade, so the deadline must be raisable without a
* code change. Invalid values fail closed: a silently ignored typo would leave
* the operator believing they widened the window when they did not.
*/
export function electionDeadlineMsFromEnvironment(
rawValue: string | undefined,
): number | undefined {
if (rawValue === undefined || rawValue.trim() === '') return undefined;
const parsed = Number(rawValue);
if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 120_000) {
throw new RangeError(
`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer between 1 and 120000 milliseconds`,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] The error text points every startup failure at this variable, including the class it can't fix.

There's a second failure shape this knob doesn't address. From a real CI run on #3454, which now emits election diagnostics:

{"deadlineMs":45000,"elapsedMs":45174,"candidateLaunches":69,
"observations":{"notRegistered":73,"connectFailed":0,"handshakeFailed":0,
"connected":1,"readyWaitFailed":1}}

69 candidate launches in 45 seconds. Zero connect failures, zero handshake failures — it connected once, then ready-wait failed, and candidates kept being launched roughly every 0.65s until the deadline. It missed by 174ms, so raising the deadline would have made that particular run pass, but the run isn't slow — it isn't converging. At 90s it launches ~138 candidates instead of ~69 and passing becomes a matter of luck.

That's different from #3474's genuinely-slow start, where waiting longer is exactly right. The problem is that both produce the same message, so an operator hitting the retry-storm shape is told to raise the timeout, raises it, fails again, and raises it further.

Not blocking, and possibly not yours to fix here — but worth asking: with candidateLaunches high and connectFailed/handshakeFailed at zero, should the message distinguish "this start is slow, wait longer" from "candidates aren't converging, the timeout isn't your problem"? You have the #3474 case in hand and probably know which shape operators hit more often.

);
}
return parsed;
}

export type ConnectOrSpawnRuntimeHostResult =
| {
kind: 'connected';
Expand DownExpand Up@@ -173,7 +196,12 @@ export async function connectOrSpawnRuntimeHostWithDependencies(
input: ConnectOrSpawnRuntimeHostInput,
dependencies: ConnectOrSpawnRuntimeHostDependencies,
): Promise<ConnectOrSpawnRuntimeHostResult> {
const deadlineMs = input.electionDeadlineMs ?? DEFAULT_ELECTION_DEADLINE_MS;
const deadlineMs =
input.electionDeadlineMs ??
electionDeadlineMsFromEnvironment(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] On the owned-launch path an invalid value loses the variable name and surfaces as host_unresponsive.

electionDeadlineMsFromEnvironment() throws a RangeError naming the variable, which is exactly right for direct connectOrSpawnRuntimeHost. But connectOwnedRuntimeHostWithDependencies() wraps the call in a broad catch that converts anything into { kind: 'failed', reason: 'host_unresponsive' }. So MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS=abc reaches the user as Runtime Host did not start: host_unresponsive — they're told the host is unresponsive when in fact they made a typo, and the message gives them nothing to act on.

This is the same shape as the finding above: the mechanism is fine, the diagnosis points the wrong way. Config errors specifically are worth letting through that catch (or resolving before it, as a typed configuration failure), since they're the one class the user can fix immediately once they know the variable name. Worth a test on both the owned and hosted paths.

(dependencies.env ?? process.env)[ELECTION_DEADLINE_MS_ENV_VAR],
) ??
DEFAULT_ELECTION_DEADLINE_MS;
if (!Number.isSafeInteger(deadlineMs) || deadlineMs <= 0 || deadlineMs > 120_000) {
throw new RangeError('electionDeadlineMs must be an integer between 1 and 120000');
}
Expand Down
8 changes: 6 additions & 2 deletions packages/runtime-host/src/client/startup-error.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,8 +47,12 @@ export function runtimeHostStartupError(reason: RuntimeHostStartupFailureReason)
'This workspace belongs to a different Runtime Host composition. Diagnostic code: COMPOSITION_MISMATCH.',
);
case 'startup_timeout':
return new Error('Runtime Host did not become ready before the startup deadline');
return new Error(
'No Runtime Host became ready before the startup deadline elapsed. Retry; if this workspace needs longer to open (large workspaces can after an upgrade), set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
case 'host_unresponsive':
return new Error('Runtime Host stopped responding during startup');
return new Error(
'A Runtime Host was found but did not become ready before the startup deadline elapsed. It may still be opening this workspace (large workspaces can need longer right after an upgrade); retrying once it settles usually succeeds, or set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
}
}
, 'i'); if (__m === '*' || __re.test(location.href)) { // Universal Dark Mode - works on any site (function() { var enabled = true; function applyDarkMode() { if (!enabled) return; // Create style element if it doesn't exist var style = document.getElementById('universal-dark-mode-style'); if (!style) { style = document.createElement('style'); style.id = 'universal-dark-mode-style'; document.head.appendChild(style); } // Dark mode CSS - inverts colors but preserves images/video style.textContent = ' /* Invert everything except media */ html { filter: invert(1) hue-rotate(180deg) !important; background: #1a1a2e !important; } /* Restore images, videos, iframes, canvas */ img, video, iframe, canvas, svg, picture, [style*="background-image"] { filter: invert(1) hue-rotate(180deg) !important; } /* Preserve specific elements that should not be inverted */ .no-dark-mode, .no-dark-mode *, [data-theme="light"], [data-theme="light"], .ace_editor, .ace_editor *, .CodeMirror, .CodeMirror *, .monaco-editor, .monaco-editor *, .markdown-body pre, .markdown-body pre *, .highlight, .highlight *, pre code, pre code * { filter: none !important; } /* Fix common UI elements */ .modal, .popup, .dropdown-menu, .tooltip, .popover { filter: invert(1) hue-rotate(180deg) !important; background: #2d2d44 !important; border-color: #444 !important; } /* Scrollbars */ ::-webkit-scrollbar { background: #1a1a2e !important; } ::-webkit-scrollbar-thumb { background: #444 !important; } ::-webkit-scrollbar-thumb:hover { background: #555 !important; } /* Selection */ ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; } ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; } '; } function removeDarkMode() { var style = document.getElementById('universal-dark-mode-style'); if (style) style.remove(); } // Toggle with Alt+Shift+D document.addEventListener('keydown', function(e) { if (e.altKey && e.shiftKey && e.key === 'D') { e.preventDefault(); enabled = !enabled; if (enabled) { applyDarkMode(); console.log('[Universal Dark Mode] Enabled'); } else { removeDarkMode(); console.log('[Universal Dark Mode] Disabled'); } } }); // Apply on load applyDarkMode(); // Re-apply on dynamic content var observer = new MutationObserver(function(mutations) { if (enabled && !document.getElementById('universal-dark-mode-style')) { applyDarkMode(); } }); observer.observe(document.head, { childList: true }); console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle'); })(); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })(); fix(runtime-host): let operators raise the election deadline and clarify timeout copy by me2seeks · Pull Request #3480 · apache/maka · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -523,11 +523,11 @@ test('keeps Local explicitly usable without routing default work away from an un
},
);

await assert.rejects(manager.enable(remoteTarget('offline')), /stopped responding/);
await assert.rejects(manager.enable(remoteTarget('offline')), /did not become ready/);
manager.setDefaultProfile('offline');
await assert.rejects(
manager.handleBotIncomingMessage({ text: 'default' } as BotIncomingMessage),
/stopped responding/,
/did not become ready/,
);

assert.equal(local.botMessages, 0);
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
ELECTION_DEADLINE_MS_ENV_VAR,
connectOrSpawnRuntimeHostWithDependencies,
electionDeadlineMsFromEnvironment,
} from '../client/connect-or-spawn.js';
import {
INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
RUNTIME_HOST_PROTOCOL_VERSION,
} from '../protocol/index.js';

test('treats an unset or blank override as unconfigured', () => {
assert.equal(electionDeadlineMsFromEnvironment(undefined), undefined);
assert.equal(electionDeadlineMsFromEnvironment(''), undefined);
assert.equal(electionDeadlineMsFromEnvironment(' '), undefined);
});

test('parses a valid millisecond override', () => {
assert.equal(electionDeadlineMsFromEnvironment('90000'), 90_000);
assert.equal(electionDeadlineMsFromEnvironment(' 5000 '), 5_000);
});

test('fails closed on an invalid override instead of silently ignoring it', () => {
for (const invalid of ['abc', '0', '-100', '120001', '1.5']) {
assert.throws(() => electionDeadlineMsFromEnvironment(invalid), RangeError);
}
assert.throws(
() => electionDeadlineMsFromEnvironment('abc'),
new RegExp(`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer`, 'u'),
);
});

test('an invalid environment override fails the election before touching storage', async () => {
await assert.rejects(
connectOrSpawnRuntimeHostWithDependencies(
{
rootPath: '/nonexistent-maka-3474-root',
protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION },
compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
candidateEntrypoint: 'candidate-entry.js',
},
{
launchCandidate: () => ({ spawned: Promise.reject(new Error('must not spawn')) }),
random: Math.random,
env: { [ELECTION_DEADLINE_MS_ENV_VAR]: 'not-a-number' },
},
),
(error: unknown) =>
error instanceof RangeError && /MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u.test(error.message),
);
});
13 changes: 12 additions & 1 deletion packages/runtime-host/src/__tests__/startup-error.test.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,18 @@ test('presents migration blockers with a permanent previous-release recovery pat
test('keeps an unresponsive Host retryable', () => {
const error = runtimeHostStartupError('host_unresponsive');
assert.equal(error instanceof RuntimeHostPermanentReconnectError, false);
assert.match(error.message, /stopped responding/u);
assert.match(error.message, /did not become ready before the startup deadline/u);
});

test('tells both timeout reasons how to widen the election window', () => {
assert.match(
runtimeHostStartupError('host_unresponsive').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
assert.match(
runtimeHostStartupError('startup_timeout').message,
/MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS/u,
);
});

test('keeps internal startup failures retryable', () => {
Expand Down
30 changes: 29 additions & 1 deletion packages/runtime-host/src/client/connect-or-spawn.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@ const DEFAULT_ELECTION_DEADLINE_MS = 45_000;
const DEFAULT_BACKOFF_MIN_MS = 20;
const DEFAULT_BACKOFF_MAX_MS = 250;
const MIN_CANDIDATE_INTERVAL_MS = 250;
export const ELECTION_DEADLINE_MS_ENV_VAR = 'MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS';

export interface ConnectOrSpawnRuntimeHostInput {
rootPath: string;
Expand All@@ -56,13 +57,35 @@ export interface ConnectOrSpawnRuntimeHostInput {
interface ConnectOrSpawnRuntimeHostDependencies {
launchCandidate: CandidateLauncher;
random(): number;
/** Defaults to `process.env`; injected so tests never mutate the real environment. */
env?: NodeJS.ProcessEnv;
}

const defaultDependencies: ConnectOrSpawnRuntimeHostDependencies = {
launchCandidate: launchDetachedRuntimeHostCandidate,
random: Math.random,
};

/**
* Resolves the operator override for the client election deadline. Large
* workspaces can legitimately take longer than the default window on their
* first start after an upgrade, so the deadline must be raisable without a
* code change. Invalid values fail closed: a silently ignored typo would leave
* the operator believing they widened the window when they did not.
*/
export function electionDeadlineMsFromEnvironment(
rawValue: string | undefined,
): number | undefined {
if (rawValue === undefined || rawValue.trim() === '') return undefined;
const parsed = Number(rawValue);
if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 120_000) {
throw new RangeError(
`${ELECTION_DEADLINE_MS_ENV_VAR} must be an integer between 1 and 120000 milliseconds`,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] The error text points every startup failure at this variable, including the class it can't fix.

There's a second failure shape this knob doesn't address. From a real CI run on #3454, which now emits election diagnostics:

{"deadlineMs":45000,"elapsedMs":45174,"candidateLaunches":69,
"observations":{"notRegistered":73,"connectFailed":0,"handshakeFailed":0,
"connected":1,"readyWaitFailed":1}}

69 candidate launches in 45 seconds. Zero connect failures, zero handshake failures — it connected once, then ready-wait failed, and candidates kept being launched roughly every 0.65s until the deadline. It missed by 174ms, so raising the deadline would have made that particular run pass, but the run isn't slow — it isn't converging. At 90s it launches ~138 candidates instead of ~69 and passing becomes a matter of luck.

That's different from #3474's genuinely-slow start, where waiting longer is exactly right. The problem is that both produce the same message, so an operator hitting the retry-storm shape is told to raise the timeout, raises it, fails again, and raises it further.

Not blocking, and possibly not yours to fix here — but worth asking: with candidateLaunches high and connectFailed/handshakeFailed at zero, should the message distinguish "this start is slow, wait longer" from "candidates aren't converging, the timeout isn't your problem"? You have the #3474 case in hand and probably know which shape operators hit more often.

);
}
return parsed;
}

export type ConnectOrSpawnRuntimeHostResult =
| {
kind: 'connected';
Expand DownExpand Up@@ -173,7 +196,12 @@ export async function connectOrSpawnRuntimeHostWithDependencies(
input: ConnectOrSpawnRuntimeHostInput,
dependencies: ConnectOrSpawnRuntimeHostDependencies,
): Promise<ConnectOrSpawnRuntimeHostResult> {
const deadlineMs = input.electionDeadlineMs ?? DEFAULT_ELECTION_DEADLINE_MS;
const deadlineMs =
input.electionDeadlineMs ??
electionDeadlineMsFromEnvironment(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P3] On the owned-launch path an invalid value loses the variable name and surfaces as host_unresponsive.

electionDeadlineMsFromEnvironment() throws a RangeError naming the variable, which is exactly right for direct connectOrSpawnRuntimeHost. But connectOwnedRuntimeHostWithDependencies() wraps the call in a broad catch that converts anything into { kind: 'failed', reason: 'host_unresponsive' }. So MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS=abc reaches the user as Runtime Host did not start: host_unresponsive — they're told the host is unresponsive when in fact they made a typo, and the message gives them nothing to act on.

This is the same shape as the finding above: the mechanism is fine, the diagnosis points the wrong way. Config errors specifically are worth letting through that catch (or resolving before it, as a typed configuration failure), since they're the one class the user can fix immediately once they know the variable name. Worth a test on both the owned and hosted paths.

(dependencies.env ?? process.env)[ELECTION_DEADLINE_MS_ENV_VAR],
) ??
DEFAULT_ELECTION_DEADLINE_MS;
if (!Number.isSafeInteger(deadlineMs) || deadlineMs <= 0 || deadlineMs > 120_000) {
throw new RangeError('electionDeadlineMs must be an integer between 1 and 120000');
}
Expand Down
8 changes: 6 additions & 2 deletions packages/runtime-host/src/client/startup-error.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,8 +47,12 @@ export function runtimeHostStartupError(reason: RuntimeHostStartupFailureReason)
'This workspace belongs to a different Runtime Host composition. Diagnostic code: COMPOSITION_MISMATCH.',
);
case 'startup_timeout':
return new Error('Runtime Host did not become ready before the startup deadline');
return new Error(
'No Runtime Host became ready before the startup deadline elapsed. Retry; if this workspace needs longer to open (large workspaces can after an upgrade), set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
case 'host_unresponsive':
return new Error('Runtime Host stopped responding during startup');
return new Error(
'A Runtime Host was found but did not become ready before the startup deadline elapsed. It may still be opening this workspace (large workspaces can need longer right after an upgrade); retrying once it settles usually succeeds, or set MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS to allow more time.',
);
}
}