') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Make server startup and teardown failures diagnosable by subwire · Pull Request #223 · binsync/declib · GitHub
Skip to content

Make server startup and teardown failures diagnosable - #223

Merged
mahaloz merged 2 commits into
mainfrom
server-startup-reliability
Aug 2, 2026
Merged

Make server startup and teardown failures diagnosable#223
mahaloz merged 2 commits into
mainfrom
server-startup-reliability

Conversation

@subwire

@subwiresubwire commented Jul 26, 2026

Copy link
Copy Markdown
Collaborator

Context

We ran DecLib as the sole decompiler interface for a fleet of autonomous CTF agents at D3CTF 2026 and replayed every session afterwards. 54 of the 62 DecLib errors we logged were these three. None told the user what had actually happened, and each one ended with an agent abandoning the CLI for a hand-rolled IDAPython script.

1. Failed to open database <binary> (14×)

Reproduced, and the cause is not what the message suggests. It is what you get when a second server opens a project database another server already holds:

$ decompiler load /tmp/orig --backend ida # fine
$ decompiler load /tmp/orig --backend ida --force
Decompiler server 7f5bf7f798 exited with status 1 before registering.
ERROR - Failed to start server: Failed to open database /tmp/orig

--force is documented as "run a second copy", but the default project dir is derived from binary+backend alone — so the second server always collided with the first and died. Forced copies now get their own project dir:

after: two live servers
project_dir .../orig-5a3e46a4
project_dir .../orig-5a3e46a4-722556f738

The bare message also reads like file corruption, so any server log carrying it now gets an explanation naming the real cause and the three ways out (--id, --replace, --project-dir).

2. Timed out waiting Ns for server to start (19×)

Two problems.

The advice read "Check backend dependencies (e.g. GHIDRA_INSTALL_DIR)" regardless of backend — actively misleading when the backend is IDA. It is now backend-aware.

And the wait was silent, so a legitimately slow load was indistinguishable from a hang. Agents retried at 180s, then 240s, then 300s — ~12 minutes of solve budget per abandonment. The wait now reports every 10s, surfacing the backend's own last log line:

 [10s] INFO | declib.api.decompiler_server | Using headless interface utilizing ida
[20s] still starting (5s left)...
Timed out waiting 25s for server 89ec2f75dd to start. Check the IDA install and
that its licence/EULA is accepted (`decompiler backend status ida`).

(That run is a real 137 MB libLLVM.so — genuinely slow rather than stuck, which is now visible.)

3. No running decompiler server matches {...} (21×)

The registry prunes dead records during the lookup, so a server that died left no trace and the error read as "you never started one". list_servers / find_servers can now report what they reaped, and the error names the corpse:

Decompiler server 078c974380 is no longer running (binary: /tmp/orig).
It died or was stopped; its analysis is gone.
Reload it with:
decompiler load /tmp/orig --backend ida

Deliberately not doing silent auto-revive. Re-analysis can take minutes, and quietly spending them — or quietly discarding unsaved annotations — is a worse surprise than an accurate error naming the exact command to run. Happy to add it behind an explicit flag if you'd prefer.

Tests

Backend-aware hints for every backend (asserting GHIDRA_INSTALL_DIR appears for ghidra and only ghidra); the lock explanation fires on lock failures and not on unrelated ones; the registry reports pruned records with enough detail to name the binary.

🤖 Generated with Claude Code

@subwire@claude
Make server startup and teardown failures diagnosable
094a72d
Three failures accounted for 54 of the 62 DecLib errors we logged across
a CTF where DecLib was the sole decompiler interface. None of them told
the user what had actually happened, and each one ended with an agent
abandoning the CLI for a hand-rolled script.
**"Failed to open database <binary>" (14x).** Reproduced: it is what you
get when a second server opens a project another server already holds.
`--force` is documented as "run a second copy", but the default project
dir is derived from binary+backend alone, so the second server always
collided with the first and died. Forced copies now get their own project
dir, and `--force` works:
before: Failed to open database /tmp/orig
after: two live servers, project_dir orig-5a3e46a4
and orig-5a3e46a4-722556f738
The bare message also reads like file corruption, so any log carrying it
now gets an explanation naming the real cause and the three ways out
(`--id`, `--replace`, `--project-dir`).
**"Timed out waiting Ns for server to start" (19x).** Two problems. The
advice said "Check backend dependencies (e.g. GHIDRA_INSTALL_DIR)"
whatever the backend was -- actively misleading on IDA. It is now
backend-aware. And the wait was silent, so a legitimately slow load was
indistinguishable from a hang; agents retried at 180s, 240s, then 300s,
burning ~12 minutes before giving up. The wait now reports every 10s,
surfacing the backend's own last log line:
[10s] INFO | declib.api.decompiler_server | Using headless interface utilizing ida
[20s] still starting (5s left)...
Timed out waiting 25s ... Check the IDA install and that its licence/EULA
is accepted (`decompiler backend status ida`).
**"No running decompiler server matches {...}" (21x).** The registry
prunes dead records during lookup, so a server that died left no trace
and the error read as "you never started one". list_servers/find_servers
can now report what they reaped, and the error names the corpse:
Decompiler server 078c974380 is no longer running (binary: /tmp/orig).
It died or was stopped; its analysis is gone.
Reload it with:
decompiler load /tmp/orig --backend ida
Deliberately not doing silent auto-revive: re-analysis can take minutes,
and quietly spending them (or quietly losing unsaved annotations) is a
worse surprise than an accurate error naming the command to run.
Tests: backend-aware hints for every backend, the lock explanation fires
only on lock failures, and the registry reports pruned records.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
@subwire
subwireforce-pushed the server-startup-reliability branch from 9ef3493 to 094a72dCompareJuly 26, 2026 17:35
@subwiresubwire mentioned this pull request Jul 29, 2026
Merged
@mahaloz
mahaloz merged commit ac41709 into mainAug 2, 2026
5 checks passed
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

No reviews

Assignees

No one assigned

Labels

None yet

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

2 participants

@subwire@mahaloz