') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Make the scheduler Dag cache size and TTL configurable by jason810496 · Pull Request #71816 · apache/airflow · GitHub
Skip to content

Make the scheduler Dag cache size and TTL configurable - #71816

Open
jason810496 wants to merge 1 commit into
apache:mainfrom
jason810496:feat/scheduler-dag-cache-config
Open

Make the scheduler Dag cache size and TTL configurable#71816
jason810496 wants to merge 1 commit into
apache:mainfrom
jason810496:feat/scheduler-dag-cache-config

Conversation

@jason810496

@jason810496jason810496 commented Aug 19, 2026•
edited
Loading

Copy link
Copy Markdown
Member

Why

The scheduler's cache bound landed in #71704 as a fixed constant. Deployments whose active Dag-version working set exceeds that limit cannot raise it, while memory-constrained deployments cannot lower it or choose time-based eviction.

How

The API server and scheduler read and validate their size and TTL pairs through one helper. Each component constructs CachedDBDagBag for every configuration. When both values are 0, the configured cache uses a plain unbounded dict and performs no eviction.

What

  • Adds [scheduler] dag_cache_size, defaulting to the existing LRU limit of 512 versions.
  • Adds [scheduler] dag_cache_ttl, defaulting to 0 with no time-based eviction.
  • Defines 0 consistently: size 0 disables size-based eviction, TTL 0 disables time-based eviction, and both 0 disable eviction for the process lifetime.
  • Documents the scheduler tuning guidance and the corresponding metrics.
  • Adds a feature newsfragment for the new scheduler settings.
Was generative AI tooling used to co-author this PR?

@boring-cyborgboring-cyborgBot added area:API Airflow's REST/HTTP API area:ConfigTemplates area:dev-tools area:Scheduler including HA (high availability) scheduler backport-to-v3-3-test Backport to v3-3-test kind:documentation labels Aug 19, 2026
This was referenced Aug 19, 2026
Merged
Merged
Closed
Merged
@jason810496jason810496 added this to the Airflow 3.4.0 milestone Aug 19, 2026
@jason810496
jason810496force-pushed the feat/scheduler-dag-cache-config branch 7 times, most recently from 47dda94 to e2f7da7CompareAugust 21, 2026 12:19
@jason810496jason810496 removed the area:API Airflow's REST/HTTP API label Aug 23, 2026
@jason810496
jason810496 marked this pull request as ready for review August 23, 2026 14:03
@jason810496jason810496 self-assigned this Aug 23, 2026
@jason810496
Make the scheduler Dag cache size and TTL configurable
9380281
A fixed 512-entry cache cannot fit every deployment: larger active working sets churn while memory-constrained schedulers may need a lower cap. Exposing size and TTL controls lets operators trade memory for refetch and deserialization cost while preserving the current defaults.
@jason810496
jason810496force-pushed the feat/scheduler-dag-cache-config branch from e2f7da7 to 9380281CompareAugust 23, 2026 14:29
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@ashbashbAwaiting requested review from ashbashb is a code owner
@XD-DENGXD-DENGAwaiting requested review from XD-DENGXD-DENG is a code owner
@uranusjruranusjrAwaiting requested review from uranusjr

At least 1 approving review is required to merge this pull request.

Assignees

@jason810496jason810496

Labels

area:ConfigTemplatesarea:dev-toolsarea:Schedulerincluding HA (high availability) schedulerbackport-to-v3-3-testBackport to v3-3-testkind:documentation

Projects

None yet

Milestone

Airflow 3.4.0

Development

Successfully merging this pull request may close these issues.

1 participant

@jason810496