') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Make the Dag cache configurable and per-component in metrics by jason810496 · Pull Request #71813 · apache/airflow · GitHub
Skip to content

Make the Dag cache configurable and per-component in metrics - #71813

Closed
jason810496 wants to merge 6 commits into
apache:mainfrom
jason810496:feature/dagbag-cache-config-and-metrics
Closed

Make the Dag cache configurable and per-component in metrics#71813
jason810496 wants to merge 6 commits into
apache:mainfrom
jason810496:feature/dagbag-cache-config-and-metrics

Conversation

@jason810496

@jason810496jason810496 commented Aug 19, 2026

Copy link
Copy Markdown
Member

Draft. Holds the work carved out of #71704 so nothing is lost while that PR is reduced to the minimal, cherry-pickable scheduler fix.

related: #71704

Why this exists

#71704 originally bundled four separable things. Per review feedback there, it now ships only the scheduler cache bound — one line, no new configuration, no DBDagBag API change — so it can be cherry-picked cleanly. This branch carries the rest.

What is still here

  • [api] dag_cache_size = 0 silently ignored [api] dag_cache_ttl. Selecting "no size limit" also disabled TTL eviction, so TTL could not be enabled without also accepting a size cap. A pre-existing bug in shipped behaviour, and backportable on its own.
  • DBDagBag gains stats_prefix, so each component reports cache activity under its own metric namespace instead of every caller emitting api_server.dag_bag.*. The minimal fix in Fix scheduler DBDagBag unbounded cache #71704 leaves the scheduler reporting under the API server's names; this is what corrects that.
  • [scheduler] dag_cache_size / dag_cache_ttl, making the bound introduced in Fix scheduler DBDagBag unbounded cache #71704 tunable, plus the [api] option docs and their version_added correction (3.3.0 was wrong; those options shipped in 3.2.2).
  • Metrics registry check matches dynamic names by their static parts, so a {variable} can sit anywhere in a metric name rather than only after a fixed prefix. Supersedes Support dynamic metric name prefixes in the metrics registry check #71276.

Planned split

This is not intended to merge as one PR. It will be split into at least:

  1. the [api] TTL bugfix, targeted for backport;
  2. the stats_prefix / metric-namespace change plus the registry check;
  3. the new [scheduler] configuration, for a minor release.

Kept as a single draft for now so the deferred work is reviewable in one place and visibly not dropped.

Note

Branched before #71704 was reduced, so it needs rebasing onto current main before any of the above is split out for real review.

Was generative AI tooling used to co-author this PR?
  • Yes — Claude Code (Opus 5)

Generated-by: Claude Code (Opus 5) following the guidelines

@jason810496
Bound the scheduler's Dag cache and honor TTL without a size cap
95c235d
The scheduler cached deserialized Dags in a dict that never evicted, so every
Dag version it had ever seen stayed resident and the process was eventually
OOM killed.
``[api] dag_cache_size = 0`` had a related gap: it selected that same
never-evicting dict and silently ignored ``[api] dag_cache_ttl``, so TTL
eviction could not be enabled without also accepting a size limit.
Each re-check resets a cached entry's expiry, so the TTL reclaims a version
once its Dag runs finish and it stops being requested; ``dag_cache_size``
remains the only hard ceiling.
closes: apache#69001
@jason810496
Address Ash's comments
dfca456
- Move the scheduler's DagBag factory into `scheduler_job_runner` as
`_create_scheduler_dag_bag`, dropping the `airflow/jobs/scheduler_dagbag.py`
module and the `SchedulerDBDagBag` subclass it held.
- Collapse `APIServerDBDagBag` back into the existing `create_dag_bag`, so both
components resolve their config section and metric prefix in a plain factory
next to where they build the bag.
- Drop `airflow-core/tests/unit/jobs/test_scheduler_dagbag.py`.
- Trim `test_cache_selection` from 11 cases to the 5 distinct branches of the
mapping selection, and drop `test_uncapped_ttl_cache_accepts_entries`, which
only asserted that cachetools honours an unbounded `maxsize`.
- Un-parametrize the four cache-metric tests over the two components. The
subclasses never overrode the `_stat_*` hooks, so both runs exercised the same
base code; `test_stats_prefix_expands_to_registered_metrics` still pins each
component's real prefix against the metrics registry.
- Drop `dev/airflow_perf/dag_bag_cache_overhead.py`. An in-memory dict lookup is
orders of magnitude cheaper than the DB load and deserialize it guards, so the
harness has no long-term value in the repo.
@jason810496
Default the scheduler Dag cache to a bounded LRU of 1024 versions
ef09e82
A TTL cannot cap the cache on its own. Each re-check re-arms an entry's expiry,
so a TTL reclaims a Dag version only once its runs finish and it stops being
requested — that bounds memory by the concurrently active set, which no fixed
number predicts, rather than outright. A default that leaves memory dependent on
request patterns is the wrong default for the OOM this PR set out to fix.
A size limit is the only hard ceiling, so the scheduler now defaults to
`dag_cache_size = 1024` with `dag_cache_ttl = 0`. 1024 is meant to sit above the
versions-with-runs-in-flight working set of a typical deployment, so eviction
costs a re-fetch only where that working set is genuinely larger; the
`scheduler.dag_bag.cache_miss` metric is what tells an operator to raise it. The
TTL-only and no-eviction modes both remain reachable by configuration.
@jason810496
Mark the scheduler Dag cache options as added in 3.4.0
80a08bb
These options are new on main, which is 3.4.0. Claiming 3.3.2 would advertise
them as available in a patch release that never carried them, sending anyone on
3.3.x looking for settings they cannot configure. The surrounding docs described
the pre-cache behaviour as ending at the same wrong version, so they move
together.
The `[api]` pair keeps 3.2.2: those options already shipped, and that value is
the correction this PR makes to their previously mis-stated 3.3.0.
@jason810496
Lower the scheduler Dag cache default from 1024 to 512 versions
f5ec422
Ash flagged 1024 as possibly too aggressive a default. 512 still sits above the
versions-with-runs-in-flight working set of a typical deployment while roughly
halving the worst-case memory footprint of the default cache.
@boring-cyborgboring-cyborgBot added area:API Airflow's REST/HTTP API area:ConfigTemplates area:dev-tools area:Scheduler including HA (high availability) scheduler backport-to-v3-3-test Backport to v3-3-test kind:documentation labels Aug 19, 2026
@jason810496jason810496 mentioned this pull request Aug 19, 2026
Merged
1 task
@jason810496
Renumber the newsfragment to this PR
451e634
The scheduler cache bound it originally accompanied now ships separately in
apache#71704; what remains here is the configuration, metric namespacing, and the
`[api]` TTL fix, so the entry belongs to this PR's number.
This was referenced Aug 19, 2026
Merged
Merged
Open
@jason810496

jason810496 commented Aug 19, 2026

Copy link
Copy Markdown
MemberAuthor

Split into three focused PRs, so closing this:

The scheduler cache bound itself ships separately in #71704.

Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@ashbashbAwaiting requested review from ashbashb will be requested when the pull request is marked ready for reviewashb is a code owner
@amoghrajeshamoghrajeshAwaiting requested review from amoghrajeshamoghrajesh will be requested when the pull request is marked ready for reviewamoghrajesh is a code owner
@potiukpotiukAwaiting requested review from potiukpotiuk will be requested when the pull request is marked ready for reviewpotiuk is a code owner
@gopidesupavangopidesupavanAwaiting requested review from gopidesupavangopidesupavan will be requested when the pull request is marked ready for reviewgopidesupavan is a code owner
@jscheffljschefflAwaiting requested review from jscheffljscheffl will be requested when the pull request is marked ready for reviewjscheffl is a code owner
@bugraoz93bugraoz93Awaiting requested review from bugraoz93bugraoz93 will be requested when the pull request is marked ready for reviewbugraoz93 is a code owner
@ephraimbuddyephraimbuddyAwaiting requested review from ephraimbuddyephraimbuddy will be requested when the pull request is marked ready for reviewephraimbuddy is a code owner
@pierrejeambrunpierrejeambrunAwaiting requested review from pierrejeambrunpierrejeambrun will be requested when the pull request is marked ready for reviewpierrejeambrun is a code owner
@rawwarrawwarAwaiting requested review from rawwarrawwar will be requested when the pull request is marked ready for reviewrawwar is a code owner
@shubhamraj-gitshubhamraj-gitAwaiting requested review from shubhamraj-gitshubhamraj-git will be requested when the pull request is marked ready for reviewshubhamraj-git is a code owner
@choo121600choo121600Awaiting requested review from choo121600choo121600 will be requested when the pull request is marked ready for reviewchoo121600 is a code owner
@henry3260henry3260Awaiting requested review from henry3260henry3260 will be requested when the pull request is marked ready for reviewhenry3260 is a code owner
@XD-DENGXD-DENGAwaiting requested review from XD-DENGXD-DENG will be requested when the pull request is marked ready for reviewXD-DENG is a code owner

Assignees

No one assigned

Labels

area:APIAirflow's REST/HTTP APIarea:ConfigTemplatesarea:dev-toolsarea:Schedulerincluding HA (high availability) schedulerbackport-to-v3-3-testBackport to v3-3-testkind:documentation

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

1 participant

@jason810496