Pin task bundle manifest to the dagrun's version#69941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
o-nikolas merged 1 commit into from
Jul 17, 2026
Merged

Pin task bundle manifest to the dagrun's version #69941

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion airflow-core/src/airflow/executors/workloads/task.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,11 @@ def make(
bundle_info = BundleInfo(
name=ti.dag_model.bundle_name,
version=ti.dag_run.bundle_version,
version_data=_resolve_version_data(ti.dag_version, ti.dag_run.bundle_version),
# Source version_data from the run's pinned version (matching ``version`` above),
# not the TI's dag_version. A mid-run DAG re-parse can bump the TI's dag_version
# to a newer version while the run stays pinned; sourcing from created_dag_version
# keeps the shipped hash and manifest consistent so versioned bundles stay reproducible.
version_data=_resolve_version_data(ti.dag_run.created_dag_version, ti.dag_run.bundle_version),
)
fname = log_filename_template_renderer()(ti=ti)

Expand Down
17 changes: 11 additions & 6 deletions airflow-core/src/airflow/jobs/scheduler_job_runner.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -684,12 +684,17 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -
ranked_query.c.map_index_for_ordering,
)
.options(selectinload(TI.dag_model))
# Eager-load dag_version: TIs become transient (via make_transient) before
# ExecuteTask.make() reads ti.dag_version.version_data. Lazy loads on
# transient objects silently return None instead of raising DetachedInstanceError.
# Scope the second SELECT to version_data (the PK is auto-included) so we read
# two columns rather than the full DagVersion row.
.options(selectinload(TI.dag_version).load_only(DagVersion.version_data))
# Eager-load the run's pinned DagVersion (dag_run.created_dag_version): TIs become
# transient (via make_transient) before ExecuteTask.make() reads
# ti.dag_run.created_dag_version.version_data to ship the bundle manifest matching
# the run's pinned bundle_version. Lazy loads on transient objects silently return
# None instead of raising DetachedInstanceError. Scope the SELECT to version_data
# (the PK is auto-included) so we read two columns rather than the full row.
.options(
joinedload(TI.dag_run)
.selectinload(DagRun.created_dag_version)
.load_only(DagVersion.version_data)
)
)

query = query.limit(max_tis)
Expand Down
55 changes: 43 additions & 12 deletions airflow-core/tests/unit/executors/test_workloads.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -184,12 +184,21 @@ def _stub_log_template(self, monkeypatch):
)

@staticmethod
def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):
def _make_mock_ti(
bundle_version,
version_data,
*,
has_created_dag_version=True,
ti_dag_version_data=None,
):
"""Build a mock TI with the attributes ExecuteTask.make() reads.

``has_dag_version`` controls whether the TI has an associated DagVersion
(legacy/backfilled TIs may not), independently of ``version_data`` so the
pin-guard can be exercised with version_data present on an unpinned run.
``version_data`` is the manifest on the run's pinned version
(``dag_run.created_dag_version``) -- the source make() must use.
``ti_dag_version_data`` is the manifest on ``ti.dag_version``, which make()
must IGNORE (it can diverge from the run's pin after a mid-run DAG re-parse).
``has_created_dag_version`` toggles whether the run has a pinned DagVersion
(legacy/backfilled runs may not).
"""
from unittest.mock import Mock

Expand All@@ -215,15 +224,18 @@ def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):

ti.dag_run.bundle_version = bundle_version

if has_dag_version:
ti.dag_version.version_data = version_data
# make() must source version_data from the run's pinned version, never ti.dag_version.
ti.dag_version.version_data = ti_dag_version_data

if has_created_dag_version:
ti.dag_run.created_dag_version.version_data = version_data
else:
ti.dag_version = None
ti.dag_run.created_dag_version = None

return ti

def test_pinned_run_populates_version_data(self):
"""When the run is pinned, version_data from dag_version flows to BundleInfo."""
"""When the run is pinned, version_data from the run's created_dag_version flows to BundleInfo."""
version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
ti = self._make_mock_ti(bundle_version="abc123", version_data=version_data)

Expand All@@ -233,7 +245,7 @@ def test_pinned_run_populates_version_data(self):
assert workload.bundle_info.version_data == version_data

def test_unpinned_run_suppresses_present_version_data(self):
"""An unpinned run must not expose version_data even when the dag_version carries it."""
"""An unpinned run must not expose version_data even when created_dag_version carries it."""
version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
ti = self._make_mock_ti(bundle_version=None, version_data=version_data)

Expand All@@ -242,15 +254,34 @@ def test_unpinned_run_suppresses_present_version_data(self):
assert workload.bundle_info.version is None
assert workload.bundle_info.version_data is None

def test_missing_dag_version_yields_none(self):
"""A pinned run whose TI has no dag_version (legacy/backfilled) yields no version_data."""
ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_dag_version=False)
def test_missing_created_dag_version_yields_none(self):
"""A pinned run whose DagRun has no created_dag_version yields no version_data."""
ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_created_dag_version=False)

workload = ExecuteTask.make(ti)

assert workload.bundle_info.version == "abc123"
assert workload.bundle_info.version_data is None

def test_mid_run_dag_version_bump_uses_run_pinned_manifest(self):
"""Regression: a mid-run DAG re-parse can bump ti.dag_version to a newer version while the
run stays pinned. make() must ship the run's pinned manifest (created_dag_version), not the
TI's bumped one -- otherwise a versioned bundle would fetch the wrong (latest) code.
"""
run_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v1-object-id"}}
bumped_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v2-object-id"}}
ti = self._make_mock_ti(
bundle_version="v1hash",
version_data=run_manifest,
ti_dag_version_data=bumped_manifest,
)

workload = ExecuteTask.make(ti)

assert workload.bundle_info.version == "v1hash"
assert workload.bundle_info.version_data == run_manifest
assert workload.bundle_info.version_data != bumped_manifest


class TestExecuteCallbackMakeVersionData:
"""Tests for ExecuteCallback.make() threading version_data through BundleInfo."""
Expand Down
37 changes: 36 additions & 1 deletion airflow-core/tests/unit/jobs/test_scheduler_job.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -142,7 +142,7 @@
from airflow.utils.types import DagRunTriggeredByType, DagRunType

from tests_common.pytest_plugin import AIRFLOW_ROOT_PATH
from tests_common.test_utils.asserts import assert_queries_count
from tests_common.test_utils.asserts import assert_queries_count, count_queries
from tests_common.test_utils.config import conf_vars, env_vars
from tests_common.test_utils.dag import create_scheduler_dag, sync_dag_to_db, sync_dags_to_db
from tests_common.test_utils.db import (
Expand DownExpand Up@@ -1425,6 +1425,41 @@ def test_find_executable_task_instances_backfill(self, dag_maker):
assert {x.key for x in queued_tis} == {ti_non_backfill.key, ti_backfill.key}
session.rollback()

def test_executable_task_instances_no_per_ti_queries(self, dag_maker, session):
"""Guard against an N+1 when enqueuing task instances.

``ExecuteTask.make()`` reads ``ti.dag_run.created_dag_version.version_data`` to ship the
run's pinned bundle manifest. ``dag_run`` is eager-joined and ``created_dag_version`` is a
single batched ``selectin``, so the number of queries in
``_executable_task_instances_to_queued`` must be independent of how many task instances are
in the batch. If a future change lazy-loads ``dag_run``/``created_dag_version`` per TI, the
count would scale with the task count and this test fails.
"""
scheduler_job = Job()
runner = SchedulerJobRunner(job=scheduler_job)
self.job_runner = runner

def _measure(dag_id: str, num_tasks: int) -> int:
with dag_maker(dag_id=dag_id, max_active_tasks=64, session=session):
for i in range(num_tasks):
EmptyOperator(task_id=f"t{i}")
dr = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
for ti in dr.task_instances:
ti.state = State.SCHEDULED
session.flush()
with count_queries(session=session) as result:
runner._executable_task_instances_to_queued(max_tis=64, session=session)
session.rollback()
return sum(result.values())

one_task = _measure("q_count_one", 1)
many_tasks = _measure("q_count_many", 10)

assert one_task == many_tasks, (
f"query count scaled with task-instance count ({one_task} -> {many_tasks}); "
"likely a per-TI lazy load (N+1) of dag_run/created_dag_version"
)

def test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query(self, dag_maker, session):
dag_id = "SchedulerJobTest.test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query"
task_id = "dummy"
Expand Down
2 changes: 2 additions & 0 deletions providers/amazon/tests/unit/amazon/aws/executors/batch/test_batch_executor.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -817,6 +817,8 @@ def test_try_adopt_task_instances(self, mock_executor):
task.dag_version = mock.Mock(version_data=None)
task.dag_run = mock.Mock()
task.dag_run.bundle_version = "1.0.0"
# ExecuteTask.make() sources version_data from the run's pinned version.
task.dag_run.created_dag_version = mock.Mock(version_data=None)
task.dag_run.context_carrier = {}

if not AIRFLOW_V_3_0_PLUS:
Expand Down
2 changes: 2 additions & 0 deletions providers/amazon/tests/unit/amazon/aws/executors/ecs/test_ecs_executor.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1312,6 +1312,8 @@ def test_try_adopt_task_instances(self, mock_executor):
task.dag_version = mock.Mock(version_data=None)
task.dag_run = mock.Mock()
task.dag_run.bundle_version = "1.0.0"
# ExecuteTask.make() sources version_data from the run's pinned version.
task.dag_run.created_dag_version = mock.Mock(version_data=None)
task.dag_run.context_carrier = {}

# Mock command generation based on Airflow version
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Pin task bundle manifest to the dagrun's version#69941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

o-nikolas
merged 1 commit into

from

Jul 17, 2026
 Merged


Pin task bundle manifest to the dagrun's version
#69941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 5 additions & 1 deletion
airflow-core/src/airflow/executors/workloads/task.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,11 @@ def make(
 bundle_info = BundleInfo(
 name=ti.dag_model.bundle_name,
 version=ti.dag_run.bundle_version,
 version_data=_resolve_version_data(ti.dag_version, ti.dag_run.bundle_version),
 # Source version_data from the run's pinned version (matching ``version`` above),
 # not the TI's dag_version. A mid-run DAG re-parse can bump the TI's dag_version
 # to a newer version while the run stays pinned; sourcing from created_dag_version
 # keeps the shipped hash and manifest consistent so versioned bundles stay reproducible.
 version_data=_resolve_version_data(ti.dag_run.created_dag_version, ti.dag_run.bundle_version),
 )
 fname = log_filename_template_renderer()(ti=ti)

Expand Down

17 changes: 11 additions & 6 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -684,12 +684,17 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -
 ranked_query.c.map_index_for_ordering,
 )
 .options(selectinload(TI.dag_model))
 # Eager-load dag_version: TIs become transient (via make_transient) before
 # ExecuteTask.make() reads ti.dag_version.version_data. Lazy loads on
 # transient objects silently return None instead of raising DetachedInstanceError.
 # Scope the second SELECT to version_data (the PK is auto-included) so we read
 # two columns rather than the full DagVersion row.
 .options(selectinload(TI.dag_version).load_only(DagVersion.version_data))
 # Eager-load the run's pinned DagVersion (dag_run.created_dag_version): TIs become
 # transient (via make_transient) before ExecuteTask.make() reads
 # ti.dag_run.created_dag_version.version_data to ship the bundle manifest matching
 # the run's pinned bundle_version. Lazy loads on transient objects silently return
 # None instead of raising DetachedInstanceError. Scope the SELECT to version_data
 # (the PK is auto-included) so we read two columns rather than the full row.
 .options(
 joinedload(TI.dag_run)
 .selectinload(DagRun.created_dag_version)
 .load_only(DagVersion.version_data)
 )
 )

 query = query.limit(max_tis)
Expand Down

55 changes: 43 additions & 12 deletions
airflow-core/tests/unit/executors/test_workloads.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,12 +184,21 @@ def _stub_log_template(self, monkeypatch):
 )

 @staticmethod
 def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):
 def _make_mock_ti(
 bundle_version,
 version_data,
 *,
 has_created_dag_version=True,
 ti_dag_version_data=None,
 ):
 """Build a mock TI with the attributes ExecuteTask.make() reads.

 ``has_dag_version`` controls whether the TI has an associated DagVersion
 (legacy/backfilled TIs may not), independently of ``version_data`` so the
 pin-guard can be exercised with version_data present on an unpinned run.
 ``version_data`` is the manifest on the run's pinned version
 (``dag_run.created_dag_version``) -- the source make() must use.
 ``ti_dag_version_data`` is the manifest on ``ti.dag_version``, which make()
 must IGNORE (it can diverge from the run's pin after a mid-run DAG re-parse).
 ``has_created_dag_version`` toggles whether the run has a pinned DagVersion
 (legacy/backfilled runs may not).
 """
 from unittest.mock import Mock

Expand All@@ -215,15 +224,18 @@ def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):

 ti.dag_run.bundle_version = bundle_version

 if has_dag_version:
 ti.dag_version.version_data = version_data
 # make() must source version_data from the run's pinned version, never ti.dag_version.
 ti.dag_version.version_data = ti_dag_version_data

 if has_created_dag_version:
 ti.dag_run.created_dag_version.version_data = version_data
 else:
 ti.dag_version = None
 ti.dag_run.created_dag_version = None

 return ti

 def test_pinned_run_populates_version_data(self):
 """When the run is pinned, version_data from dag_version flows to BundleInfo."""
 """When the run is pinned, version_data from the run's created_dag_version flows to BundleInfo."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version="abc123", version_data=version_data)

Expand All@@ -233,7 +245,7 @@ def test_pinned_run_populates_version_data(self):
 assert workload.bundle_info.version_data == version_data

 def test_unpinned_run_suppresses_present_version_data(self):
 """An unpinned run must not expose version_data even when the dag_version carries it."""
 """An unpinned run must not expose version_data even when created_dag_version carries it."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version=None, version_data=version_data)

Expand All@@ -242,15 +254,34 @@ def test_unpinned_run_suppresses_present_version_data(self):
 assert workload.bundle_info.version is None
 assert workload.bundle_info.version_data is None

 def test_missing_dag_version_yields_none(self):
 """A pinned run whose TI has no dag_version (legacy/backfilled) yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_dag_version=False)
 def test_missing_created_dag_version_yields_none(self):
 """A pinned run whose DagRun has no created_dag_version yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_created_dag_version=False)

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "abc123"
 assert workload.bundle_info.version_data is None

 def test_mid_run_dag_version_bump_uses_run_pinned_manifest(self):
 """Regression: a mid-run DAG re-parse can bump ti.dag_version to a newer version while the
 run stays pinned. make() must ship the run's pinned manifest (created_dag_version), not the
 TI's bumped one -- otherwise a versioned bundle would fetch the wrong (latest) code.
 """
 run_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v1-object-id"}}
 bumped_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v2-object-id"}}
 ti = self._make_mock_ti(
 bundle_version="v1hash",
 version_data=run_manifest,
 ti_dag_version_data=bumped_manifest,
 )

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "v1hash"
 assert workload.bundle_info.version_data == run_manifest
 assert workload.bundle_info.version_data != bumped_manifest


class TestExecuteCallbackMakeVersionData:
 """Tests for ExecuteCallback.make() threading version_data through BundleInfo."""
Expand Down

37 changes: 36 additions & 1 deletion
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -142,7 +142,7 @@
from airflow.utils.types import DagRunTriggeredByType, DagRunType

from tests_common.pytest_plugin import AIRFLOW_ROOT_PATH
from tests_common.test_utils.asserts import assert_queries_count
from tests_common.test_utils.asserts import assert_queries_count, count_queries
from tests_common.test_utils.config import conf_vars, env_vars
from tests_common.test_utils.dag import create_scheduler_dag, sync_dag_to_db, sync_dags_to_db
from tests_common.test_utils.db import (
Expand DownExpand Up@@ -1425,6 +1425,41 @@ def test_find_executable_task_instances_backfill(self, dag_maker):
 assert {x.key for x in queued_tis} == {ti_non_backfill.key, ti_backfill.key}
 session.rollback()

 def test_executable_task_instances_no_per_ti_queries(self, dag_maker, session):
 """Guard against an N+1 when enqueuing task instances.

 ``ExecuteTask.make()`` reads ``ti.dag_run.created_dag_version.version_data`` to ship the
 run's pinned bundle manifest. ``dag_run`` is eager-joined and ``created_dag_version`` is a
 single batched ``selectin``, so the number of queries in
 ``_executable_task_instances_to_queued`` must be independent of how many task instances are
 in the batch. If a future change lazy-loads ``dag_run``/``created_dag_version`` per TI, the
 count would scale with the task count and this test fails.
 """
 scheduler_job = Job()
 runner = SchedulerJobRunner(job=scheduler_job)
 self.job_runner = runner

 def _measure(dag_id: str, num_tasks: int) -> int:
 with dag_maker(dag_id=dag_id, max_active_tasks=64, session=session):
 for i in range(num_tasks):
 EmptyOperator(task_id=f"t{i}")
 dr = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
 for ti in dr.task_instances:
 ti.state = State.SCHEDULED
 session.flush()
 with count_queries(session=session) as result:
 runner._executable_task_instances_to_queued(max_tis=64, session=session)
 session.rollback()
 return sum(result.values())

 one_task = _measure("q_count_one", 1)
 many_tasks = _measure("q_count_many", 10)

 assert one_task == many_tasks, (
 f"query count scaled with task-instance count ({one_task} -> {many_tasks}); "
 "likely a per-TI lazy load (N+1) of dag_run/created_dag_version"
 )

 def test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query(self, dag_maker, session):
 dag_id = "SchedulerJobTest.test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query"
 task_id = "dummy"
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/batch/test_batch_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -817,6 +817,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 if not AIRFLOW_V_3_0_PLUS:
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/ecs/test_ecs_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1312,6 +1312,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 # Mock command generation based on Airflow version
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Pin task bundle manifest to the dagrun's version#69941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

o-nikolas
merged 1 commit into

from

Jul 17, 2026
 Merged


Pin task bundle manifest to the dagrun's version
#69941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 5 additions & 1 deletion
airflow-core/src/airflow/executors/workloads/task.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,11 @@ def make(
 bundle_info = BundleInfo(
 name=ti.dag_model.bundle_name,
 version=ti.dag_run.bundle_version,
 version_data=_resolve_version_data(ti.dag_version, ti.dag_run.bundle_version),
 # Source version_data from the run's pinned version (matching ``version`` above),
 # not the TI's dag_version. A mid-run DAG re-parse can bump the TI's dag_version
 # to a newer version while the run stays pinned; sourcing from created_dag_version
 # keeps the shipped hash and manifest consistent so versioned bundles stay reproducible.
 version_data=_resolve_version_data(ti.dag_run.created_dag_version, ti.dag_run.bundle_version),
 )
 fname = log_filename_template_renderer()(ti=ti)

Expand Down

17 changes: 11 additions & 6 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -684,12 +684,17 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -
 ranked_query.c.map_index_for_ordering,
 )
 .options(selectinload(TI.dag_model))
 # Eager-load dag_version: TIs become transient (via make_transient) before
 # ExecuteTask.make() reads ti.dag_version.version_data. Lazy loads on
 # transient objects silently return None instead of raising DetachedInstanceError.
 # Scope the second SELECT to version_data (the PK is auto-included) so we read
 # two columns rather than the full DagVersion row.
 .options(selectinload(TI.dag_version).load_only(DagVersion.version_data))
 # Eager-load the run's pinned DagVersion (dag_run.created_dag_version): TIs become
 # transient (via make_transient) before ExecuteTask.make() reads
 # ti.dag_run.created_dag_version.version_data to ship the bundle manifest matching
 # the run's pinned bundle_version. Lazy loads on transient objects silently return
 # None instead of raising DetachedInstanceError. Scope the SELECT to version_data
 # (the PK is auto-included) so we read two columns rather than the full row.
 .options(
 joinedload(TI.dag_run)
 .selectinload(DagRun.created_dag_version)
 .load_only(DagVersion.version_data)
 )
 )

 query = query.limit(max_tis)
Expand Down

55 changes: 43 additions & 12 deletions
airflow-core/tests/unit/executors/test_workloads.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,12 +184,21 @@ def _stub_log_template(self, monkeypatch):
 )

 @staticmethod
 def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):
 def _make_mock_ti(
 bundle_version,
 version_data,
 *,
 has_created_dag_version=True,
 ti_dag_version_data=None,
 ):
 """Build a mock TI with the attributes ExecuteTask.make() reads.

 ``has_dag_version`` controls whether the TI has an associated DagVersion
 (legacy/backfilled TIs may not), independently of ``version_data`` so the
 pin-guard can be exercised with version_data present on an unpinned run.
 ``version_data`` is the manifest on the run's pinned version
 (``dag_run.created_dag_version``) -- the source make() must use.
 ``ti_dag_version_data`` is the manifest on ``ti.dag_version``, which make()
 must IGNORE (it can diverge from the run's pin after a mid-run DAG re-parse).
 ``has_created_dag_version`` toggles whether the run has a pinned DagVersion
 (legacy/backfilled runs may not).
 """
 from unittest.mock import Mock

Expand All@@ -215,15 +224,18 @@ def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):

 ti.dag_run.bundle_version = bundle_version

 if has_dag_version:
 ti.dag_version.version_data = version_data
 # make() must source version_data from the run's pinned version, never ti.dag_version.
 ti.dag_version.version_data = ti_dag_version_data

 if has_created_dag_version:
 ti.dag_run.created_dag_version.version_data = version_data
 else:
 ti.dag_version = None
 ti.dag_run.created_dag_version = None

 return ti

 def test_pinned_run_populates_version_data(self):
 """When the run is pinned, version_data from dag_version flows to BundleInfo."""
 """When the run is pinned, version_data from the run's created_dag_version flows to BundleInfo."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version="abc123", version_data=version_data)

Expand All@@ -233,7 +245,7 @@ def test_pinned_run_populates_version_data(self):
 assert workload.bundle_info.version_data == version_data

 def test_unpinned_run_suppresses_present_version_data(self):
 """An unpinned run must not expose version_data even when the dag_version carries it."""
 """An unpinned run must not expose version_data even when created_dag_version carries it."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version=None, version_data=version_data)

Expand All@@ -242,15 +254,34 @@ def test_unpinned_run_suppresses_present_version_data(self):
 assert workload.bundle_info.version is None
 assert workload.bundle_info.version_data is None

 def test_missing_dag_version_yields_none(self):
 """A pinned run whose TI has no dag_version (legacy/backfilled) yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_dag_version=False)
 def test_missing_created_dag_version_yields_none(self):
 """A pinned run whose DagRun has no created_dag_version yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_created_dag_version=False)

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "abc123"
 assert workload.bundle_info.version_data is None

 def test_mid_run_dag_version_bump_uses_run_pinned_manifest(self):
 """Regression: a mid-run DAG re-parse can bump ti.dag_version to a newer version while the
 run stays pinned. make() must ship the run's pinned manifest (created_dag_version), not the
 TI's bumped one -- otherwise a versioned bundle would fetch the wrong (latest) code.
 """
 run_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v1-object-id"}}
 bumped_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v2-object-id"}}
 ti = self._make_mock_ti(
 bundle_version="v1hash",
 version_data=run_manifest,
 ti_dag_version_data=bumped_manifest,
 )

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "v1hash"
 assert workload.bundle_info.version_data == run_manifest
 assert workload.bundle_info.version_data != bumped_manifest


class TestExecuteCallbackMakeVersionData:
 """Tests for ExecuteCallback.make() threading version_data through BundleInfo."""
Expand Down

37 changes: 36 additions & 1 deletion
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -142,7 +142,7 @@
from airflow.utils.types import DagRunTriggeredByType, DagRunType

from tests_common.pytest_plugin import AIRFLOW_ROOT_PATH
from tests_common.test_utils.asserts import assert_queries_count
from tests_common.test_utils.asserts import assert_queries_count, count_queries
from tests_common.test_utils.config import conf_vars, env_vars
from tests_common.test_utils.dag import create_scheduler_dag, sync_dag_to_db, sync_dags_to_db
from tests_common.test_utils.db import (
Expand DownExpand Up@@ -1425,6 +1425,41 @@ def test_find_executable_task_instances_backfill(self, dag_maker):
 assert {x.key for x in queued_tis} == {ti_non_backfill.key, ti_backfill.key}
 session.rollback()

 def test_executable_task_instances_no_per_ti_queries(self, dag_maker, session):
 """Guard against an N+1 when enqueuing task instances.

 ``ExecuteTask.make()`` reads ``ti.dag_run.created_dag_version.version_data`` to ship the
 run's pinned bundle manifest. ``dag_run`` is eager-joined and ``created_dag_version`` is a
 single batched ``selectin``, so the number of queries in
 ``_executable_task_instances_to_queued`` must be independent of how many task instances are
 in the batch. If a future change lazy-loads ``dag_run``/``created_dag_version`` per TI, the
 count would scale with the task count and this test fails.
 """
 scheduler_job = Job()
 runner = SchedulerJobRunner(job=scheduler_job)
 self.job_runner = runner

 def _measure(dag_id: str, num_tasks: int) -> int:
 with dag_maker(dag_id=dag_id, max_active_tasks=64, session=session):
 for i in range(num_tasks):
 EmptyOperator(task_id=f"t{i}")
 dr = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
 for ti in dr.task_instances:
 ti.state = State.SCHEDULED
 session.flush()
 with count_queries(session=session) as result:
 runner._executable_task_instances_to_queued(max_tis=64, session=session)
 session.rollback()
 return sum(result.values())

 one_task = _measure("q_count_one", 1)
 many_tasks = _measure("q_count_many", 10)

 assert one_task == many_tasks, (
 f"query count scaled with task-instance count ({one_task} -> {many_tasks}); "
 "likely a per-TI lazy load (N+1) of dag_run/created_dag_version"
 )

 def test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query(self, dag_maker, session):
 dag_id = "SchedulerJobTest.test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query"
 task_id = "dummy"
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/batch/test_batch_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -817,6 +817,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 if not AIRFLOW_V_3_0_PLUS:
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/ecs/test_ecs_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1312,6 +1312,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 # Mock command generation based on Airflow version
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Pin task bundle manifest to the dagrun's version#69941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

o-nikolas
merged 1 commit into

from

Jul 17, 2026
 Merged


Pin task bundle manifest to the dagrun's version
#69941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 5 additions & 1 deletion
airflow-core/src/airflow/executors/workloads/task.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,11 @@ def make(
 bundle_info = BundleInfo(
 name=ti.dag_model.bundle_name,
 version=ti.dag_run.bundle_version,
 version_data=_resolve_version_data(ti.dag_version, ti.dag_run.bundle_version),
 # Source version_data from the run's pinned version (matching ``version`` above),
 # not the TI's dag_version. A mid-run DAG re-parse can bump the TI's dag_version
 # to a newer version while the run stays pinned; sourcing from created_dag_version
 # keeps the shipped hash and manifest consistent so versioned bundles stay reproducible.
 version_data=_resolve_version_data(ti.dag_run.created_dag_version, ti.dag_run.bundle_version),
 )
 fname = log_filename_template_renderer()(ti=ti)

Expand Down

17 changes: 11 additions & 6 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -684,12 +684,17 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -
 ranked_query.c.map_index_for_ordering,
 )
 .options(selectinload(TI.dag_model))
 # Eager-load dag_version: TIs become transient (via make_transient) before
 # ExecuteTask.make() reads ti.dag_version.version_data. Lazy loads on
 # transient objects silently return None instead of raising DetachedInstanceError.
 # Scope the second SELECT to version_data (the PK is auto-included) so we read
 # two columns rather than the full DagVersion row.
 .options(selectinload(TI.dag_version).load_only(DagVersion.version_data))
 # Eager-load the run's pinned DagVersion (dag_run.created_dag_version): TIs become
 # transient (via make_transient) before ExecuteTask.make() reads
 # ti.dag_run.created_dag_version.version_data to ship the bundle manifest matching
 # the run's pinned bundle_version. Lazy loads on transient objects silently return
 # None instead of raising DetachedInstanceError. Scope the SELECT to version_data
 # (the PK is auto-included) so we read two columns rather than the full row.
 .options(
 joinedload(TI.dag_run)
 .selectinload(DagRun.created_dag_version)
 .load_only(DagVersion.version_data)
 )
 )

 query = query.limit(max_tis)
Expand Down

55 changes: 43 additions & 12 deletions
airflow-core/tests/unit/executors/test_workloads.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,12 +184,21 @@ def _stub_log_template(self, monkeypatch):
 )

 @staticmethod
 def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):
 def _make_mock_ti(
 bundle_version,
 version_data,
 *,
 has_created_dag_version=True,
 ti_dag_version_data=None,
 ):
 """Build a mock TI with the attributes ExecuteTask.make() reads.

 ``has_dag_version`` controls whether the TI has an associated DagVersion
 (legacy/backfilled TIs may not), independently of ``version_data`` so the
 pin-guard can be exercised with version_data present on an unpinned run.
 ``version_data`` is the manifest on the run's pinned version
 (``dag_run.created_dag_version``) -- the source make() must use.
 ``ti_dag_version_data`` is the manifest on ``ti.dag_version``, which make()
 must IGNORE (it can diverge from the run's pin after a mid-run DAG re-parse).
 ``has_created_dag_version`` toggles whether the run has a pinned DagVersion
 (legacy/backfilled runs may not).
 """
 from unittest.mock import Mock

Expand All@@ -215,15 +224,18 @@ def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):

 ti.dag_run.bundle_version = bundle_version

 if has_dag_version:
 ti.dag_version.version_data = version_data
 # make() must source version_data from the run's pinned version, never ti.dag_version.
 ti.dag_version.version_data = ti_dag_version_data

 if has_created_dag_version:
 ti.dag_run.created_dag_version.version_data = version_data
 else:
 ti.dag_version = None
 ti.dag_run.created_dag_version = None

 return ti

 def test_pinned_run_populates_version_data(self):
 """When the run is pinned, version_data from dag_version flows to BundleInfo."""
 """When the run is pinned, version_data from the run's created_dag_version flows to BundleInfo."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version="abc123", version_data=version_data)

Expand All@@ -233,7 +245,7 @@ def test_pinned_run_populates_version_data(self):
 assert workload.bundle_info.version_data == version_data

 def test_unpinned_run_suppresses_present_version_data(self):
 """An unpinned run must not expose version_data even when the dag_version carries it."""
 """An unpinned run must not expose version_data even when created_dag_version carries it."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version=None, version_data=version_data)

Expand All@@ -242,15 +254,34 @@ def test_unpinned_run_suppresses_present_version_data(self):
 assert workload.bundle_info.version is None
 assert workload.bundle_info.version_data is None

 def test_missing_dag_version_yields_none(self):
 """A pinned run whose TI has no dag_version (legacy/backfilled) yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_dag_version=False)
 def test_missing_created_dag_version_yields_none(self):
 """A pinned run whose DagRun has no created_dag_version yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_created_dag_version=False)

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "abc123"
 assert workload.bundle_info.version_data is None

 def test_mid_run_dag_version_bump_uses_run_pinned_manifest(self):
 """Regression: a mid-run DAG re-parse can bump ti.dag_version to a newer version while the
 run stays pinned. make() must ship the run's pinned manifest (created_dag_version), not the
 TI's bumped one -- otherwise a versioned bundle would fetch the wrong (latest) code.
 """
 run_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v1-object-id"}}
 bumped_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v2-object-id"}}
 ti = self._make_mock_ti(
 bundle_version="v1hash",
 version_data=run_manifest,
 ti_dag_version_data=bumped_manifest,
 )

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "v1hash"
 assert workload.bundle_info.version_data == run_manifest
 assert workload.bundle_info.version_data != bumped_manifest


class TestExecuteCallbackMakeVersionData:
 """Tests for ExecuteCallback.make() threading version_data through BundleInfo."""
Expand Down

37 changes: 36 additions & 1 deletion
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -142,7 +142,7 @@
from airflow.utils.types import DagRunTriggeredByType, DagRunType

from tests_common.pytest_plugin import AIRFLOW_ROOT_PATH
from tests_common.test_utils.asserts import assert_queries_count
from tests_common.test_utils.asserts import assert_queries_count, count_queries
from tests_common.test_utils.config import conf_vars, env_vars
from tests_common.test_utils.dag import create_scheduler_dag, sync_dag_to_db, sync_dags_to_db
from tests_common.test_utils.db import (
Expand DownExpand Up@@ -1425,6 +1425,41 @@ def test_find_executable_task_instances_backfill(self, dag_maker):
 assert {x.key for x in queued_tis} == {ti_non_backfill.key, ti_backfill.key}
 session.rollback()

 def test_executable_task_instances_no_per_ti_queries(self, dag_maker, session):
 """Guard against an N+1 when enqueuing task instances.

 ``ExecuteTask.make()`` reads ``ti.dag_run.created_dag_version.version_data`` to ship the
 run's pinned bundle manifest. ``dag_run`` is eager-joined and ``created_dag_version`` is a
 single batched ``selectin``, so the number of queries in
 ``_executable_task_instances_to_queued`` must be independent of how many task instances are
 in the batch. If a future change lazy-loads ``dag_run``/``created_dag_version`` per TI, the
 count would scale with the task count and this test fails.
 """
 scheduler_job = Job()
 runner = SchedulerJobRunner(job=scheduler_job)
 self.job_runner = runner

 def _measure(dag_id: str, num_tasks: int) -> int:
 with dag_maker(dag_id=dag_id, max_active_tasks=64, session=session):
 for i in range(num_tasks):
 EmptyOperator(task_id=f"t{i}")
 dr = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
 for ti in dr.task_instances:
 ti.state = State.SCHEDULED
 session.flush()
 with count_queries(session=session) as result:
 runner._executable_task_instances_to_queued(max_tis=64, session=session)
 session.rollback()
 return sum(result.values())

 one_task = _measure("q_count_one", 1)
 many_tasks = _measure("q_count_many", 10)

 assert one_task == many_tasks, (
 f"query count scaled with task-instance count ({one_task} -> {many_tasks}); "
 "likely a per-TI lazy load (N+1) of dag_run/created_dag_version"
 )

 def test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query(self, dag_maker, session):
 dag_id = "SchedulerJobTest.test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query"
 task_id = "dummy"
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/batch/test_batch_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -817,6 +817,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 if not AIRFLOW_V_3_0_PLUS:
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/ecs/test_ecs_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1312,6 +1312,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 # Mock command generation based on Airflow version
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Pin task bundle manifest to the dagrun's version#69941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

o-nikolas
merged 1 commit into

from

Jul 17, 2026
 Merged


Pin task bundle manifest to the dagrun's version
#69941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 5 additions & 1 deletion
airflow-core/src/airflow/executors/workloads/task.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,11 @@ def make(
 bundle_info = BundleInfo(
 name=ti.dag_model.bundle_name,
 version=ti.dag_run.bundle_version,
 version_data=_resolve_version_data(ti.dag_version, ti.dag_run.bundle_version),
 # Source version_data from the run's pinned version (matching ``version`` above),
 # not the TI's dag_version. A mid-run DAG re-parse can bump the TI's dag_version
 # to a newer version while the run stays pinned; sourcing from created_dag_version
 # keeps the shipped hash and manifest consistent so versioned bundles stay reproducible.
 version_data=_resolve_version_data(ti.dag_run.created_dag_version, ti.dag_run.bundle_version),
 )
 fname = log_filename_template_renderer()(ti=ti)

Expand Down

17 changes: 11 additions & 6 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -684,12 +684,17 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -
 ranked_query.c.map_index_for_ordering,
 )
 .options(selectinload(TI.dag_model))
 # Eager-load dag_version: TIs become transient (via make_transient) before
 # ExecuteTask.make() reads ti.dag_version.version_data. Lazy loads on
 # transient objects silently return None instead of raising DetachedInstanceError.
 # Scope the second SELECT to version_data (the PK is auto-included) so we read
 # two columns rather than the full DagVersion row.
 .options(selectinload(TI.dag_version).load_only(DagVersion.version_data))
 # Eager-load the run's pinned DagVersion (dag_run.created_dag_version): TIs become
 # transient (via make_transient) before ExecuteTask.make() reads
 # ti.dag_run.created_dag_version.version_data to ship the bundle manifest matching
 # the run's pinned bundle_version. Lazy loads on transient objects silently return
 # None instead of raising DetachedInstanceError. Scope the SELECT to version_data
 # (the PK is auto-included) so we read two columns rather than the full row.
 .options(
 joinedload(TI.dag_run)
 .selectinload(DagRun.created_dag_version)
 .load_only(DagVersion.version_data)
 )
 )

 query = query.limit(max_tis)
Expand Down

55 changes: 43 additions & 12 deletions
airflow-core/tests/unit/executors/test_workloads.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,12 +184,21 @@ def _stub_log_template(self, monkeypatch):
 )

 @staticmethod
 def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):
 def _make_mock_ti(
 bundle_version,
 version_data,
 *,
 has_created_dag_version=True,
 ti_dag_version_data=None,
 ):
 """Build a mock TI with the attributes ExecuteTask.make() reads.

 ``has_dag_version`` controls whether the TI has an associated DagVersion
 (legacy/backfilled TIs may not), independently of ``version_data`` so the
 pin-guard can be exercised with version_data present on an unpinned run.
 ``version_data`` is the manifest on the run's pinned version
 (``dag_run.created_dag_version``) -- the source make() must use.
 ``ti_dag_version_data`` is the manifest on ``ti.dag_version``, which make()
 must IGNORE (it can diverge from the run's pin after a mid-run DAG re-parse).
 ``has_created_dag_version`` toggles whether the run has a pinned DagVersion
 (legacy/backfilled runs may not).
 """
 from unittest.mock import Mock

Expand All@@ -215,15 +224,18 @@ def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):

 ti.dag_run.bundle_version = bundle_version

 if has_dag_version:
 ti.dag_version.version_data = version_data
 # make() must source version_data from the run's pinned version, never ti.dag_version.
 ti.dag_version.version_data = ti_dag_version_data

 if has_created_dag_version:
 ti.dag_run.created_dag_version.version_data = version_data
 else:
 ti.dag_version = None
 ti.dag_run.created_dag_version = None

 return ti

 def test_pinned_run_populates_version_data(self):
 """When the run is pinned, version_data from dag_version flows to BundleInfo."""
 """When the run is pinned, version_data from the run's created_dag_version flows to BundleInfo."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version="abc123", version_data=version_data)

Expand All@@ -233,7 +245,7 @@ def test_pinned_run_populates_version_data(self):
 assert workload.bundle_info.version_data == version_data

 def test_unpinned_run_suppresses_present_version_data(self):
 """An unpinned run must not expose version_data even when the dag_version carries it."""
 """An unpinned run must not expose version_data even when created_dag_version carries it."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version=None, version_data=version_data)

Expand All@@ -242,15 +254,34 @@ def test_unpinned_run_suppresses_present_version_data(self):
 assert workload.bundle_info.version is None
 assert workload.bundle_info.version_data is None

 def test_missing_dag_version_yields_none(self):
 """A pinned run whose TI has no dag_version (legacy/backfilled) yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_dag_version=False)
 def test_missing_created_dag_version_yields_none(self):
 """A pinned run whose DagRun has no created_dag_version yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_created_dag_version=False)

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "abc123"
 assert workload.bundle_info.version_data is None

 def test_mid_run_dag_version_bump_uses_run_pinned_manifest(self):
 """Regression: a mid-run DAG re-parse can bump ti.dag_version to a newer version while the
 run stays pinned. make() must ship the run's pinned manifest (created_dag_version), not the
 TI's bumped one -- otherwise a versioned bundle would fetch the wrong (latest) code.
 """
 run_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v1-object-id"}}
 bumped_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v2-object-id"}}
 ti = self._make_mock_ti(
 bundle_version="v1hash",
 version_data=run_manifest,
 ti_dag_version_data=bumped_manifest,
 )

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "v1hash"
 assert workload.bundle_info.version_data == run_manifest
 assert workload.bundle_info.version_data != bumped_manifest


class TestExecuteCallbackMakeVersionData:
 """Tests for ExecuteCallback.make() threading version_data through BundleInfo."""
Expand Down

37 changes: 36 additions & 1 deletion
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -142,7 +142,7 @@
from airflow.utils.types import DagRunTriggeredByType, DagRunType

from tests_common.pytest_plugin import AIRFLOW_ROOT_PATH
from tests_common.test_utils.asserts import assert_queries_count
from tests_common.test_utils.asserts import assert_queries_count, count_queries
from tests_common.test_utils.config import conf_vars, env_vars
from tests_common.test_utils.dag import create_scheduler_dag, sync_dag_to_db, sync_dags_to_db
from tests_common.test_utils.db import (
Expand DownExpand Up@@ -1425,6 +1425,41 @@ def test_find_executable_task_instances_backfill(self, dag_maker):
 assert {x.key for x in queued_tis} == {ti_non_backfill.key, ti_backfill.key}
 session.rollback()

 def test_executable_task_instances_no_per_ti_queries(self, dag_maker, session):
 """Guard against an N+1 when enqueuing task instances.

 ``ExecuteTask.make()`` reads ``ti.dag_run.created_dag_version.version_data`` to ship the
 run's pinned bundle manifest. ``dag_run`` is eager-joined and ``created_dag_version`` is a
 single batched ``selectin``, so the number of queries in
 ``_executable_task_instances_to_queued`` must be independent of how many task instances are
 in the batch. If a future change lazy-loads ``dag_run``/``created_dag_version`` per TI, the
 count would scale with the task count and this test fails.
 """
 scheduler_job = Job()
 runner = SchedulerJobRunner(job=scheduler_job)
 self.job_runner = runner

 def _measure(dag_id: str, num_tasks: int) -> int:
 with dag_maker(dag_id=dag_id, max_active_tasks=64, session=session):
 for i in range(num_tasks):
 EmptyOperator(task_id=f"t{i}")
 dr = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
 for ti in dr.task_instances:
 ti.state = State.SCHEDULED
 session.flush()
 with count_queries(session=session) as result:
 runner._executable_task_instances_to_queued(max_tis=64, session=session)
 session.rollback()
 return sum(result.values())

 one_task = _measure("q_count_one", 1)
 many_tasks = _measure("q_count_many", 10)

 assert one_task == many_tasks, (
 f"query count scaled with task-instance count ({one_task} -> {many_tasks}); "
 "likely a per-TI lazy load (N+1) of dag_run/created_dag_version"
 )

 def test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query(self, dag_maker, session):
 dag_id = "SchedulerJobTest.test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query"
 task_id = "dummy"
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/batch/test_batch_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -817,6 +817,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 if not AIRFLOW_V_3_0_PLUS:
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/ecs/test_ecs_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1312,6 +1312,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 # Mock command generation based on Airflow version
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Pin task bundle manifest to the dagrun's version#69941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

o-nikolas
merged 1 commit into

from

Jul 17, 2026
 Merged


Pin task bundle manifest to the dagrun's version
#69941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 5 additions & 1 deletion
airflow-core/src/airflow/executors/workloads/task.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,11 @@ def make(
 bundle_info = BundleInfo(
 name=ti.dag_model.bundle_name,
 version=ti.dag_run.bundle_version,
 version_data=_resolve_version_data(ti.dag_version, ti.dag_run.bundle_version),
 # Source version_data from the run's pinned version (matching ``version`` above),
 # not the TI's dag_version. A mid-run DAG re-parse can bump the TI's dag_version
 # to a newer version while the run stays pinned; sourcing from created_dag_version
 # keeps the shipped hash and manifest consistent so versioned bundles stay reproducible.
 version_data=_resolve_version_data(ti.dag_run.created_dag_version, ti.dag_run.bundle_version),
 )
 fname = log_filename_template_renderer()(ti=ti)

Expand Down

17 changes: 11 additions & 6 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -684,12 +684,17 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -
 ranked_query.c.map_index_for_ordering,
 )
 .options(selectinload(TI.dag_model))
 # Eager-load dag_version: TIs become transient (via make_transient) before
 # ExecuteTask.make() reads ti.dag_version.version_data. Lazy loads on
 # transient objects silently return None instead of raising DetachedInstanceError.
 # Scope the second SELECT to version_data (the PK is auto-included) so we read
 # two columns rather than the full DagVersion row.
 .options(selectinload(TI.dag_version).load_only(DagVersion.version_data))
 # Eager-load the run's pinned DagVersion (dag_run.created_dag_version): TIs become
 # transient (via make_transient) before ExecuteTask.make() reads
 # ti.dag_run.created_dag_version.version_data to ship the bundle manifest matching
 # the run's pinned bundle_version. Lazy loads on transient objects silently return
 # None instead of raising DetachedInstanceError. Scope the SELECT to version_data
 # (the PK is auto-included) so we read two columns rather than the full row.
 .options(
 joinedload(TI.dag_run)
 .selectinload(DagRun.created_dag_version)
 .load_only(DagVersion.version_data)
 )
 )

 query = query.limit(max_tis)
Expand Down

55 changes: 43 additions & 12 deletions
airflow-core/tests/unit/executors/test_workloads.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,12 +184,21 @@ def _stub_log_template(self, monkeypatch):
 )

 @staticmethod
 def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):
 def _make_mock_ti(
 bundle_version,
 version_data,
 *,
 has_created_dag_version=True,
 ti_dag_version_data=None,
 ):
 """Build a mock TI with the attributes ExecuteTask.make() reads.

 ``has_dag_version`` controls whether the TI has an associated DagVersion
 (legacy/backfilled TIs may not), independently of ``version_data`` so the
 pin-guard can be exercised with version_data present on an unpinned run.
 ``version_data`` is the manifest on the run's pinned version
 (``dag_run.created_dag_version``) -- the source make() must use.
 ``ti_dag_version_data`` is the manifest on ``ti.dag_version``, which make()
 must IGNORE (it can diverge from the run's pin after a mid-run DAG re-parse).
 ``has_created_dag_version`` toggles whether the run has a pinned DagVersion
 (legacy/backfilled runs may not).
 """
 from unittest.mock import Mock

Expand All@@ -215,15 +224,18 @@ def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):

 ti.dag_run.bundle_version = bundle_version

 if has_dag_version:
 ti.dag_version.version_data = version_data
 # make() must source version_data from the run's pinned version, never ti.dag_version.
 ti.dag_version.version_data = ti_dag_version_data

 if has_created_dag_version:
 ti.dag_run.created_dag_version.version_data = version_data
 else:
 ti.dag_version = None
 ti.dag_run.created_dag_version = None

 return ti

 def test_pinned_run_populates_version_data(self):
 """When the run is pinned, version_data from dag_version flows to BundleInfo."""
 """When the run is pinned, version_data from the run's created_dag_version flows to BundleInfo."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version="abc123", version_data=version_data)

Expand All@@ -233,7 +245,7 @@ def test_pinned_run_populates_version_data(self):
 assert workload.bundle_info.version_data == version_data

 def test_unpinned_run_suppresses_present_version_data(self):
 """An unpinned run must not expose version_data even when the dag_version carries it."""
 """An unpinned run must not expose version_data even when created_dag_version carries it."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version=None, version_data=version_data)

Expand All@@ -242,15 +254,34 @@ def test_unpinned_run_suppresses_present_version_data(self):
 assert workload.bundle_info.version is None
 assert workload.bundle_info.version_data is None

 def test_missing_dag_version_yields_none(self):
 """A pinned run whose TI has no dag_version (legacy/backfilled) yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_dag_version=False)
 def test_missing_created_dag_version_yields_none(self):
 """A pinned run whose DagRun has no created_dag_version yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_created_dag_version=False)

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "abc123"
 assert workload.bundle_info.version_data is None

 def test_mid_run_dag_version_bump_uses_run_pinned_manifest(self):
 """Regression: a mid-run DAG re-parse can bump ti.dag_version to a newer version while the
 run stays pinned. make() must ship the run's pinned manifest (created_dag_version), not the
 TI's bumped one -- otherwise a versioned bundle would fetch the wrong (latest) code.
 """
 run_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v1-object-id"}}
 bumped_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v2-object-id"}}
 ti = self._make_mock_ti(
 bundle_version="v1hash",
 version_data=run_manifest,
 ti_dag_version_data=bumped_manifest,
 )

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "v1hash"
 assert workload.bundle_info.version_data == run_manifest
 assert workload.bundle_info.version_data != bumped_manifest


class TestExecuteCallbackMakeVersionData:
 """Tests for ExecuteCallback.make() threading version_data through BundleInfo."""
Expand Down

37 changes: 36 additions & 1 deletion
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -142,7 +142,7 @@
from airflow.utils.types import DagRunTriggeredByType, DagRunType

from tests_common.pytest_plugin import AIRFLOW_ROOT_PATH
from tests_common.test_utils.asserts import assert_queries_count
from tests_common.test_utils.asserts import assert_queries_count, count_queries
from tests_common.test_utils.config import conf_vars, env_vars
from tests_common.test_utils.dag import create_scheduler_dag, sync_dag_to_db, sync_dags_to_db
from tests_common.test_utils.db import (
Expand DownExpand Up@@ -1425,6 +1425,41 @@ def test_find_executable_task_instances_backfill(self, dag_maker):
 assert {x.key for x in queued_tis} == {ti_non_backfill.key, ti_backfill.key}
 session.rollback()

 def test_executable_task_instances_no_per_ti_queries(self, dag_maker, session):
 """Guard against an N+1 when enqueuing task instances.

 ``ExecuteTask.make()`` reads ``ti.dag_run.created_dag_version.version_data`` to ship the
 run's pinned bundle manifest. ``dag_run`` is eager-joined and ``created_dag_version`` is a
 single batched ``selectin``, so the number of queries in
 ``_executable_task_instances_to_queued`` must be independent of how many task instances are
 in the batch. If a future change lazy-loads ``dag_run``/``created_dag_version`` per TI, the
 count would scale with the task count and this test fails.
 """
 scheduler_job = Job()
 runner = SchedulerJobRunner(job=scheduler_job)
 self.job_runner = runner

 def _measure(dag_id: str, num_tasks: int) -> int:
 with dag_maker(dag_id=dag_id, max_active_tasks=64, session=session):
 for i in range(num_tasks):
 EmptyOperator(task_id=f"t{i}")
 dr = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
 for ti in dr.task_instances:
 ti.state = State.SCHEDULED
 session.flush()
 with count_queries(session=session) as result:
 runner._executable_task_instances_to_queued(max_tis=64, session=session)
 session.rollback()
 return sum(result.values())

 one_task = _measure("q_count_one", 1)
 many_tasks = _measure("q_count_many", 10)

 assert one_task == many_tasks, (
 f"query count scaled with task-instance count ({one_task} -> {many_tasks}); "
 "likely a per-TI lazy load (N+1) of dag_run/created_dag_version"
 )

 def test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query(self, dag_maker, session):
 dag_id = "SchedulerJobTest.test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query"
 task_id = "dummy"
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/batch/test_batch_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -817,6 +817,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 if not AIRFLOW_V_3_0_PLUS:
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/ecs/test_ecs_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1312,6 +1312,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 # Mock command generation based on Airflow version
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Pin task bundle manifest to the dagrun's version#69941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

o-nikolas
merged 1 commit into

from

Jul 17, 2026
 Merged


Pin task bundle manifest to the dagrun's version
#69941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 5 additions & 1 deletion
airflow-core/src/airflow/executors/workloads/task.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,11 @@ def make(
 bundle_info = BundleInfo(
 name=ti.dag_model.bundle_name,
 version=ti.dag_run.bundle_version,
 version_data=_resolve_version_data(ti.dag_version, ti.dag_run.bundle_version),
 # Source version_data from the run's pinned version (matching ``version`` above),
 # not the TI's dag_version. A mid-run DAG re-parse can bump the TI's dag_version
 # to a newer version while the run stays pinned; sourcing from created_dag_version
 # keeps the shipped hash and manifest consistent so versioned bundles stay reproducible.
 version_data=_resolve_version_data(ti.dag_run.created_dag_version, ti.dag_run.bundle_version),
 )
 fname = log_filename_template_renderer()(ti=ti)

Expand Down

17 changes: 11 additions & 6 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -684,12 +684,17 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -
 ranked_query.c.map_index_for_ordering,
 )
 .options(selectinload(TI.dag_model))
 # Eager-load dag_version: TIs become transient (via make_transient) before
 # ExecuteTask.make() reads ti.dag_version.version_data. Lazy loads on
 # transient objects silently return None instead of raising DetachedInstanceError.
 # Scope the second SELECT to version_data (the PK is auto-included) so we read
 # two columns rather than the full DagVersion row.
 .options(selectinload(TI.dag_version).load_only(DagVersion.version_data))
 # Eager-load the run's pinned DagVersion (dag_run.created_dag_version): TIs become
 # transient (via make_transient) before ExecuteTask.make() reads
 # ti.dag_run.created_dag_version.version_data to ship the bundle manifest matching
 # the run's pinned bundle_version. Lazy loads on transient objects silently return
 # None instead of raising DetachedInstanceError. Scope the SELECT to version_data
 # (the PK is auto-included) so we read two columns rather than the full row.
 .options(
 joinedload(TI.dag_run)
 .selectinload(DagRun.created_dag_version)
 .load_only(DagVersion.version_data)
 )
 )

 query = query.limit(max_tis)
Expand Down

55 changes: 43 additions & 12 deletions
airflow-core/tests/unit/executors/test_workloads.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,12 +184,21 @@ def _stub_log_template(self, monkeypatch):
 )

 @staticmethod
 def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):
 def _make_mock_ti(
 bundle_version,
 version_data,
 *,
 has_created_dag_version=True,
 ti_dag_version_data=None,
 ):
 """Build a mock TI with the attributes ExecuteTask.make() reads.

 ``has_dag_version`` controls whether the TI has an associated DagVersion
 (legacy/backfilled TIs may not), independently of ``version_data`` so the
 pin-guard can be exercised with version_data present on an unpinned run.
 ``version_data`` is the manifest on the run's pinned version
 (``dag_run.created_dag_version``) -- the source make() must use.
 ``ti_dag_version_data`` is the manifest on ``ti.dag_version``, which make()
 must IGNORE (it can diverge from the run's pin after a mid-run DAG re-parse).
 ``has_created_dag_version`` toggles whether the run has a pinned DagVersion
 (legacy/backfilled runs may not).
 """
 from unittest.mock import Mock

Expand All@@ -215,15 +224,18 @@ def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):

 ti.dag_run.bundle_version = bundle_version

 if has_dag_version:
 ti.dag_version.version_data = version_data
 # make() must source version_data from the run's pinned version, never ti.dag_version.
 ti.dag_version.version_data = ti_dag_version_data

 if has_created_dag_version:
 ti.dag_run.created_dag_version.version_data = version_data
 else:
 ti.dag_version = None
 ti.dag_run.created_dag_version = None

 return ti

 def test_pinned_run_populates_version_data(self):
 """When the run is pinned, version_data from dag_version flows to BundleInfo."""
 """When the run is pinned, version_data from the run's created_dag_version flows to BundleInfo."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version="abc123", version_data=version_data)

Expand All@@ -233,7 +245,7 @@ def test_pinned_run_populates_version_data(self):
 assert workload.bundle_info.version_data == version_data

 def test_unpinned_run_suppresses_present_version_data(self):
 """An unpinned run must not expose version_data even when the dag_version carries it."""
 """An unpinned run must not expose version_data even when created_dag_version carries it."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version=None, version_data=version_data)

Expand All@@ -242,15 +254,34 @@ def test_unpinned_run_suppresses_present_version_data(self):
 assert workload.bundle_info.version is None
 assert workload.bundle_info.version_data is None

 def test_missing_dag_version_yields_none(self):
 """A pinned run whose TI has no dag_version (legacy/backfilled) yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_dag_version=False)
 def test_missing_created_dag_version_yields_none(self):
 """A pinned run whose DagRun has no created_dag_version yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_created_dag_version=False)

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "abc123"
 assert workload.bundle_info.version_data is None

 def test_mid_run_dag_version_bump_uses_run_pinned_manifest(self):
 """Regression: a mid-run DAG re-parse can bump ti.dag_version to a newer version while the
 run stays pinned. make() must ship the run's pinned manifest (created_dag_version), not the
 TI's bumped one -- otherwise a versioned bundle would fetch the wrong (latest) code.
 """
 run_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v1-object-id"}}
 bumped_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v2-object-id"}}
 ti = self._make_mock_ti(
 bundle_version="v1hash",
 version_data=run_manifest,
 ti_dag_version_data=bumped_manifest,
 )

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "v1hash"
 assert workload.bundle_info.version_data == run_manifest
 assert workload.bundle_info.version_data != bumped_manifest


class TestExecuteCallbackMakeVersionData:
 """Tests for ExecuteCallback.make() threading version_data through BundleInfo."""
Expand Down

37 changes: 36 additions & 1 deletion
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -142,7 +142,7 @@
from airflow.utils.types import DagRunTriggeredByType, DagRunType

from tests_common.pytest_plugin import AIRFLOW_ROOT_PATH
from tests_common.test_utils.asserts import assert_queries_count
from tests_common.test_utils.asserts import assert_queries_count, count_queries
from tests_common.test_utils.config import conf_vars, env_vars
from tests_common.test_utils.dag import create_scheduler_dag, sync_dag_to_db, sync_dags_to_db
from tests_common.test_utils.db import (
Expand DownExpand Up@@ -1425,6 +1425,41 @@ def test_find_executable_task_instances_backfill(self, dag_maker):
 assert {x.key for x in queued_tis} == {ti_non_backfill.key, ti_backfill.key}
 session.rollback()

 def test_executable_task_instances_no_per_ti_queries(self, dag_maker, session):
 """Guard against an N+1 when enqueuing task instances.

 ``ExecuteTask.make()`` reads ``ti.dag_run.created_dag_version.version_data`` to ship the
 run's pinned bundle manifest. ``dag_run`` is eager-joined and ``created_dag_version`` is a
 single batched ``selectin``, so the number of queries in
 ``_executable_task_instances_to_queued`` must be independent of how many task instances are
 in the batch. If a future change lazy-loads ``dag_run``/``created_dag_version`` per TI, the
 count would scale with the task count and this test fails.
 """
 scheduler_job = Job()
 runner = SchedulerJobRunner(job=scheduler_job)
 self.job_runner = runner

 def _measure(dag_id: str, num_tasks: int) -> int:
 with dag_maker(dag_id=dag_id, max_active_tasks=64, session=session):
 for i in range(num_tasks):
 EmptyOperator(task_id=f"t{i}")
 dr = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
 for ti in dr.task_instances:
 ti.state = State.SCHEDULED
 session.flush()
 with count_queries(session=session) as result:
 runner._executable_task_instances_to_queued(max_tis=64, session=session)
 session.rollback()
 return sum(result.values())

 one_task = _measure("q_count_one", 1)
 many_tasks = _measure("q_count_many", 10)

 assert one_task == many_tasks, (
 f"query count scaled with task-instance count ({one_task} -> {many_tasks}); "
 "likely a per-TI lazy load (N+1) of dag_run/created_dag_version"
 )

 def test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query(self, dag_maker, session):
 dag_id = "SchedulerJobTest.test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query"
 task_id = "dummy"
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/batch/test_batch_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -817,6 +817,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 if not AIRFLOW_V_3_0_PLUS:
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/ecs/test_ecs_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1312,6 +1312,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 # Mock command generation based on Airflow version
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Pin task bundle manifest to the dagrun's version#69941

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

o-nikolas
merged 1 commit into

from

Jul 17, 2026
 Merged


Pin task bundle manifest to the dagrun's version
#69941
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


6 changes: 5 additions & 1 deletion
airflow-core/src/airflow/executors/workloads/task.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -107,7 +107,11 @@ def make(
 bundle_info = BundleInfo(
 name=ti.dag_model.bundle_name,
 version=ti.dag_run.bundle_version,
 version_data=_resolve_version_data(ti.dag_version, ti.dag_run.bundle_version),
 # Source version_data from the run's pinned version (matching ``version`` above),
 # not the TI's dag_version. A mid-run DAG re-parse can bump the TI's dag_version
 # to a newer version while the run stays pinned; sourcing from created_dag_version
 # keeps the shipped hash and manifest consistent so versioned bundles stay reproducible.
 version_data=_resolve_version_data(ti.dag_run.created_dag_version, ti.dag_run.bundle_version),
 )
 fname = log_filename_template_renderer()(ti=ti)

Expand Down

17 changes: 11 additions & 6 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -684,12 +684,17 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -
 ranked_query.c.map_index_for_ordering,
 )
 .options(selectinload(TI.dag_model))
 # Eager-load dag_version: TIs become transient (via make_transient) before
 # ExecuteTask.make() reads ti.dag_version.version_data. Lazy loads on
 # transient objects silently return None instead of raising DetachedInstanceError.
 # Scope the second SELECT to version_data (the PK is auto-included) so we read
 # two columns rather than the full DagVersion row.
 .options(selectinload(TI.dag_version).load_only(DagVersion.version_data))
 # Eager-load the run's pinned DagVersion (dag_run.created_dag_version): TIs become
 # transient (via make_transient) before ExecuteTask.make() reads
 # ti.dag_run.created_dag_version.version_data to ship the bundle manifest matching
 # the run's pinned bundle_version. Lazy loads on transient objects silently return
 # None instead of raising DetachedInstanceError. Scope the SELECT to version_data
 # (the PK is auto-included) so we read two columns rather than the full row.
 .options(
 joinedload(TI.dag_run)
 .selectinload(DagRun.created_dag_version)
 .load_only(DagVersion.version_data)
 )
 )

 query = query.limit(max_tis)
Expand Down

55 changes: 43 additions & 12 deletions
airflow-core/tests/unit/executors/test_workloads.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -184,12 +184,21 @@ def _stub_log_template(self, monkeypatch):
 )

 @staticmethod
 def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):
 def _make_mock_ti(
 bundle_version,
 version_data,
 *,
 has_created_dag_version=True,
 ti_dag_version_data=None,
 ):
 """Build a mock TI with the attributes ExecuteTask.make() reads.

 ``has_dag_version`` controls whether the TI has an associated DagVersion
 (legacy/backfilled TIs may not), independently of ``version_data`` so the
 pin-guard can be exercised with version_data present on an unpinned run.
 ``version_data`` is the manifest on the run's pinned version
 (``dag_run.created_dag_version``) -- the source make() must use.
 ``ti_dag_version_data`` is the manifest on ``ti.dag_version``, which make()
 must IGNORE (it can diverge from the run's pin after a mid-run DAG re-parse).
 ``has_created_dag_version`` toggles whether the run has a pinned DagVersion
 (legacy/backfilled runs may not).
 """
 from unittest.mock import Mock

Expand All@@ -215,15 +224,18 @@ def _make_mock_ti(bundle_version, version_data, *, has_dag_version=True):

 ti.dag_run.bundle_version = bundle_version

 if has_dag_version:
 ti.dag_version.version_data = version_data
 # make() must source version_data from the run's pinned version, never ti.dag_version.
 ti.dag_version.version_data = ti_dag_version_data

 if has_created_dag_version:
 ti.dag_run.created_dag_version.version_data = version_data
 else:
 ti.dag_version = None
 ti.dag_run.created_dag_version = None

 return ti

 def test_pinned_run_populates_version_data(self):
 """When the run is pinned, version_data from dag_version flows to BundleInfo."""
 """When the run is pinned, version_data from the run's created_dag_version flows to BundleInfo."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version="abc123", version_data=version_data)

Expand All@@ -233,7 +245,7 @@ def test_pinned_run_populates_version_data(self):
 assert workload.bundle_info.version_data == version_data

 def test_unpinned_run_suppresses_present_version_data(self):
 """An unpinned run must not expose version_data even when the dag_version carries it."""
 """An unpinned run must not expose version_data even when created_dag_version carries it."""
 version_data = {"schema_version": 1, "files": {"dags/my_dag.py": "ver123"}}
 ti = self._make_mock_ti(bundle_version=None, version_data=version_data)

Expand All@@ -242,15 +254,34 @@ def test_unpinned_run_suppresses_present_version_data(self):
 assert workload.bundle_info.version is None
 assert workload.bundle_info.version_data is None

 def test_missing_dag_version_yields_none(self):
 """A pinned run whose TI has no dag_version (legacy/backfilled) yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_dag_version=False)
 def test_missing_created_dag_version_yields_none(self):
 """A pinned run whose DagRun has no created_dag_version yields no version_data."""
 ti = self._make_mock_ti(bundle_version="abc123", version_data=None, has_created_dag_version=False)

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "abc123"
 assert workload.bundle_info.version_data is None

 def test_mid_run_dag_version_bump_uses_run_pinned_manifest(self):
 """Regression: a mid-run DAG re-parse can bump ti.dag_version to a newer version while the
 run stays pinned. make() must ship the run's pinned manifest (created_dag_version), not the
 TI's bumped one -- otherwise a versioned bundle would fetch the wrong (latest) code.
 """
 run_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v1-object-id"}}
 bumped_manifest = {"schema_version": 1, "files": {"dags/my_dag.py": "v2-object-id"}}
 ti = self._make_mock_ti(
 bundle_version="v1hash",
 version_data=run_manifest,
 ti_dag_version_data=bumped_manifest,
 )

 workload = ExecuteTask.make(ti)

 assert workload.bundle_info.version == "v1hash"
 assert workload.bundle_info.version_data == run_manifest
 assert workload.bundle_info.version_data != bumped_manifest


class TestExecuteCallbackMakeVersionData:
 """Tests for ExecuteCallback.make() threading version_data through BundleInfo."""
Expand Down

37 changes: 36 additions & 1 deletion
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -142,7 +142,7 @@
from airflow.utils.types import DagRunTriggeredByType, DagRunType

from tests_common.pytest_plugin import AIRFLOW_ROOT_PATH
from tests_common.test_utils.asserts import assert_queries_count
from tests_common.test_utils.asserts import assert_queries_count, count_queries
from tests_common.test_utils.config import conf_vars, env_vars
from tests_common.test_utils.dag import create_scheduler_dag, sync_dag_to_db, sync_dags_to_db
from tests_common.test_utils.db import (
Expand DownExpand Up@@ -1425,6 +1425,41 @@ def test_find_executable_task_instances_backfill(self, dag_maker):
 assert {x.key for x in queued_tis} == {ti_non_backfill.key, ti_backfill.key}
 session.rollback()

 def test_executable_task_instances_no_per_ti_queries(self, dag_maker, session):
 """Guard against an N+1 when enqueuing task instances.

 ``ExecuteTask.make()`` reads ``ti.dag_run.created_dag_version.version_data`` to ship the
 run's pinned bundle manifest. ``dag_run`` is eager-joined and ``created_dag_version`` is a
 single batched ``selectin``, so the number of queries in
 ``_executable_task_instances_to_queued`` must be independent of how many task instances are
 in the batch. If a future change lazy-loads ``dag_run``/``created_dag_version`` per TI, the
 count would scale with the task count and this test fails.
 """
 scheduler_job = Job()
 runner = SchedulerJobRunner(job=scheduler_job)
 self.job_runner = runner

 def _measure(dag_id: str, num_tasks: int) -> int:
 with dag_maker(dag_id=dag_id, max_active_tasks=64, session=session):
 for i in range(num_tasks):
 EmptyOperator(task_id=f"t{i}")
 dr = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
 for ti in dr.task_instances:
 ti.state = State.SCHEDULED
 session.flush()
 with count_queries(session=session) as result:
 runner._executable_task_instances_to_queued(max_tis=64, session=session)
 session.rollback()
 return sum(result.values())

 one_task = _measure("q_count_one", 1)
 many_tasks = _measure("q_count_many", 10)

 assert one_task == many_tasks, (
 f"query count scaled with task-instance count ({one_task} -> {many_tasks}); "
 "likely a per-TI lazy load (N+1) of dag_run/created_dag_version"
 )

 def test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query(self, dag_maker, session):
 dag_id = "SchedulerJobTest.test_find_executable_task_instances_mysql_hint_only_applies_to_inner_query"
 task_id = "dummy"
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/batch/test_batch_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -817,6 +817,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 if not AIRFLOW_V_3_0_PLUS:
Expand Down

2 changes: 2 additions & 0 deletions
providers/amazon/tests/unit/amazon/aws/executors/ecs/test_ecs_executor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1312,6 +1312,8 @@ def test_try_adopt_task_instances(self, mock_executor):
 task.dag_version = mock.Mock(version_data=None)
 task.dag_run = mock.Mock()
 task.dag_run.bundle_version = "1.0.0"
 # ExecuteTask.make() sources version_data from the run's pinned version.
 task.dag_run.created_dag_version = mock.Mock(version_data=None)
 task.dag_run.context_carrier = {}

 # Mock command generation based on Airflow version
Expand Down