Skip to content

[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)#69824

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eladkal merged 1 commit into from
Jul 13, 2026
Merged

[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767) #69824

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions airflow-core/src/airflow/jobs/scheduler_job_runner.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3548,12 +3548,21 @@ def _purge_task_instances_without_heartbeats(
# Backfill dag_version_id for legacy tasks (Pydantic requires uuid.UUID).
if not _ensure_ti_has_dag_version_id(ti, session, self.log):
continue
# ti.task isn't loaded in this purge path, so is_eligible_to_retry() uses its
# no-task fallback (``try_number <= max_tries``), which skips the retries-configured
# check its task-loaded branch applies; guard with ``max_tries > 0`` so a task
# declared with retries=0 isn't treated as retry-eligible here.
if ti.max_tries > 0 and ti.is_eligible_to_retry():
task_callback_type = TaskInstanceState.UP_FOR_RETRY
else:
task_callback_type = TaskInstanceState.FAILED
request = TaskCallbackRequest(
filepath=ti.dag_model.relative_fileloc or "",
bundle_name=_hb_bundle_name,
bundle_version=_hb_bundle_version,
ti=ti,
msg=str(task_instance_heartbeat_timeout_message_details),
task_callback_type=task_callback_type,
context_from_server=TIRunContext(
dag_run=DRDataModel.model_validate(ti.dag_run, from_attributes=True),
max_tries=ti.max_tries,
Expand Down
115 changes: 115 additions & 0 deletions airflow-core/tests/unit/jobs/test_scheduler_job.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -8608,6 +8608,121 @@ def test_scheduler_passes_context_from_server_on_heartbeat_timeout(self, dag_mak
assert callback_request.context_from_server.dag_run.logical_date == dag_run.logical_date
assert callback_request.context_from_server.max_tries == ti.max_tries

@pytest.mark.parametrize(
("state", "retries", "try_number", "expected_callback_type", "expected_dispatched_callback"),
[
pytest.param(
TaskInstanceState.RUNNING,
0,
1,
TaskInstanceState.FAILED,
"on_failure_callback",
id="no_retries",
),
pytest.param(
TaskInstanceState.RUNNING,
2,
1,
TaskInstanceState.UP_FOR_RETRY,
"on_retry_callback",
id="retries_available_first_attempt",
),
pytest.param(
TaskInstanceState.RUNNING,
2,
2,
TaskInstanceState.UP_FOR_RETRY,
"on_retry_callback",
id="retries_available_mid_chain",
),
pytest.param(
TaskInstanceState.RUNNING,
2,
3,
TaskInstanceState.FAILED,
"on_failure_callback",
id="retries_exhausted",
),
pytest.param(
TaskInstanceState.RESTARTING,
1,
5,
TaskInstanceState.UP_FOR_RETRY,
"on_retry_callback",
id="restarting_stays_eligible_past_max_tries",
),
],
)
def test_heartbeat_timeout_sets_callback_type_by_retry_eligibility(
self,
dag_maker,
session,
state,
retries,
try_number,
expected_callback_type,
expected_dispatched_callback,
):
"""Heartbeat-timeout cleanup must populate ``task_callback_type`` so the Dag processor
fires ``on_retry_callback`` when the task still has retries left, not
``on_failure_callback``.

Reproduces the bug end-to-end through the actual scheduler purge path:

1. A TI is ``RUNNING`` (or ``RESTARTING``) with a stale ``last_heartbeat_at`` (worker
OOMKilled, node evicted, scheduler restarted, etc.).
2. ``_find_and_purge_task_instances_without_heartbeats`` builds a
``TaskCallbackRequest`` and hands it to the executor's ``send_callback``.
3. The Dag processor branches on ``request.task_callback_type``:
``UP_FOR_RETRY`` -> ``task.on_retry_callback``; anything else (including ``None``)
-> ``task.on_failure_callback``. See
``airflow-core/src/airflow/dag_processing/processor.py``::``_execute_task_callbacks``.

Before the fix, step 2 left ``task_callback_type`` as ``None``, so step 3 always fell
into the ``else`` branch and ``on_failure_callback`` fired even when the task still had
retries left -- producing spurious failure alerts for tasks that ultimately succeeded on
retry.

The parametrized cases cover the full ``max_tries`` / ``try_number`` matrix for a
``RUNNING`` TI -- no retries, retries available (first attempt and mid-chain), and
retries exhausted (``try_number > max_tries``) -- plus a ``RESTARTING`` TI (cleared
while running), which ``is_eligible_to_retry`` keeps retry-eligible even past
``max_tries``. The ``expected_dispatched_callback`` column mirrors the Dag processor's
branch so the assertion captures the user-visible outcome, not just the field value.
"""
with dag_maker(dag_id=f"hb_timeout_r{retries}_t{try_number}", session=session):
EmptyOperator(task_id="test_task", retries=retries)

dag_run = dag_maker.create_dagrun(run_id="test_run", state=DagRunState.RUNNING)

mock_executor = MagicMock()
scheduler_job = Job()
self.job_runner = SchedulerJobRunner(scheduler_job, executors=[mock_executor])

ti = dag_run.get_task_instance(task_id="test_task")
ti.state = state
ti.try_number = try_number
ti.queued_by_job_id = scheduler_job.id
ti.last_heartbeat_at = timezone.utcnow() - timedelta(seconds=600)
session.merge(ti)
session.commit()

self.job_runner._find_and_purge_task_instances_without_heartbeats()

mock_executor.send_callback.assert_called_once()
request = mock_executor.send_callback.call_args[0][0]
assert isinstance(request, TaskCallbackRequest)
assert request.task_callback_type == expected_callback_type
# Mirror processor._execute_task_callbacks: UP_FOR_RETRY -> on_retry_callback, else
# on_failure_callback. Asserting the dispatched callback closes the loop on the
# user-visible behaviour, not just the field value.
dispatched_callback = (
"on_retry_callback"
if request.task_callback_type is TaskInstanceState.UP_FOR_RETRY
else "on_failure_callback"
)
assert dispatched_callback == expected_dispatched_callback

@pytest.mark.parametrize(
("retries", "callback_kind", "expected"),
[
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767) by github-actions[bot] · Pull Request #69824 · apache/airflow · GitHub
Skip to content
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)#69824

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

eladkal
merged 1 commit into

from

Jul 13, 2026
 Merged


[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)
#69824
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3548,12 +3548,21 @@ def _purge_task_instances_without_heartbeats(
 # Backfill dag_version_id for legacy tasks (Pydantic requires uuid.UUID).
 if not _ensure_ti_has_dag_version_id(ti, session, self.log):
 continue
 # ti.task isn't loaded in this purge path, so is_eligible_to_retry() uses its
 # no-task fallback (``try_number <= max_tries``), which skips the retries-configured
 # check its task-loaded branch applies; guard with ``max_tries > 0`` so a task
 # declared with retries=0 isn't treated as retry-eligible here.
 if ti.max_tries > 0 and ti.is_eligible_to_retry():
 task_callback_type = TaskInstanceState.UP_FOR_RETRY
 else:
 task_callback_type = TaskInstanceState.FAILED
 request = TaskCallbackRequest(
 filepath=ti.dag_model.relative_fileloc or "",
 bundle_name=_hb_bundle_name,
 bundle_version=_hb_bundle_version,
 ti=ti,
 msg=str(task_instance_heartbeat_timeout_message_details),
 task_callback_type=task_callback_type,
 context_from_server=TIRunContext(
 dag_run=DRDataModel.model_validate(ti.dag_run, from_attributes=True),
 max_tries=ti.max_tries,
Expand Down

115 changes: 115 additions & 0 deletions
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8608,6 +8608,121 @@ def test_scheduler_passes_context_from_server_on_heartbeat_timeout(self, dag_mak
 assert callback_request.context_from_server.dag_run.logical_date == dag_run.logical_date
 assert callback_request.context_from_server.max_tries == ti.max_tries

 @pytest.mark.parametrize(
 ("state", "retries", "try_number", "expected_callback_type", "expected_dispatched_callback"),
 [
 pytest.param(
 TaskInstanceState.RUNNING,
 0,
 1,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="no_retries",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 1,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_first_attempt",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 2,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_mid_chain",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 3,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="retries_exhausted",
 ),
 pytest.param(
 TaskInstanceState.RESTARTING,
 1,
 5,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="restarting_stays_eligible_past_max_tries",
 ),
 ],
 )
 def test_heartbeat_timeout_sets_callback_type_by_retry_eligibility(
 self,
 dag_maker,
 session,
 state,
 retries,
 try_number,
 expected_callback_type,
 expected_dispatched_callback,
 ):
 """Heartbeat-timeout cleanup must populate ``task_callback_type`` so the Dag processor
 fires ``on_retry_callback`` when the task still has retries left, not
 ``on_failure_callback``.

 Reproduces the bug end-to-end through the actual scheduler purge path:

 1. A TI is ``RUNNING`` (or ``RESTARTING``) with a stale ``last_heartbeat_at`` (worker
 OOMKilled, node evicted, scheduler restarted, etc.).
 2. ``_find_and_purge_task_instances_without_heartbeats`` builds a
 ``TaskCallbackRequest`` and hands it to the executor's ``send_callback``.
 3. The Dag processor branches on ``request.task_callback_type``:
 ``UP_FOR_RETRY`` -> ``task.on_retry_callback``; anything else (including ``None``)
 -> ``task.on_failure_callback``. See
 ``airflow-core/src/airflow/dag_processing/processor.py``::``_execute_task_callbacks``.

 Before the fix, step 2 left ``task_callback_type`` as ``None``, so step 3 always fell
 into the ``else`` branch and ``on_failure_callback`` fired even when the task still had
 retries left -- producing spurious failure alerts for tasks that ultimately succeeded on
 retry.

 The parametrized cases cover the full ``max_tries`` / ``try_number`` matrix for a
 ``RUNNING`` TI -- no retries, retries available (first attempt and mid-chain), and
 retries exhausted (``try_number > max_tries``) -- plus a ``RESTARTING`` TI (cleared
 while running), which ``is_eligible_to_retry`` keeps retry-eligible even past
 ``max_tries``. The ``expected_dispatched_callback`` column mirrors the Dag processor's
 branch so the assertion captures the user-visible outcome, not just the field value.
 """
 with dag_maker(dag_id=f"hb_timeout_r{retries}_t{try_number}", session=session):
 EmptyOperator(task_id="test_task", retries=retries)

 dag_run = dag_maker.create_dagrun(run_id="test_run", state=DagRunState.RUNNING)

 mock_executor = MagicMock()
 scheduler_job = Job()
 self.job_runner = SchedulerJobRunner(scheduler_job, executors=[mock_executor])

 ti = dag_run.get_task_instance(task_id="test_task")
 ti.state = state
 ti.try_number = try_number
 ti.queued_by_job_id = scheduler_job.id
 ti.last_heartbeat_at = timezone.utcnow() - timedelta(seconds=600)
 session.merge(ti)
 session.commit()

 self.job_runner._find_and_purge_task_instances_without_heartbeats()

 mock_executor.send_callback.assert_called_once()
 request = mock_executor.send_callback.call_args[0][0]
 assert isinstance(request, TaskCallbackRequest)
 assert request.task_callback_type == expected_callback_type
 # Mirror processor._execute_task_callbacks: UP_FOR_RETRY -> on_retry_callback, else
 # on_failure_callback. Asserting the dispatched callback closes the loop on the
 # user-visible behaviour, not just the field value.
 dispatched_callback = (
 "on_retry_callback"
 if request.task_callback_type is TaskInstanceState.UP_FOR_RETRY
 else "on_failure_callback"
 )
 assert dispatched_callback == expected_dispatched_callback

 @pytest.mark.parametrize(
 ("retries", "callback_kind", "expected"),
 [
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767) by github-actions[bot] · Pull Request #69824 · apache/airflow · GitHub
Skip to content
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)#69824

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

eladkal
merged 1 commit into

from

Jul 13, 2026
 Merged


[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)
#69824
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3548,12 +3548,21 @@ def _purge_task_instances_without_heartbeats(
 # Backfill dag_version_id for legacy tasks (Pydantic requires uuid.UUID).
 if not _ensure_ti_has_dag_version_id(ti, session, self.log):
 continue
 # ti.task isn't loaded in this purge path, so is_eligible_to_retry() uses its
 # no-task fallback (``try_number <= max_tries``), which skips the retries-configured
 # check its task-loaded branch applies; guard with ``max_tries > 0`` so a task
 # declared with retries=0 isn't treated as retry-eligible here.
 if ti.max_tries > 0 and ti.is_eligible_to_retry():
 task_callback_type = TaskInstanceState.UP_FOR_RETRY
 else:
 task_callback_type = TaskInstanceState.FAILED
 request = TaskCallbackRequest(
 filepath=ti.dag_model.relative_fileloc or "",
 bundle_name=_hb_bundle_name,
 bundle_version=_hb_bundle_version,
 ti=ti,
 msg=str(task_instance_heartbeat_timeout_message_details),
 task_callback_type=task_callback_type,
 context_from_server=TIRunContext(
 dag_run=DRDataModel.model_validate(ti.dag_run, from_attributes=True),
 max_tries=ti.max_tries,
Expand Down

115 changes: 115 additions & 0 deletions
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8608,6 +8608,121 @@ def test_scheduler_passes_context_from_server_on_heartbeat_timeout(self, dag_mak
 assert callback_request.context_from_server.dag_run.logical_date == dag_run.logical_date
 assert callback_request.context_from_server.max_tries == ti.max_tries

 @pytest.mark.parametrize(
 ("state", "retries", "try_number", "expected_callback_type", "expected_dispatched_callback"),
 [
 pytest.param(
 TaskInstanceState.RUNNING,
 0,
 1,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="no_retries",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 1,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_first_attempt",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 2,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_mid_chain",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 3,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="retries_exhausted",
 ),
 pytest.param(
 TaskInstanceState.RESTARTING,
 1,
 5,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="restarting_stays_eligible_past_max_tries",
 ),
 ],
 )
 def test_heartbeat_timeout_sets_callback_type_by_retry_eligibility(
 self,
 dag_maker,
 session,
 state,
 retries,
 try_number,
 expected_callback_type,
 expected_dispatched_callback,
 ):
 """Heartbeat-timeout cleanup must populate ``task_callback_type`` so the Dag processor
 fires ``on_retry_callback`` when the task still has retries left, not
 ``on_failure_callback``.

 Reproduces the bug end-to-end through the actual scheduler purge path:

 1. A TI is ``RUNNING`` (or ``RESTARTING``) with a stale ``last_heartbeat_at`` (worker
 OOMKilled, node evicted, scheduler restarted, etc.).
 2. ``_find_and_purge_task_instances_without_heartbeats`` builds a
 ``TaskCallbackRequest`` and hands it to the executor's ``send_callback``.
 3. The Dag processor branches on ``request.task_callback_type``:
 ``UP_FOR_RETRY`` -> ``task.on_retry_callback``; anything else (including ``None``)
 -> ``task.on_failure_callback``. See
 ``airflow-core/src/airflow/dag_processing/processor.py``::``_execute_task_callbacks``.

 Before the fix, step 2 left ``task_callback_type`` as ``None``, so step 3 always fell
 into the ``else`` branch and ``on_failure_callback`` fired even when the task still had
 retries left -- producing spurious failure alerts for tasks that ultimately succeeded on
 retry.

 The parametrized cases cover the full ``max_tries`` / ``try_number`` matrix for a
 ``RUNNING`` TI -- no retries, retries available (first attempt and mid-chain), and
 retries exhausted (``try_number > max_tries``) -- plus a ``RESTARTING`` TI (cleared
 while running), which ``is_eligible_to_retry`` keeps retry-eligible even past
 ``max_tries``. The ``expected_dispatched_callback`` column mirrors the Dag processor's
 branch so the assertion captures the user-visible outcome, not just the field value.
 """
 with dag_maker(dag_id=f"hb_timeout_r{retries}_t{try_number}", session=session):
 EmptyOperator(task_id="test_task", retries=retries)

 dag_run = dag_maker.create_dagrun(run_id="test_run", state=DagRunState.RUNNING)

 mock_executor = MagicMock()
 scheduler_job = Job()
 self.job_runner = SchedulerJobRunner(scheduler_job, executors=[mock_executor])

 ti = dag_run.get_task_instance(task_id="test_task")
 ti.state = state
 ti.try_number = try_number
 ti.queued_by_job_id = scheduler_job.id
 ti.last_heartbeat_at = timezone.utcnow() - timedelta(seconds=600)
 session.merge(ti)
 session.commit()

 self.job_runner._find_and_purge_task_instances_without_heartbeats()

 mock_executor.send_callback.assert_called_once()
 request = mock_executor.send_callback.call_args[0][0]
 assert isinstance(request, TaskCallbackRequest)
 assert request.task_callback_type == expected_callback_type
 # Mirror processor._execute_task_callbacks: UP_FOR_RETRY -> on_retry_callback, else
 # on_failure_callback. Asserting the dispatched callback closes the loop on the
 # user-visible behaviour, not just the field value.
 dispatched_callback = (
 "on_retry_callback"
 if request.task_callback_type is TaskInstanceState.UP_FOR_RETRY
 else "on_failure_callback"
 )
 assert dispatched_callback == expected_dispatched_callback

 @pytest.mark.parametrize(
 ("retries", "callback_kind", "expected"),
 [
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767) by github-actions[bot] · Pull Request #69824 · apache/airflow · GitHub
Skip to content
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)#69824

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

eladkal
merged 1 commit into

from

Jul 13, 2026
 Merged


[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)
#69824
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3548,12 +3548,21 @@ def _purge_task_instances_without_heartbeats(
 # Backfill dag_version_id for legacy tasks (Pydantic requires uuid.UUID).
 if not _ensure_ti_has_dag_version_id(ti, session, self.log):
 continue
 # ti.task isn't loaded in this purge path, so is_eligible_to_retry() uses its
 # no-task fallback (``try_number <= max_tries``), which skips the retries-configured
 # check its task-loaded branch applies; guard with ``max_tries > 0`` so a task
 # declared with retries=0 isn't treated as retry-eligible here.
 if ti.max_tries > 0 and ti.is_eligible_to_retry():
 task_callback_type = TaskInstanceState.UP_FOR_RETRY
 else:
 task_callback_type = TaskInstanceState.FAILED
 request = TaskCallbackRequest(
 filepath=ti.dag_model.relative_fileloc or "",
 bundle_name=_hb_bundle_name,
 bundle_version=_hb_bundle_version,
 ti=ti,
 msg=str(task_instance_heartbeat_timeout_message_details),
 task_callback_type=task_callback_type,
 context_from_server=TIRunContext(
 dag_run=DRDataModel.model_validate(ti.dag_run, from_attributes=True),
 max_tries=ti.max_tries,
Expand Down

115 changes: 115 additions & 0 deletions
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8608,6 +8608,121 @@ def test_scheduler_passes_context_from_server_on_heartbeat_timeout(self, dag_mak
 assert callback_request.context_from_server.dag_run.logical_date == dag_run.logical_date
 assert callback_request.context_from_server.max_tries == ti.max_tries

 @pytest.mark.parametrize(
 ("state", "retries", "try_number", "expected_callback_type", "expected_dispatched_callback"),
 [
 pytest.param(
 TaskInstanceState.RUNNING,
 0,
 1,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="no_retries",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 1,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_first_attempt",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 2,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_mid_chain",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 3,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="retries_exhausted",
 ),
 pytest.param(
 TaskInstanceState.RESTARTING,
 1,
 5,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="restarting_stays_eligible_past_max_tries",
 ),
 ],
 )
 def test_heartbeat_timeout_sets_callback_type_by_retry_eligibility(
 self,
 dag_maker,
 session,
 state,
 retries,
 try_number,
 expected_callback_type,
 expected_dispatched_callback,
 ):
 """Heartbeat-timeout cleanup must populate ``task_callback_type`` so the Dag processor
 fires ``on_retry_callback`` when the task still has retries left, not
 ``on_failure_callback``.

 Reproduces the bug end-to-end through the actual scheduler purge path:

 1. A TI is ``RUNNING`` (or ``RESTARTING``) with a stale ``last_heartbeat_at`` (worker
 OOMKilled, node evicted, scheduler restarted, etc.).
 2. ``_find_and_purge_task_instances_without_heartbeats`` builds a
 ``TaskCallbackRequest`` and hands it to the executor's ``send_callback``.
 3. The Dag processor branches on ``request.task_callback_type``:
 ``UP_FOR_RETRY`` -> ``task.on_retry_callback``; anything else (including ``None``)
 -> ``task.on_failure_callback``. See
 ``airflow-core/src/airflow/dag_processing/processor.py``::``_execute_task_callbacks``.

 Before the fix, step 2 left ``task_callback_type`` as ``None``, so step 3 always fell
 into the ``else`` branch and ``on_failure_callback`` fired even when the task still had
 retries left -- producing spurious failure alerts for tasks that ultimately succeeded on
 retry.

 The parametrized cases cover the full ``max_tries`` / ``try_number`` matrix for a
 ``RUNNING`` TI -- no retries, retries available (first attempt and mid-chain), and
 retries exhausted (``try_number > max_tries``) -- plus a ``RESTARTING`` TI (cleared
 while running), which ``is_eligible_to_retry`` keeps retry-eligible even past
 ``max_tries``. The ``expected_dispatched_callback`` column mirrors the Dag processor's
 branch so the assertion captures the user-visible outcome, not just the field value.
 """
 with dag_maker(dag_id=f"hb_timeout_r{retries}_t{try_number}", session=session):
 EmptyOperator(task_id="test_task", retries=retries)

 dag_run = dag_maker.create_dagrun(run_id="test_run", state=DagRunState.RUNNING)

 mock_executor = MagicMock()
 scheduler_job = Job()
 self.job_runner = SchedulerJobRunner(scheduler_job, executors=[mock_executor])

 ti = dag_run.get_task_instance(task_id="test_task")
 ti.state = state
 ti.try_number = try_number
 ti.queued_by_job_id = scheduler_job.id
 ti.last_heartbeat_at = timezone.utcnow() - timedelta(seconds=600)
 session.merge(ti)
 session.commit()

 self.job_runner._find_and_purge_task_instances_without_heartbeats()

 mock_executor.send_callback.assert_called_once()
 request = mock_executor.send_callback.call_args[0][0]
 assert isinstance(request, TaskCallbackRequest)
 assert request.task_callback_type == expected_callback_type
 # Mirror processor._execute_task_callbacks: UP_FOR_RETRY -> on_retry_callback, else
 # on_failure_callback. Asserting the dispatched callback closes the loop on the
 # user-visible behaviour, not just the field value.
 dispatched_callback = (
 "on_retry_callback"
 if request.task_callback_type is TaskInstanceState.UP_FOR_RETRY
 else "on_failure_callback"
 )
 assert dispatched_callback == expected_dispatched_callback

 @pytest.mark.parametrize(
 ("retries", "callback_kind", "expected"),
 [
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767) by github-actions[bot] · Pull Request #69824 · apache/airflow · GitHub
Skip to content
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)#69824

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

eladkal
merged 1 commit into

from

Jul 13, 2026
 Merged


[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)
#69824
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3548,12 +3548,21 @@ def _purge_task_instances_without_heartbeats(
 # Backfill dag_version_id for legacy tasks (Pydantic requires uuid.UUID).
 if not _ensure_ti_has_dag_version_id(ti, session, self.log):
 continue
 # ti.task isn't loaded in this purge path, so is_eligible_to_retry() uses its
 # no-task fallback (``try_number <= max_tries``), which skips the retries-configured
 # check its task-loaded branch applies; guard with ``max_tries > 0`` so a task
 # declared with retries=0 isn't treated as retry-eligible here.
 if ti.max_tries > 0 and ti.is_eligible_to_retry():
 task_callback_type = TaskInstanceState.UP_FOR_RETRY
 else:
 task_callback_type = TaskInstanceState.FAILED
 request = TaskCallbackRequest(
 filepath=ti.dag_model.relative_fileloc or "",
 bundle_name=_hb_bundle_name,
 bundle_version=_hb_bundle_version,
 ti=ti,
 msg=str(task_instance_heartbeat_timeout_message_details),
 task_callback_type=task_callback_type,
 context_from_server=TIRunContext(
 dag_run=DRDataModel.model_validate(ti.dag_run, from_attributes=True),
 max_tries=ti.max_tries,
Expand Down

115 changes: 115 additions & 0 deletions
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8608,6 +8608,121 @@ def test_scheduler_passes_context_from_server_on_heartbeat_timeout(self, dag_mak
 assert callback_request.context_from_server.dag_run.logical_date == dag_run.logical_date
 assert callback_request.context_from_server.max_tries == ti.max_tries

 @pytest.mark.parametrize(
 ("state", "retries", "try_number", "expected_callback_type", "expected_dispatched_callback"),
 [
 pytest.param(
 TaskInstanceState.RUNNING,
 0,
 1,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="no_retries",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 1,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_first_attempt",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 2,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_mid_chain",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 3,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="retries_exhausted",
 ),
 pytest.param(
 TaskInstanceState.RESTARTING,
 1,
 5,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="restarting_stays_eligible_past_max_tries",
 ),
 ],
 )
 def test_heartbeat_timeout_sets_callback_type_by_retry_eligibility(
 self,
 dag_maker,
 session,
 state,
 retries,
 try_number,
 expected_callback_type,
 expected_dispatched_callback,
 ):
 """Heartbeat-timeout cleanup must populate ``task_callback_type`` so the Dag processor
 fires ``on_retry_callback`` when the task still has retries left, not
 ``on_failure_callback``.

 Reproduces the bug end-to-end through the actual scheduler purge path:

 1. A TI is ``RUNNING`` (or ``RESTARTING``) with a stale ``last_heartbeat_at`` (worker
 OOMKilled, node evicted, scheduler restarted, etc.).
 2. ``_find_and_purge_task_instances_without_heartbeats`` builds a
 ``TaskCallbackRequest`` and hands it to the executor's ``send_callback``.
 3. The Dag processor branches on ``request.task_callback_type``:
 ``UP_FOR_RETRY`` -> ``task.on_retry_callback``; anything else (including ``None``)
 -> ``task.on_failure_callback``. See
 ``airflow-core/src/airflow/dag_processing/processor.py``::``_execute_task_callbacks``.

 Before the fix, step 2 left ``task_callback_type`` as ``None``, so step 3 always fell
 into the ``else`` branch and ``on_failure_callback`` fired even when the task still had
 retries left -- producing spurious failure alerts for tasks that ultimately succeeded on
 retry.

 The parametrized cases cover the full ``max_tries`` / ``try_number`` matrix for a
 ``RUNNING`` TI -- no retries, retries available (first attempt and mid-chain), and
 retries exhausted (``try_number > max_tries``) -- plus a ``RESTARTING`` TI (cleared
 while running), which ``is_eligible_to_retry`` keeps retry-eligible even past
 ``max_tries``. The ``expected_dispatched_callback`` column mirrors the Dag processor's
 branch so the assertion captures the user-visible outcome, not just the field value.
 """
 with dag_maker(dag_id=f"hb_timeout_r{retries}_t{try_number}", session=session):
 EmptyOperator(task_id="test_task", retries=retries)

 dag_run = dag_maker.create_dagrun(run_id="test_run", state=DagRunState.RUNNING)

 mock_executor = MagicMock()
 scheduler_job = Job()
 self.job_runner = SchedulerJobRunner(scheduler_job, executors=[mock_executor])

 ti = dag_run.get_task_instance(task_id="test_task")
 ti.state = state
 ti.try_number = try_number
 ti.queued_by_job_id = scheduler_job.id
 ti.last_heartbeat_at = timezone.utcnow() - timedelta(seconds=600)
 session.merge(ti)
 session.commit()

 self.job_runner._find_and_purge_task_instances_without_heartbeats()

 mock_executor.send_callback.assert_called_once()
 request = mock_executor.send_callback.call_args[0][0]
 assert isinstance(request, TaskCallbackRequest)
 assert request.task_callback_type == expected_callback_type
 # Mirror processor._execute_task_callbacks: UP_FOR_RETRY -> on_retry_callback, else
 # on_failure_callback. Asserting the dispatched callback closes the loop on the
 # user-visible behaviour, not just the field value.
 dispatched_callback = (
 "on_retry_callback"
 if request.task_callback_type is TaskInstanceState.UP_FOR_RETRY
 else "on_failure_callback"
 )
 assert dispatched_callback == expected_dispatched_callback

 @pytest.mark.parametrize(
 ("retries", "callback_kind", "expected"),
 [
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767) by github-actions[bot] · Pull Request #69824 · apache/airflow · GitHub
Skip to content
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)#69824

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

eladkal
merged 1 commit into

from

Jul 13, 2026
 Merged


[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)
#69824
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3548,12 +3548,21 @@ def _purge_task_instances_without_heartbeats(
 # Backfill dag_version_id for legacy tasks (Pydantic requires uuid.UUID).
 if not _ensure_ti_has_dag_version_id(ti, session, self.log):
 continue
 # ti.task isn't loaded in this purge path, so is_eligible_to_retry() uses its
 # no-task fallback (``try_number <= max_tries``), which skips the retries-configured
 # check its task-loaded branch applies; guard with ``max_tries > 0`` so a task
 # declared with retries=0 isn't treated as retry-eligible here.
 if ti.max_tries > 0 and ti.is_eligible_to_retry():
 task_callback_type = TaskInstanceState.UP_FOR_RETRY
 else:
 task_callback_type = TaskInstanceState.FAILED
 request = TaskCallbackRequest(
 filepath=ti.dag_model.relative_fileloc or "",
 bundle_name=_hb_bundle_name,
 bundle_version=_hb_bundle_version,
 ti=ti,
 msg=str(task_instance_heartbeat_timeout_message_details),
 task_callback_type=task_callback_type,
 context_from_server=TIRunContext(
 dag_run=DRDataModel.model_validate(ti.dag_run, from_attributes=True),
 max_tries=ti.max_tries,
Expand Down

115 changes: 115 additions & 0 deletions
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8608,6 +8608,121 @@ def test_scheduler_passes_context_from_server_on_heartbeat_timeout(self, dag_mak
 assert callback_request.context_from_server.dag_run.logical_date == dag_run.logical_date
 assert callback_request.context_from_server.max_tries == ti.max_tries

 @pytest.mark.parametrize(
 ("state", "retries", "try_number", "expected_callback_type", "expected_dispatched_callback"),
 [
 pytest.param(
 TaskInstanceState.RUNNING,
 0,
 1,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="no_retries",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 1,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_first_attempt",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 2,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_mid_chain",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 3,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="retries_exhausted",
 ),
 pytest.param(
 TaskInstanceState.RESTARTING,
 1,
 5,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="restarting_stays_eligible_past_max_tries",
 ),
 ],
 )
 def test_heartbeat_timeout_sets_callback_type_by_retry_eligibility(
 self,
 dag_maker,
 session,
 state,
 retries,
 try_number,
 expected_callback_type,
 expected_dispatched_callback,
 ):
 """Heartbeat-timeout cleanup must populate ``task_callback_type`` so the Dag processor
 fires ``on_retry_callback`` when the task still has retries left, not
 ``on_failure_callback``.

 Reproduces the bug end-to-end through the actual scheduler purge path:

 1. A TI is ``RUNNING`` (or ``RESTARTING``) with a stale ``last_heartbeat_at`` (worker
 OOMKilled, node evicted, scheduler restarted, etc.).
 2. ``_find_and_purge_task_instances_without_heartbeats`` builds a
 ``TaskCallbackRequest`` and hands it to the executor's ``send_callback``.
 3. The Dag processor branches on ``request.task_callback_type``:
 ``UP_FOR_RETRY`` -> ``task.on_retry_callback``; anything else (including ``None``)
 -> ``task.on_failure_callback``. See
 ``airflow-core/src/airflow/dag_processing/processor.py``::``_execute_task_callbacks``.

 Before the fix, step 2 left ``task_callback_type`` as ``None``, so step 3 always fell
 into the ``else`` branch and ``on_failure_callback`` fired even when the task still had
 retries left -- producing spurious failure alerts for tasks that ultimately succeeded on
 retry.

 The parametrized cases cover the full ``max_tries`` / ``try_number`` matrix for a
 ``RUNNING`` TI -- no retries, retries available (first attempt and mid-chain), and
 retries exhausted (``try_number > max_tries``) -- plus a ``RESTARTING`` TI (cleared
 while running), which ``is_eligible_to_retry`` keeps retry-eligible even past
 ``max_tries``. The ``expected_dispatched_callback`` column mirrors the Dag processor's
 branch so the assertion captures the user-visible outcome, not just the field value.
 """
 with dag_maker(dag_id=f"hb_timeout_r{retries}_t{try_number}", session=session):
 EmptyOperator(task_id="test_task", retries=retries)

 dag_run = dag_maker.create_dagrun(run_id="test_run", state=DagRunState.RUNNING)

 mock_executor = MagicMock()
 scheduler_job = Job()
 self.job_runner = SchedulerJobRunner(scheduler_job, executors=[mock_executor])

 ti = dag_run.get_task_instance(task_id="test_task")
 ti.state = state
 ti.try_number = try_number
 ti.queued_by_job_id = scheduler_job.id
 ti.last_heartbeat_at = timezone.utcnow() - timedelta(seconds=600)
 session.merge(ti)
 session.commit()

 self.job_runner._find_and_purge_task_instances_without_heartbeats()

 mock_executor.send_callback.assert_called_once()
 request = mock_executor.send_callback.call_args[0][0]
 assert isinstance(request, TaskCallbackRequest)
 assert request.task_callback_type == expected_callback_type
 # Mirror processor._execute_task_callbacks: UP_FOR_RETRY -> on_retry_callback, else
 # on_failure_callback. Asserting the dispatched callback closes the loop on the
 # user-visible behaviour, not just the field value.
 dispatched_callback = (
 "on_retry_callback"
 if request.task_callback_type is TaskInstanceState.UP_FOR_RETRY
 else "on_failure_callback"
 )
 assert dispatched_callback == expected_dispatched_callback

 @pytest.mark.parametrize(
 ("retries", "callback_kind", "expected"),
 [
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767) by github-actions[bot] · Pull Request #69824 · apache/airflow · GitHub
Skip to content
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)#69824

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

eladkal
merged 1 commit into

from

Jul 13, 2026
 Merged


[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)
#69824
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3548,12 +3548,21 @@ def _purge_task_instances_without_heartbeats(
 # Backfill dag_version_id for legacy tasks (Pydantic requires uuid.UUID).
 if not _ensure_ti_has_dag_version_id(ti, session, self.log):
 continue
 # ti.task isn't loaded in this purge path, so is_eligible_to_retry() uses its
 # no-task fallback (``try_number <= max_tries``), which skips the retries-configured
 # check its task-loaded branch applies; guard with ``max_tries > 0`` so a task
 # declared with retries=0 isn't treated as retry-eligible here.
 if ti.max_tries > 0 and ti.is_eligible_to_retry():
 task_callback_type = TaskInstanceState.UP_FOR_RETRY
 else:
 task_callback_type = TaskInstanceState.FAILED
 request = TaskCallbackRequest(
 filepath=ti.dag_model.relative_fileloc or "",
 bundle_name=_hb_bundle_name,
 bundle_version=_hb_bundle_version,
 ti=ti,
 msg=str(task_instance_heartbeat_timeout_message_details),
 task_callback_type=task_callback_type,
 context_from_server=TIRunContext(
 dag_run=DRDataModel.model_validate(ti.dag_run, from_attributes=True),
 max_tries=ti.max_tries,
Expand Down

115 changes: 115 additions & 0 deletions
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8608,6 +8608,121 @@ def test_scheduler_passes_context_from_server_on_heartbeat_timeout(self, dag_mak
 assert callback_request.context_from_server.dag_run.logical_date == dag_run.logical_date
 assert callback_request.context_from_server.max_tries == ti.max_tries

 @pytest.mark.parametrize(
 ("state", "retries", "try_number", "expected_callback_type", "expected_dispatched_callback"),
 [
 pytest.param(
 TaskInstanceState.RUNNING,
 0,
 1,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="no_retries",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 1,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_first_attempt",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 2,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_mid_chain",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 3,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="retries_exhausted",
 ),
 pytest.param(
 TaskInstanceState.RESTARTING,
 1,
 5,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="restarting_stays_eligible_past_max_tries",
 ),
 ],
 )
 def test_heartbeat_timeout_sets_callback_type_by_retry_eligibility(
 self,
 dag_maker,
 session,
 state,
 retries,
 try_number,
 expected_callback_type,
 expected_dispatched_callback,
 ):
 """Heartbeat-timeout cleanup must populate ``task_callback_type`` so the Dag processor
 fires ``on_retry_callback`` when the task still has retries left, not
 ``on_failure_callback``.

 Reproduces the bug end-to-end through the actual scheduler purge path:

 1. A TI is ``RUNNING`` (or ``RESTARTING``) with a stale ``last_heartbeat_at`` (worker
 OOMKilled, node evicted, scheduler restarted, etc.).
 2. ``_find_and_purge_task_instances_without_heartbeats`` builds a
 ``TaskCallbackRequest`` and hands it to the executor's ``send_callback``.
 3. The Dag processor branches on ``request.task_callback_type``:
 ``UP_FOR_RETRY`` -> ``task.on_retry_callback``; anything else (including ``None``)
 -> ``task.on_failure_callback``. See
 ``airflow-core/src/airflow/dag_processing/processor.py``::``_execute_task_callbacks``.

 Before the fix, step 2 left ``task_callback_type`` as ``None``, so step 3 always fell
 into the ``else`` branch and ``on_failure_callback`` fired even when the task still had
 retries left -- producing spurious failure alerts for tasks that ultimately succeeded on
 retry.

 The parametrized cases cover the full ``max_tries`` / ``try_number`` matrix for a
 ``RUNNING`` TI -- no retries, retries available (first attempt and mid-chain), and
 retries exhausted (``try_number > max_tries``) -- plus a ``RESTARTING`` TI (cleared
 while running), which ``is_eligible_to_retry`` keeps retry-eligible even past
 ``max_tries``. The ``expected_dispatched_callback`` column mirrors the Dag processor's
 branch so the assertion captures the user-visible outcome, not just the field value.
 """
 with dag_maker(dag_id=f"hb_timeout_r{retries}_t{try_number}", session=session):
 EmptyOperator(task_id="test_task", retries=retries)

 dag_run = dag_maker.create_dagrun(run_id="test_run", state=DagRunState.RUNNING)

 mock_executor = MagicMock()
 scheduler_job = Job()
 self.job_runner = SchedulerJobRunner(scheduler_job, executors=[mock_executor])

 ti = dag_run.get_task_instance(task_id="test_task")
 ti.state = state
 ti.try_number = try_number
 ti.queued_by_job_id = scheduler_job.id
 ti.last_heartbeat_at = timezone.utcnow() - timedelta(seconds=600)
 session.merge(ti)
 session.commit()

 self.job_runner._find_and_purge_task_instances_without_heartbeats()

 mock_executor.send_callback.assert_called_once()
 request = mock_executor.send_callback.call_args[0][0]
 assert isinstance(request, TaskCallbackRequest)
 assert request.task_callback_type == expected_callback_type
 # Mirror processor._execute_task_callbacks: UP_FOR_RETRY -> on_retry_callback, else
 # on_failure_callback. Asserting the dispatched callback closes the loop on the
 # user-visible behaviour, not just the field value.
 dispatched_callback = (
 "on_retry_callback"
 if request.task_callback_type is TaskInstanceState.UP_FOR_RETRY
 else "on_failure_callback"
 )
 assert dispatched_callback == expected_dispatched_callback

 @pytest.mark.parametrize(
 ("retries", "callback_kind", "expected"),
 [
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767) by github-actions[bot] · Pull Request #69824 · apache/airflow · GitHub
Skip to content
[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)#69824

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

eladkal
merged 1 commit into

from

Jul 13, 2026
 Merged


[v3-3-test] Fix scheduler firing on_failure_callback for heartbeat-timed-out retries (#66767)
#69824
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
airflow-core/src/airflow/jobs/scheduler_job_runner.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -3548,12 +3548,21 @@ def _purge_task_instances_without_heartbeats(
 # Backfill dag_version_id for legacy tasks (Pydantic requires uuid.UUID).
 if not _ensure_ti_has_dag_version_id(ti, session, self.log):
 continue
 # ti.task isn't loaded in this purge path, so is_eligible_to_retry() uses its
 # no-task fallback (``try_number <= max_tries``), which skips the retries-configured
 # check its task-loaded branch applies; guard with ``max_tries > 0`` so a task
 # declared with retries=0 isn't treated as retry-eligible here.
 if ti.max_tries > 0 and ti.is_eligible_to_retry():
 task_callback_type = TaskInstanceState.UP_FOR_RETRY
 else:
 task_callback_type = TaskInstanceState.FAILED
 request = TaskCallbackRequest(
 filepath=ti.dag_model.relative_fileloc or "",
 bundle_name=_hb_bundle_name,
 bundle_version=_hb_bundle_version,
 ti=ti,
 msg=str(task_instance_heartbeat_timeout_message_details),
 task_callback_type=task_callback_type,
 context_from_server=TIRunContext(
 dag_run=DRDataModel.model_validate(ti.dag_run, from_attributes=True),
 max_tries=ti.max_tries,
Expand Down

115 changes: 115 additions & 0 deletions
airflow-core/tests/unit/jobs/test_scheduler_job.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8608,6 +8608,121 @@ def test_scheduler_passes_context_from_server_on_heartbeat_timeout(self, dag_mak
 assert callback_request.context_from_server.dag_run.logical_date == dag_run.logical_date
 assert callback_request.context_from_server.max_tries == ti.max_tries

 @pytest.mark.parametrize(
 ("state", "retries", "try_number", "expected_callback_type", "expected_dispatched_callback"),
 [
 pytest.param(
 TaskInstanceState.RUNNING,
 0,
 1,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="no_retries",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 1,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_first_attempt",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 2,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="retries_available_mid_chain",
 ),
 pytest.param(
 TaskInstanceState.RUNNING,
 2,
 3,
 TaskInstanceState.FAILED,
 "on_failure_callback",
 id="retries_exhausted",
 ),
 pytest.param(
 TaskInstanceState.RESTARTING,
 1,
 5,
 TaskInstanceState.UP_FOR_RETRY,
 "on_retry_callback",
 id="restarting_stays_eligible_past_max_tries",
 ),
 ],
 )
 def test_heartbeat_timeout_sets_callback_type_by_retry_eligibility(
 self,
 dag_maker,
 session,
 state,
 retries,
 try_number,
 expected_callback_type,
 expected_dispatched_callback,
 ):
 """Heartbeat-timeout cleanup must populate ``task_callback_type`` so the Dag processor
 fires ``on_retry_callback`` when the task still has retries left, not
 ``on_failure_callback``.

 Reproduces the bug end-to-end through the actual scheduler purge path:

 1. A TI is ``RUNNING`` (or ``RESTARTING``) with a stale ``last_heartbeat_at`` (worker
 OOMKilled, node evicted, scheduler restarted, etc.).
 2. ``_find_and_purge_task_instances_without_heartbeats`` builds a
 ``TaskCallbackRequest`` and hands it to the executor's ``send_callback``.
 3. The Dag processor branches on ``request.task_callback_type``:
 ``UP_FOR_RETRY`` -> ``task.on_retry_callback``; anything else (including ``None``)
 -> ``task.on_failure_callback``. See
 ``airflow-core/src/airflow/dag_processing/processor.py``::``_execute_task_callbacks``.

 Before the fix, step 2 left ``task_callback_type`` as ``None``, so step 3 always fell
 into the ``else`` branch and ``on_failure_callback`` fired even when the task still had
 retries left -- producing spurious failure alerts for tasks that ultimately succeeded on
 retry.

 The parametrized cases cover the full ``max_tries`` / ``try_number`` matrix for a
 ``RUNNING`` TI -- no retries, retries available (first attempt and mid-chain), and
 retries exhausted (``try_number > max_tries``) -- plus a ``RESTARTING`` TI (cleared
 while running), which ``is_eligible_to_retry`` keeps retry-eligible even past
 ``max_tries``. The ``expected_dispatched_callback`` column mirrors the Dag processor's
 branch so the assertion captures the user-visible outcome, not just the field value.
 """
 with dag_maker(dag_id=f"hb_timeout_r{retries}_t{try_number}", session=session):
 EmptyOperator(task_id="test_task", retries=retries)

 dag_run = dag_maker.create_dagrun(run_id="test_run", state=DagRunState.RUNNING)

 mock_executor = MagicMock()
 scheduler_job = Job()
 self.job_runner = SchedulerJobRunner(scheduler_job, executors=[mock_executor])

 ti = dag_run.get_task_instance(task_id="test_task")
 ti.state = state
 ti.try_number = try_number
 ti.queued_by_job_id = scheduler_job.id
 ti.last_heartbeat_at = timezone.utcnow() - timedelta(seconds=600)
 session.merge(ti)
 session.commit()

 self.job_runner._find_and_purge_task_instances_without_heartbeats()

 mock_executor.send_callback.assert_called_once()
 request = mock_executor.send_callback.call_args[0][0]
 assert isinstance(request, TaskCallbackRequest)
 assert request.task_callback_type == expected_callback_type
 # Mirror processor._execute_task_callbacks: UP_FOR_RETRY -> on_retry_callback, else
 # on_failure_callback. Asserting the dispatched callback closes the loop on the
 # user-visible behaviour, not just the field value.
 dispatched_callback = (
 "on_retry_callback"
 if request.task_callback_type is TaskInstanceState.UP_FOR_RETRY
 else "on_failure_callback"
 )
 assert dispatched_callback == expected_dispatched_callback

 @pytest.mark.parametrize(
 ("retries", "callback_kind", "expected"),
 [
Expand Down
Loading