Fix EMR container job not cancelled on deferral timeout#64770

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

vincbeck merged 18 commits into apache:main from aurangzaib048:fix/emr-container-trigger-cleanup

May 7, 2026

providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py

-Original file line number
+Diff line change
@@ Expand Up @@
  :param tags: The tags assigned to job runs.
  Defaults to None
  :param deferrable: Run operator in the deferrable mode.
+ :param cancel_on_kill: Flag to indicate whether to cancel the job
+ when the task is killed while in deferrable mode.
  """
  aws_hook_class = EmrContainerHook
@@ Expand DownExpand Up / @@ -519,6 +521,7 @@ def __init__( @@
  max_polling_attempts: int | None = None,
  job_retry_max_attempts: int | None = None,
  deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
+ cancel_on_kill: bool = True,
  **kwargs: Any,
  ) -> None:
  super().__init__(**kwargs)
@@ Expand All / @@ -536,6 +539,7 @@ def __init__( @@
  self.tags = tags
  self.job_id: str | None = None
  self.deferrable = deferrable
+ self.cancel_on_kill = cancel_on_kill
  @property
  def _hook_parameters(self):
@@ Expand DownExpand Up / @@ -571,13 +575,15 @@ def execute(self, context: Context) -> str | None: @@
  aws_conn_id=self.aws_conn_id,
  waiter_delay=self.poll_interval,
  waiter_max_attempts=self.max_polling_attempts,
+ cancel_on_kill=self.cancel_on_kill,
  )
  if self.max_polling_attempts
  else EmrContainerTrigger(
  virtual_cluster_id=self.virtual_cluster_id,
  job_id=self.job_id,
  aws_conn_id=self.aws_conn_id,
  waiter_delay=self.poll_interval,
+ cancel_on_kill=self.cancel_on_kill,
  ),
  method_name="execute_complete",
  )
@@ Expand DownExpand Up / @@ -607,11 +613,9 @@ def check_failure(self, query_status): @@
  def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
  validated_event = validate_execute_complete_event(event)
- if validated_event["status"] != "success":
- raise AirflowException(f"Error while running job: {validated_event}")
- return validated_event["job_id"]
+ if validated_event["status"] == "success":
+ return validated_event["job_id"]
+ raise AirflowException(f"Error while running job: {validated_event}")
  def on_kill(self) -> None:
  """Cancel the submitted job run."""
@@ Expand Down @@

providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -167,6 +167,10 @@ class EmrContainerTrigger(AwsBaseWaiterTrigger): @@
  :param aws_conn_id: Reference to AWS connection id
  :param waiter_delay: polling period in seconds to check for the status
  :param waiter_max_attempts: The maximum number of attempts to be made. Defaults to an infinite wait.
+ :param cancel_on_kill: If True (default), cancel the EMR container job when the user
+ marks the deferred task failed, clears it, or mark-succeeds it. Requires
+ ``apache-airflow`` with ``BaseTrigger.on_kill()`` support; on older versions the
+ hook is silently inert.
  """
  def __init__(
@@ Expand All / @@ -176,9 +180,14 @@ def __init__( @@
  aws_conn_id: str | None = "aws_default",
  waiter_delay: int = 30,
  waiter_max_attempts: int = sys.maxsize,
+ cancel_on_kill: bool = True,
  ):
  super().__init__(
- serialized_fields={"virtual_cluster_id": virtual_cluster_id, "job_id": job_id},
+ serialized_fields={
+ "virtual_cluster_id": virtual_cluster_id,
+ "job_id": job_id,
+ "cancel_on_kill": cancel_on_kill,
+ },
  waiter_name="container_job_complete",
  waiter_args={"id": job_id, "virtualClusterId": virtual_cluster_id},
  failure_message="Job failed",
@@ Expand All / @@ -190,9 +199,31 @@ def __init__( @@
  waiter_max_attempts=waiter_max_attempts,
  aws_conn_id=aws_conn_id,
  )
+ self.virtual_cluster_id = virtual_cluster_id
+ self.job_id = job_id
+ self.cancel_on_kill = cancel_on_kill
  def hook(self) -> AwsGenericHook:
- return EmrContainerHook(aws_conn_id=self.aws_conn_id)
+ return EmrContainerHook(aws_conn_id=self.aws_conn_id, virtual_cluster_id=self.virtual_cluster_id)
+ async def on_kill(self) -> None:
+ """Cancel the EMR container job when the user acts on the deferred task."""
+ if not self.cancel_on_kill or not self.job_id:
+ return
+ self.log.info(
+ "Cancelling EMR container job. Virtual Cluster ID: %s, Job ID: %s",
+ self.virtual_cluster_id,
+ self.job_id,
+ )
+ hook: EmrContainerHook = self.hook() # type: ignore[assignment]
+ try:
+ await sync_to_async(hook.stop_query)(self.job_id)
+ self.log.info("EMR container job %s cancelled.", self.job_id)
+ except Exception:
+ self.log.exception(
+ "Failed to cancel EMR container job %s. The job may still be running.",
+ self.job_id,
+ )
 class EmrStepSensorTrigger(AwsBaseWaiterTrigger):
@@ Expand Down @@

providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py

-Original file line number
+Diff line change
@@ Expand Up @@
  assert trigger.waiter_delay == self.emr_container.poll_interval
  assert trigger.attempts == self.emr_container.max_polling_attempts
+ def test_execute_complete_returns_job_id_on_success(self):
+ event = {"status": "success", "job_id": "test_job_id"}
+ result = self.emr_container.execute_complete(context=None, event=event)
+ assert result == "test_job_id"
+ def test_execute_complete_raises_on_error_event(self):
+ event = {"status": "error", "message": "Job failed", "job_id": "test_job_id"}
+ with pytest.raises(AirflowException, match="Error while running job"):
+ self.emr_container.execute_complete(context=None, event=event)
 class TestEmrEksCreateClusterOperator:
  def setup_method(self):
@@ Expand Down @@

providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -22,6 +22,7 @@ @@
 import pytest
+from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook
 from airflow.providers.amazon.aws.triggers.emr import (
  EmrAddStepsTrigger,
  EmrContainerTrigger,
@@ Expand DownExpand Up / @@ -130,6 +131,7 @@ def test_serialization(self): @@
  "waiter_delay": 30,
  "waiter_max_attempts": 600,
  "aws_conn_id": "aws_default",
+ "cancel_on_kill": True,
  }
  def test_serialization_default_max_attempts(self):
@@ Expand All / @@ -152,8 +154,72 @@ def test_serialization_default_max_attempts(self): @@
  "waiter_delay": 30,
  "waiter_max_attempts": sys.maxsize,
  "aws_conn_id": "aws_default",
+ "cancel_on_kill": True,
  }
+ def test_serialization_cancel_on_kill_false(self):
+ """Test that cancel_on_kill=False is correctly serialized."""
+ trigger = EmrContainerTrigger(
+ virtual_cluster_id="test_cluster",
+ job_id="test_job",
+ waiter_delay=30,
+ waiter_max_attempts=60,
+ aws_conn_id="aws_default",
+ cancel_on_kill=False,
+ )
+ classpath, kwargs = trigger.serialize()
+ assert classpath == "airflow.providers.amazon.aws.triggers.emr.EmrContainerTrigger"
+ assert kwargs["cancel_on_kill"] is False
+ @pytest.mark.asyncio
+ async def test_on_kill_cancels_job(self):
+ """on_kill() stops the EMR container job when enabled and job_id is set."""
+ trigger = EmrContainerTrigger(
+ virtual_cluster_id="test_cluster",
+ job_id="test_job",
+ waiter_delay=30,
+ waiter_max_attempts=60,
+ aws_conn_id="aws_default",
+ cancel_on_kill=True,
+ )
+ mock_hook = mock.MagicMock(spec=EmrContainerHook)
+ with mock.patch.object(trigger, "hook", return_value=mock_hook):
+ await trigger.on_kill()
+ mock_hook.stop_query.assert_called_once_with("test_job")
+ @pytest.mark.asyncio
+ async def test_on_kill_noop_when_cancel_on_kill_false(self):
+ """on_kill() is a no-op when cancel_on_kill=False."""
+ trigger = EmrContainerTrigger(
+ virtual_cluster_id="test_cluster",
+ job_id="test_job",
+ waiter_delay=30,
+ waiter_max_attempts=60,
+ aws_conn_id="aws_default",
+ cancel_on_kill=False,
+ )
+ mock_hook = mock.MagicMock(spec=EmrContainerHook)
+ with mock.patch.object(trigger, "hook", return_value=mock_hook):
+ await trigger.on_kill()
+ mock_hook.stop_query.assert_not_called()
+ @pytest.mark.asyncio
+ async def test_on_kill_swallows_stop_query_error(self):
+ """on_kill() logs and swallows exceptions raised by stop_query."""
+ trigger = EmrContainerTrigger(
+ virtual_cluster_id="test_cluster",
+ job_id="test_job",
+ waiter_delay=30,
+ waiter_max_attempts=60,
+ aws_conn_id="aws_default",
+ cancel_on_kill=True,
+ )
+ mock_hook = mock.MagicMock(spec=EmrContainerHook)
+ mock_hook.stop_query.side_effect = Exception("AWS API error")
+ with mock.patch.object(trigger, "hook", return_value=mock_hook):
+ await trigger.on_kill()
+ mock_hook.stop_query.assert_called_once_with("test_job")
 class TestEmrStepSensorTrigger:
  def test_serialization(self):
@@ Expand Down @@

blocks (function() { function addCopyButtons() { document.querySelectorAll('pre code').forEach(function(codeBlock) { if (codeBlock.parentElement.hasAttribute('data-copy-added')) return; codeBlock.parentElement.setAttribute('data-copy-added', 'true'); var btn = document.createElement('button'); btn.textContent = 'Copy'; btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;'; btn.onmouseover = function() { this.style.opacity = '1'; }; btn.onmouseout = function() { this.style.opacity = '0.7'; }; btn.onclick = function() { navigator.clipboard.writeText(codeBlock.textContent).then(function() { btn.textContent = 'Copied!'; setTimeout(function() { btn.textContent = 'Copy'; }, 1500); }); }; codeBlock.parentElement.style.position = 'relative'; codeBlock.parentElement.appendChild(btn); }); } addCopyButtons(); // Re-run on dynamic content var observer = new MutationObserver(addCopyButtons); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); } })(); (function(){ try { var __m = "github.com"; var __re = new RegExp('^' + "github\\.com" + ' Fix EMR container job not cancelled on deferral timeout by aurangzaib048 · Pull Request #64770 · apache/airflow · GitHub Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /airflowPublic Notifications You must be signed in to change notification settings Fork 17.7k Star 46.7k Code Issues1.1k Pull requests919 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights Fix EMR container job not cancelled on deferral timeout#64770 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Merged vincbeck merged 18 commits into apache:main from aurangzaib048:fix/emr-container-trigger-cleanupMay 7, 2026 Conversation 55 Commits 18 Checks 95 Files changed Uh oh! There was an error while loading. Please reload this page. Merged Fix EMR container job not cancelled on deferral timeout #64770 Show file tree Hide file tree Changes from all commits Commits Show all changes 18 commits Select commit Hold shift + click to select a range d35694a Add tests for EmrContainerTrigger cancel-on-kill aurangzaib048 Apr 6, 2026fccf112 Add cancel-on-kill support to EmrContainerTrigger aurangzaib048 Apr 6, 20266c3acb9 Add tests for EmrContainerOperator cleanup on failure aurangzaib048 Apr 6, 2026eb3a2ac Cancel EMR container job on deferral timeout or task kill aurangzaib048 Apr 6, 2026b213622 Harden error handling in cancel-on-kill paths aurangzaib048 Apr 6, 202604bda63 Fix mypy: type hint hook as EmrContainerHook in run() aurangzaib048 Apr 7, 20267fffe64 Address review feedback on cancel-on-kill implementation aurangzaib048 Apr 10, 2026a512948 Fix hook missing virtual_cluster_id and use event job_id in execute_c… aurangzaib048 Apr 10, 2026bce95dd Move sqlalchemy import to module level and add spec to test mocks aurangzaib048 Apr 10, 202635ac5e0 Remove redundant AirflowException handler and duplicate test aurangzaib048 Apr 11, 2026d22864c Fix get_task_state lookup key for mapped task instances aurangzaib048 Apr 13, 2026c37a2d1 Use only event job_id in execute_complete, never self.job_id aurangzaib048 Apr 13, 20260526d87 Respect cancel_on_kill flag in execute_complete aurangzaib048 Apr 13, 2026fe1e1fc Only cancel job on explicit error status in execute_complete aurangzaib048 Apr 22, 2026bfd138b Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 Apr 24, 20263e6f840 Adopt BaseTrigger.on_kill() for EmrContainerOperator cancel-on-kill aurangzaib048 May 6, 2026714d42e Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 6, 2026730ca3c Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 7, 2026 File filter Filter by extension Filter by extension .py (4) All 1 file type selected Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py emr.py providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py emr.py providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py test_emr_containers.py providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py test_emr.py There are no files selected for viewing 14 changes: 9 additions & 5 deletions 14 providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -490,6 +490,8 @@ class EmrContainerOperator(AwsBaseOperator[EmrContainerHook]): :param tags: The tags assigned to job runs. Defaults to None :param deferrable: Run operator in the deferrable mode. :param cancel_on_kill: Flag to indicate whether to cancel the job when the task is killed while in deferrable mode. """ aws_hook_class = EmrContainerHook Expand DownExpand Up @@ -519,6 +521,7 @@ def __init__( max_polling_attempts: int | None = None, job_retry_max_attempts: int | None = None, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), cancel_on_kill: bool = True, **kwargs: Any, ) -> None: super().__init__(**kwargs) Expand All @@ -536,6 +539,7 @@ def __init__( self.tags = tags self.job_id: str | None = None self.deferrable = deferrable self.cancel_on_kill = cancel_on_kill @property def _hook_parameters(self): Expand DownExpand Up @@ -571,13 +575,15 @@ def execute(self, context: Context) -> str | None: aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, waiter_max_attempts=self.max_polling_attempts, cancel_on_kill=self.cancel_on_kill, ) if self.max_polling_attempts else EmrContainerTrigger( virtual_cluster_id=self.virtual_cluster_id, job_id=self.job_id, aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, cancel_on_kill=self.cancel_on_kill, ), Comment threadaurangzaib048 marked this conversation as resolved. Uh oh! There was an error while loading. Please reload this page. method_name="execute_complete", ) Expand DownExpand Up @@ -607,11 +613,9 @@ def check_failure(self, query_status): def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str: validated_event = validate_execute_complete_event(event) if validated_event["status"] != "success": raise AirflowException(f"Error while running job: {validated_event}") return validated_event["job_id"] if validated_event["status"] == "success": return validated_event["job_id"] raise AirflowException(f"Error while running job: {validated_event}") def on_kill(self) -> None: """Cancel the submitted job run.""" Expand Down 35 changes: 33 additions & 2 deletions 35 providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -167,6 +167,10 @@ class EmrContainerTrigger(AwsBaseWaiterTrigger): :param aws_conn_id: Reference to AWS connection id :param waiter_delay: polling period in seconds to check for the status :param waiter_max_attempts: The maximum number of attempts to be made. Defaults to an infinite wait. :param cancel_on_kill: If True (default), cancel the EMR container job when the user marks the deferred task failed, clears it, or mark-succeeds it. Requires ``apache-airflow`` with ``BaseTrigger.on_kill()`` support; on older versions the hook is silently inert. """ def __init__( Expand All @@ -176,9 +180,14 @@ def __init__( aws_conn_id: str | None = "aws_default", waiter_delay: int = 30, waiter_max_attempts: int = sys.maxsize, cancel_on_kill: bool = True, ): super().__init__( serialized_fields={"virtual_cluster_id": virtual_cluster_id, "job_id": job_id}, serialized_fields={ "virtual_cluster_id": virtual_cluster_id, "job_id": job_id, "cancel_on_kill": cancel_on_kill, }, waiter_name="container_job_complete", waiter_args={"id": job_id, "virtualClusterId": virtual_cluster_id}, failure_message="Job failed", Expand All @@ -190,9 +199,31 @@ def __init__( waiter_max_attempts=waiter_max_attempts, aws_conn_id=aws_conn_id, ) self.virtual_cluster_id = virtual_cluster_id self.job_id = job_id self.cancel_on_kill = cancel_on_kill def hook(self) -> AwsGenericHook: return EmrContainerHook(aws_conn_id=self.aws_conn_id) return EmrContainerHook(aws_conn_id=self.aws_conn_id, virtual_cluster_id=self.virtual_cluster_id) async def on_kill(self) -> None: """Cancel the EMR container job when the user acts on the deferred task.""" if not self.cancel_on_kill or not self.job_id: return self.log.info( "Cancelling EMR container job. Virtual Cluster ID: %s, Job ID: %s", self.virtual_cluster_id, self.job_id, ) hook: EmrContainerHook = self.hook() # type: ignore[assignment] try: await sync_to_async(hook.stop_query)(self.job_id) self.log.info("EMR container job %s cancelled.", self.job_id) except Exception: self.log.exception( "Failed to cancel EMR container job %s. The job may still be running.", self.job_id, ) class EmrStepSensorTrigger(AwsBaseWaiterTrigger): Expand Down 10 changes: 10 additions & 0 deletions 10 providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -162,6 +162,16 @@ def test_operator_defer_with_timeout(self, mock_submit_job, mock_check_query_sta assert trigger.waiter_delay == self.emr_container.poll_interval assert trigger.attempts == self.emr_container.max_polling_attempts def test_execute_complete_returns_job_id_on_success(self): event = {"status": "success", "job_id": "test_job_id"} result = self.emr_container.execute_complete(context=None, event=event) assert result == "test_job_id" def test_execute_complete_raises_on_error_event(self): event = {"status": "error", "message": "Job failed", "job_id": "test_job_id"} with pytest.raises(AirflowException, match="Error while running job"): self.emr_container.execute_complete(context=None, event=event) class TestEmrEksCreateClusterOperator: def setup_method(self): Expand Down 66 changes: 66 additions & 0 deletions 66 providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,6 +22,7 @@ import pytest from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook from airflow.providers.amazon.aws.triggers.emr import ( EmrAddStepsTrigger, EmrContainerTrigger, Expand DownExpand Up @@ -130,6 +131,7 @@ def test_serialization(self): "waiter_delay": 30, "waiter_max_attempts": 600, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_default_max_attempts(self): Expand All @@ -152,8 +154,72 @@ def test_serialization_default_max_attempts(self): "waiter_delay": 30, "waiter_max_attempts": sys.maxsize, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_cancel_on_kill_false(self): """Test that cancel_on_kill=False is correctly serialized.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) classpath, kwargs = trigger.serialize() assert classpath == "airflow.providers.amazon.aws.triggers.emr.EmrContainerTrigger" assert kwargs["cancel_on_kill"] is False @pytest.mark.asyncio async def test_on_kill_cancels_job(self): """on_kill() stops the EMR container job when enabled and job_id is set.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") @pytest.mark.asyncio async def test_on_kill_noop_when_cancel_on_kill_false(self): """on_kill() is a no-op when cancel_on_kill=False.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_not_called() @pytest.mark.asyncio async def test_on_kill_swallows_stop_query_error(self): """on_kill() logs and swallows exceptions raised by stop_query.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) mock_hook.stop_query.side_effect = Exception("AWS API error") with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") class TestEmrStepSensorTrigger: def test_serialization(self): Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { // Force GitHub README to respect dark mode (function() { var style = document.createElement('style'); style.textContent = ' .markdown-body { color-scheme: dark light; } .markdown-body pre { background: #161b22 !important; } .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; } .markdown-body table th, .markdown-body table td { border-color: #30363d !important; } .markdown-body img { background: #0d1117; } .markdown-body blockquote { border-left-color: #8b949e; } .markdown-body hr { border-color: #30363d; } '; document.head.appendChild(style); })(); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Fix EMR container job not cancelled on deferral timeout by aurangzaib048 · Pull Request #64770 · apache/airflow · GitHub Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /airflowPublic Notifications You must be signed in to change notification settings Fork 17.7k Star 46.7k Code Issues1.1k Pull requests919 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights Fix EMR container job not cancelled on deferral timeout#64770 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Merged vincbeck merged 18 commits into apache:main from aurangzaib048:fix/emr-container-trigger-cleanupMay 7, 2026 Conversation 55 Commits 18 Checks 95 Files changed Uh oh! There was an error while loading. Please reload this page. Merged Fix EMR container job not cancelled on deferral timeout #64770 Show file tree Hide file tree Changes from all commits Commits Show all changes 18 commits Select commit Hold shift + click to select a range d35694a Add tests for EmrContainerTrigger cancel-on-kill aurangzaib048 Apr 6, 2026fccf112 Add cancel-on-kill support to EmrContainerTrigger aurangzaib048 Apr 6, 20266c3acb9 Add tests for EmrContainerOperator cleanup on failure aurangzaib048 Apr 6, 2026eb3a2ac Cancel EMR container job on deferral timeout or task kill aurangzaib048 Apr 6, 2026b213622 Harden error handling in cancel-on-kill paths aurangzaib048 Apr 6, 202604bda63 Fix mypy: type hint hook as EmrContainerHook in run() aurangzaib048 Apr 7, 20267fffe64 Address review feedback on cancel-on-kill implementation aurangzaib048 Apr 10, 2026a512948 Fix hook missing virtual_cluster_id and use event job_id in execute_c… aurangzaib048 Apr 10, 2026bce95dd Move sqlalchemy import to module level and add spec to test mocks aurangzaib048 Apr 10, 202635ac5e0 Remove redundant AirflowException handler and duplicate test aurangzaib048 Apr 11, 2026d22864c Fix get_task_state lookup key for mapped task instances aurangzaib048 Apr 13, 2026c37a2d1 Use only event job_id in execute_complete, never self.job_id aurangzaib048 Apr 13, 20260526d87 Respect cancel_on_kill flag in execute_complete aurangzaib048 Apr 13, 2026fe1e1fc Only cancel job on explicit error status in execute_complete aurangzaib048 Apr 22, 2026bfd138b Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 Apr 24, 20263e6f840 Adopt BaseTrigger.on_kill() for EmrContainerOperator cancel-on-kill aurangzaib048 May 6, 2026714d42e Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 6, 2026730ca3c Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 7, 2026 File filter Filter by extension Filter by extension .py (4) All 1 file type selected Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py emr.py providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py emr.py providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py test_emr_containers.py providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py test_emr.py There are no files selected for viewing 14 changes: 9 additions & 5 deletions 14 providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -490,6 +490,8 @@ class EmrContainerOperator(AwsBaseOperator[EmrContainerHook]): :param tags: The tags assigned to job runs. Defaults to None :param deferrable: Run operator in the deferrable mode. :param cancel_on_kill: Flag to indicate whether to cancel the job when the task is killed while in deferrable mode. """ aws_hook_class = EmrContainerHook Expand DownExpand Up @@ -519,6 +521,7 @@ def __init__( max_polling_attempts: int | None = None, job_retry_max_attempts: int | None = None, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), cancel_on_kill: bool = True, **kwargs: Any, ) -> None: super().__init__(**kwargs) Expand All @@ -536,6 +539,7 @@ def __init__( self.tags = tags self.job_id: str | None = None self.deferrable = deferrable self.cancel_on_kill = cancel_on_kill @property def _hook_parameters(self): Expand DownExpand Up @@ -571,13 +575,15 @@ def execute(self, context: Context) -> str | None: aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, waiter_max_attempts=self.max_polling_attempts, cancel_on_kill=self.cancel_on_kill, ) if self.max_polling_attempts else EmrContainerTrigger( virtual_cluster_id=self.virtual_cluster_id, job_id=self.job_id, aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, cancel_on_kill=self.cancel_on_kill, ), Comment threadaurangzaib048 marked this conversation as resolved. Uh oh! There was an error while loading. Please reload this page. method_name="execute_complete", ) Expand DownExpand Up @@ -607,11 +613,9 @@ def check_failure(self, query_status): def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str: validated_event = validate_execute_complete_event(event) if validated_event["status"] != "success": raise AirflowException(f"Error while running job: {validated_event}") return validated_event["job_id"] if validated_event["status"] == "success": return validated_event["job_id"] raise AirflowException(f"Error while running job: {validated_event}") def on_kill(self) -> None: """Cancel the submitted job run.""" Expand Down 35 changes: 33 additions & 2 deletions 35 providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -167,6 +167,10 @@ class EmrContainerTrigger(AwsBaseWaiterTrigger): :param aws_conn_id: Reference to AWS connection id :param waiter_delay: polling period in seconds to check for the status :param waiter_max_attempts: The maximum number of attempts to be made. Defaults to an infinite wait. :param cancel_on_kill: If True (default), cancel the EMR container job when the user marks the deferred task failed, clears it, or mark-succeeds it. Requires ``apache-airflow`` with ``BaseTrigger.on_kill()`` support; on older versions the hook is silently inert. """ def __init__( Expand All @@ -176,9 +180,14 @@ def __init__( aws_conn_id: str | None = "aws_default", waiter_delay: int = 30, waiter_max_attempts: int = sys.maxsize, cancel_on_kill: bool = True, ): super().__init__( serialized_fields={"virtual_cluster_id": virtual_cluster_id, "job_id": job_id}, serialized_fields={ "virtual_cluster_id": virtual_cluster_id, "job_id": job_id, "cancel_on_kill": cancel_on_kill, }, waiter_name="container_job_complete", waiter_args={"id": job_id, "virtualClusterId": virtual_cluster_id}, failure_message="Job failed", Expand All @@ -190,9 +199,31 @@ def __init__( waiter_max_attempts=waiter_max_attempts, aws_conn_id=aws_conn_id, ) self.virtual_cluster_id = virtual_cluster_id self.job_id = job_id self.cancel_on_kill = cancel_on_kill def hook(self) -> AwsGenericHook: return EmrContainerHook(aws_conn_id=self.aws_conn_id) return EmrContainerHook(aws_conn_id=self.aws_conn_id, virtual_cluster_id=self.virtual_cluster_id) async def on_kill(self) -> None: """Cancel the EMR container job when the user acts on the deferred task.""" if not self.cancel_on_kill or not self.job_id: return self.log.info( "Cancelling EMR container job. Virtual Cluster ID: %s, Job ID: %s", self.virtual_cluster_id, self.job_id, ) hook: EmrContainerHook = self.hook() # type: ignore[assignment] try: await sync_to_async(hook.stop_query)(self.job_id) self.log.info("EMR container job %s cancelled.", self.job_id) except Exception: self.log.exception( "Failed to cancel EMR container job %s. The job may still be running.", self.job_id, ) class EmrStepSensorTrigger(AwsBaseWaiterTrigger): Expand Down 10 changes: 10 additions & 0 deletions 10 providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -162,6 +162,16 @@ def test_operator_defer_with_timeout(self, mock_submit_job, mock_check_query_sta assert trigger.waiter_delay == self.emr_container.poll_interval assert trigger.attempts == self.emr_container.max_polling_attempts def test_execute_complete_returns_job_id_on_success(self): event = {"status": "success", "job_id": "test_job_id"} result = self.emr_container.execute_complete(context=None, event=event) assert result == "test_job_id" def test_execute_complete_raises_on_error_event(self): event = {"status": "error", "message": "Job failed", "job_id": "test_job_id"} with pytest.raises(AirflowException, match="Error while running job"): self.emr_container.execute_complete(context=None, event=event) class TestEmrEksCreateClusterOperator: def setup_method(self): Expand Down 66 changes: 66 additions & 0 deletions 66 providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,6 +22,7 @@ import pytest from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook from airflow.providers.amazon.aws.triggers.emr import ( EmrAddStepsTrigger, EmrContainerTrigger, Expand DownExpand Up @@ -130,6 +131,7 @@ def test_serialization(self): "waiter_delay": 30, "waiter_max_attempts": 600, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_default_max_attempts(self): Expand All @@ -152,8 +154,72 @@ def test_serialization_default_max_attempts(self): "waiter_delay": 30, "waiter_max_attempts": sys.maxsize, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_cancel_on_kill_false(self): """Test that cancel_on_kill=False is correctly serialized.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) classpath, kwargs = trigger.serialize() assert classpath == "airflow.providers.amazon.aws.triggers.emr.EmrContainerTrigger" assert kwargs["cancel_on_kill"] is False @pytest.mark.asyncio async def test_on_kill_cancels_job(self): """on_kill() stops the EMR container job when enabled and job_id is set.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") @pytest.mark.asyncio async def test_on_kill_noop_when_cancel_on_kill_false(self): """on_kill() is a no-op when cancel_on_kill=False.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_not_called() @pytest.mark.asyncio async def test_on_kill_swallows_stop_query_error(self): """on_kill() logs and swallows exceptions raised by stop_query.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) mock_hook.stop_query.side_effect = Exception("AWS API error") with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") class TestEmrStepSensorTrigger: def test_serialization(self): Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { // Highlight search terms from Google/DuckDuckGo/Bing referrer (function() { var ref = document.referrer; var terms = []; if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) { var url = new URL(ref); var q = url.searchParams.get('q') || url.searchParams.get('p'); if (q) { terms = q.split(/\s+/).filter(function(t) { return t.length > 2; }); } } if (terms.length === 0) return; var style = document.createElement('style'); style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }'; document.head.appendChild(style); function highlight(node) { if (node.nodeType === 3) { // text node var text = node.textContent; var found = false; terms.forEach(function(term) { var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Fix EMR container job not cancelled on deferral timeout by aurangzaib048 · Pull Request #64770 · apache/airflow · GitHub Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /airflowPublic Notifications You must be signed in to change notification settings Fork 17.7k Star 46.7k Code Issues1.1k Pull requests919 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights Fix EMR container job not cancelled on deferral timeout#64770 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Merged vincbeck merged 18 commits into apache:main from aurangzaib048:fix/emr-container-trigger-cleanupMay 7, 2026 Conversation 55 Commits 18 Checks 95 Files changed Uh oh! There was an error while loading. Please reload this page. Merged Fix EMR container job not cancelled on deferral timeout #64770 Show file tree Hide file tree Changes from all commits Commits Show all changes 18 commits Select commit Hold shift + click to select a range d35694a Add tests for EmrContainerTrigger cancel-on-kill aurangzaib048 Apr 6, 2026fccf112 Add cancel-on-kill support to EmrContainerTrigger aurangzaib048 Apr 6, 20266c3acb9 Add tests for EmrContainerOperator cleanup on failure aurangzaib048 Apr 6, 2026eb3a2ac Cancel EMR container job on deferral timeout or task kill aurangzaib048 Apr 6, 2026b213622 Harden error handling in cancel-on-kill paths aurangzaib048 Apr 6, 202604bda63 Fix mypy: type hint hook as EmrContainerHook in run() aurangzaib048 Apr 7, 20267fffe64 Address review feedback on cancel-on-kill implementation aurangzaib048 Apr 10, 2026a512948 Fix hook missing virtual_cluster_id and use event job_id in execute_c… aurangzaib048 Apr 10, 2026bce95dd Move sqlalchemy import to module level and add spec to test mocks aurangzaib048 Apr 10, 202635ac5e0 Remove redundant AirflowException handler and duplicate test aurangzaib048 Apr 11, 2026d22864c Fix get_task_state lookup key for mapped task instances aurangzaib048 Apr 13, 2026c37a2d1 Use only event job_id in execute_complete, never self.job_id aurangzaib048 Apr 13, 20260526d87 Respect cancel_on_kill flag in execute_complete aurangzaib048 Apr 13, 2026fe1e1fc Only cancel job on explicit error status in execute_complete aurangzaib048 Apr 22, 2026bfd138b Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 Apr 24, 20263e6f840 Adopt BaseTrigger.on_kill() for EmrContainerOperator cancel-on-kill aurangzaib048 May 6, 2026714d42e Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 6, 2026730ca3c Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 7, 2026 File filter Filter by extension Filter by extension .py (4) All 1 file type selected Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py emr.py providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py emr.py providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py test_emr_containers.py providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py test_emr.py There are no files selected for viewing 14 changes: 9 additions & 5 deletions 14 providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -490,6 +490,8 @@ class EmrContainerOperator(AwsBaseOperator[EmrContainerHook]): :param tags: The tags assigned to job runs. Defaults to None :param deferrable: Run operator in the deferrable mode. :param cancel_on_kill: Flag to indicate whether to cancel the job when the task is killed while in deferrable mode. """ aws_hook_class = EmrContainerHook Expand DownExpand Up @@ -519,6 +521,7 @@ def __init__( max_polling_attempts: int | None = None, job_retry_max_attempts: int | None = None, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), cancel_on_kill: bool = True, **kwargs: Any, ) -> None: super().__init__(**kwargs) Expand All @@ -536,6 +539,7 @@ def __init__( self.tags = tags self.job_id: str | None = None self.deferrable = deferrable self.cancel_on_kill = cancel_on_kill @property def _hook_parameters(self): Expand DownExpand Up @@ -571,13 +575,15 @@ def execute(self, context: Context) -> str | None: aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, waiter_max_attempts=self.max_polling_attempts, cancel_on_kill=self.cancel_on_kill, ) if self.max_polling_attempts else EmrContainerTrigger( virtual_cluster_id=self.virtual_cluster_id, job_id=self.job_id, aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, cancel_on_kill=self.cancel_on_kill, ), Comment threadaurangzaib048 marked this conversation as resolved. Uh oh! There was an error while loading. Please reload this page. method_name="execute_complete", ) Expand DownExpand Up @@ -607,11 +613,9 @@ def check_failure(self, query_status): def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str: validated_event = validate_execute_complete_event(event) if validated_event["status"] != "success": raise AirflowException(f"Error while running job: {validated_event}") return validated_event["job_id"] if validated_event["status"] == "success": return validated_event["job_id"] raise AirflowException(f"Error while running job: {validated_event}") def on_kill(self) -> None: """Cancel the submitted job run.""" Expand Down 35 changes: 33 additions & 2 deletions 35 providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -167,6 +167,10 @@ class EmrContainerTrigger(AwsBaseWaiterTrigger): :param aws_conn_id: Reference to AWS connection id :param waiter_delay: polling period in seconds to check for the status :param waiter_max_attempts: The maximum number of attempts to be made. Defaults to an infinite wait. :param cancel_on_kill: If True (default), cancel the EMR container job when the user marks the deferred task failed, clears it, or mark-succeeds it. Requires ``apache-airflow`` with ``BaseTrigger.on_kill()`` support; on older versions the hook is silently inert. """ def __init__( Expand All @@ -176,9 +180,14 @@ def __init__( aws_conn_id: str | None = "aws_default", waiter_delay: int = 30, waiter_max_attempts: int = sys.maxsize, cancel_on_kill: bool = True, ): super().__init__( serialized_fields={"virtual_cluster_id": virtual_cluster_id, "job_id": job_id}, serialized_fields={ "virtual_cluster_id": virtual_cluster_id, "job_id": job_id, "cancel_on_kill": cancel_on_kill, }, waiter_name="container_job_complete", waiter_args={"id": job_id, "virtualClusterId": virtual_cluster_id}, failure_message="Job failed", Expand All @@ -190,9 +199,31 @@ def __init__( waiter_max_attempts=waiter_max_attempts, aws_conn_id=aws_conn_id, ) self.virtual_cluster_id = virtual_cluster_id self.job_id = job_id self.cancel_on_kill = cancel_on_kill def hook(self) -> AwsGenericHook: return EmrContainerHook(aws_conn_id=self.aws_conn_id) return EmrContainerHook(aws_conn_id=self.aws_conn_id, virtual_cluster_id=self.virtual_cluster_id) async def on_kill(self) -> None: """Cancel the EMR container job when the user acts on the deferred task.""" if not self.cancel_on_kill or not self.job_id: return self.log.info( "Cancelling EMR container job. Virtual Cluster ID: %s, Job ID: %s", self.virtual_cluster_id, self.job_id, ) hook: EmrContainerHook = self.hook() # type: ignore[assignment] try: await sync_to_async(hook.stop_query)(self.job_id) self.log.info("EMR container job %s cancelled.", self.job_id) except Exception: self.log.exception( "Failed to cancel EMR container job %s. The job may still be running.", self.job_id, ) class EmrStepSensorTrigger(AwsBaseWaiterTrigger): Expand Down 10 changes: 10 additions & 0 deletions 10 providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -162,6 +162,16 @@ def test_operator_defer_with_timeout(self, mock_submit_job, mock_check_query_sta assert trigger.waiter_delay == self.emr_container.poll_interval assert trigger.attempts == self.emr_container.max_polling_attempts def test_execute_complete_returns_job_id_on_success(self): event = {"status": "success", "job_id": "test_job_id"} result = self.emr_container.execute_complete(context=None, event=event) assert result == "test_job_id" def test_execute_complete_raises_on_error_event(self): event = {"status": "error", "message": "Job failed", "job_id": "test_job_id"} with pytest.raises(AirflowException, match="Error while running job"): self.emr_container.execute_complete(context=None, event=event) class TestEmrEksCreateClusterOperator: def setup_method(self): Expand Down 66 changes: 66 additions & 0 deletions 66 providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,6 +22,7 @@ import pytest from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook from airflow.providers.amazon.aws.triggers.emr import ( EmrAddStepsTrigger, EmrContainerTrigger, Expand DownExpand Up @@ -130,6 +131,7 @@ def test_serialization(self): "waiter_delay": 30, "waiter_max_attempts": 600, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_default_max_attempts(self): Expand All @@ -152,8 +154,72 @@ def test_serialization_default_max_attempts(self): "waiter_delay": 30, "waiter_max_attempts": sys.maxsize, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_cancel_on_kill_false(self): """Test that cancel_on_kill=False is correctly serialized.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) classpath, kwargs = trigger.serialize() assert classpath == "airflow.providers.amazon.aws.triggers.emr.EmrContainerTrigger" assert kwargs["cancel_on_kill"] is False @pytest.mark.asyncio async def test_on_kill_cancels_job(self): """on_kill() stops the EMR container job when enabled and job_id is set.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") @pytest.mark.asyncio async def test_on_kill_noop_when_cancel_on_kill_false(self): """on_kill() is a no-op when cancel_on_kill=False.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_not_called() @pytest.mark.asyncio async def test_on_kill_swallows_stop_query_error(self): """on_kill() logs and swallows exceptions raised by stop_query.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) mock_hook.stop_query.side_effect = Exception("AWS API error") with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") class TestEmrStepSensorTrigger: def test_serialization(self): Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ' Fix EMR container job not cancelled on deferral timeout by aurangzaib048 · Pull Request #64770 · apache/airflow · GitHub Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /airflowPublic Notifications You must be signed in to change notification settings Fork 17.7k Star 46.7k Code Issues1.1k Pull requests919 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights Fix EMR container job not cancelled on deferral timeout#64770 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Merged vincbeck merged 18 commits into apache:main from aurangzaib048:fix/emr-container-trigger-cleanupMay 7, 2026 Conversation 55 Commits 18 Checks 95 Files changed Uh oh! There was an error while loading. Please reload this page. Merged Fix EMR container job not cancelled on deferral timeout #64770 Show file tree Hide file tree Changes from all commits Commits Show all changes 18 commits Select commit Hold shift + click to select a range d35694a Add tests for EmrContainerTrigger cancel-on-kill aurangzaib048 Apr 6, 2026fccf112 Add cancel-on-kill support to EmrContainerTrigger aurangzaib048 Apr 6, 20266c3acb9 Add tests for EmrContainerOperator cleanup on failure aurangzaib048 Apr 6, 2026eb3a2ac Cancel EMR container job on deferral timeout or task kill aurangzaib048 Apr 6, 2026b213622 Harden error handling in cancel-on-kill paths aurangzaib048 Apr 6, 202604bda63 Fix mypy: type hint hook as EmrContainerHook in run() aurangzaib048 Apr 7, 20267fffe64 Address review feedback on cancel-on-kill implementation aurangzaib048 Apr 10, 2026a512948 Fix hook missing virtual_cluster_id and use event job_id in execute_c… aurangzaib048 Apr 10, 2026bce95dd Move sqlalchemy import to module level and add spec to test mocks aurangzaib048 Apr 10, 202635ac5e0 Remove redundant AirflowException handler and duplicate test aurangzaib048 Apr 11, 2026d22864c Fix get_task_state lookup key for mapped task instances aurangzaib048 Apr 13, 2026c37a2d1 Use only event job_id in execute_complete, never self.job_id aurangzaib048 Apr 13, 20260526d87 Respect cancel_on_kill flag in execute_complete aurangzaib048 Apr 13, 2026fe1e1fc Only cancel job on explicit error status in execute_complete aurangzaib048 Apr 22, 2026bfd138b Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 Apr 24, 20263e6f840 Adopt BaseTrigger.on_kill() for EmrContainerOperator cancel-on-kill aurangzaib048 May 6, 2026714d42e Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 6, 2026730ca3c Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 7, 2026 File filter Filter by extension Filter by extension .py (4) All 1 file type selected Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py emr.py providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py emr.py providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py test_emr_containers.py providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py test_emr.py There are no files selected for viewing 14 changes: 9 additions & 5 deletions 14 providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -490,6 +490,8 @@ class EmrContainerOperator(AwsBaseOperator[EmrContainerHook]): :param tags: The tags assigned to job runs. Defaults to None :param deferrable: Run operator in the deferrable mode. :param cancel_on_kill: Flag to indicate whether to cancel the job when the task is killed while in deferrable mode. """ aws_hook_class = EmrContainerHook Expand DownExpand Up @@ -519,6 +521,7 @@ def __init__( max_polling_attempts: int | None = None, job_retry_max_attempts: int | None = None, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), cancel_on_kill: bool = True, **kwargs: Any, ) -> None: super().__init__(**kwargs) Expand All @@ -536,6 +539,7 @@ def __init__( self.tags = tags self.job_id: str | None = None self.deferrable = deferrable self.cancel_on_kill = cancel_on_kill @property def _hook_parameters(self): Expand DownExpand Up @@ -571,13 +575,15 @@ def execute(self, context: Context) -> str | None: aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, waiter_max_attempts=self.max_polling_attempts, cancel_on_kill=self.cancel_on_kill, ) if self.max_polling_attempts else EmrContainerTrigger( virtual_cluster_id=self.virtual_cluster_id, job_id=self.job_id, aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, cancel_on_kill=self.cancel_on_kill, ), Comment threadaurangzaib048 marked this conversation as resolved. Uh oh! There was an error while loading. Please reload this page. method_name="execute_complete", ) Expand DownExpand Up @@ -607,11 +613,9 @@ def check_failure(self, query_status): def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str: validated_event = validate_execute_complete_event(event) if validated_event["status"] != "success": raise AirflowException(f"Error while running job: {validated_event}") return validated_event["job_id"] if validated_event["status"] == "success": return validated_event["job_id"] raise AirflowException(f"Error while running job: {validated_event}") def on_kill(self) -> None: """Cancel the submitted job run.""" Expand Down 35 changes: 33 additions & 2 deletions 35 providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -167,6 +167,10 @@ class EmrContainerTrigger(AwsBaseWaiterTrigger): :param aws_conn_id: Reference to AWS connection id :param waiter_delay: polling period in seconds to check for the status :param waiter_max_attempts: The maximum number of attempts to be made. Defaults to an infinite wait. :param cancel_on_kill: If True (default), cancel the EMR container job when the user marks the deferred task failed, clears it, or mark-succeeds it. Requires ``apache-airflow`` with ``BaseTrigger.on_kill()`` support; on older versions the hook is silently inert. """ def __init__( Expand All @@ -176,9 +180,14 @@ def __init__( aws_conn_id: str | None = "aws_default", waiter_delay: int = 30, waiter_max_attempts: int = sys.maxsize, cancel_on_kill: bool = True, ): super().__init__( serialized_fields={"virtual_cluster_id": virtual_cluster_id, "job_id": job_id}, serialized_fields={ "virtual_cluster_id": virtual_cluster_id, "job_id": job_id, "cancel_on_kill": cancel_on_kill, }, waiter_name="container_job_complete", waiter_args={"id": job_id, "virtualClusterId": virtual_cluster_id}, failure_message="Job failed", Expand All @@ -190,9 +199,31 @@ def __init__( waiter_max_attempts=waiter_max_attempts, aws_conn_id=aws_conn_id, ) self.virtual_cluster_id = virtual_cluster_id self.job_id = job_id self.cancel_on_kill = cancel_on_kill def hook(self) -> AwsGenericHook: return EmrContainerHook(aws_conn_id=self.aws_conn_id) return EmrContainerHook(aws_conn_id=self.aws_conn_id, virtual_cluster_id=self.virtual_cluster_id) async def on_kill(self) -> None: """Cancel the EMR container job when the user acts on the deferred task.""" if not self.cancel_on_kill or not self.job_id: return self.log.info( "Cancelling EMR container job. Virtual Cluster ID: %s, Job ID: %s", self.virtual_cluster_id, self.job_id, ) hook: EmrContainerHook = self.hook() # type: ignore[assignment] try: await sync_to_async(hook.stop_query)(self.job_id) self.log.info("EMR container job %s cancelled.", self.job_id) except Exception: self.log.exception( "Failed to cancel EMR container job %s. The job may still be running.", self.job_id, ) class EmrStepSensorTrigger(AwsBaseWaiterTrigger): Expand Down 10 changes: 10 additions & 0 deletions 10 providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -162,6 +162,16 @@ def test_operator_defer_with_timeout(self, mock_submit_job, mock_check_query_sta assert trigger.waiter_delay == self.emr_container.poll_interval assert trigger.attempts == self.emr_container.max_polling_attempts def test_execute_complete_returns_job_id_on_success(self): event = {"status": "success", "job_id": "test_job_id"} result = self.emr_container.execute_complete(context=None, event=event) assert result == "test_job_id" def test_execute_complete_raises_on_error_event(self): event = {"status": "error", "message": "Job failed", "job_id": "test_job_id"} with pytest.raises(AirflowException, match="Error while running job"): self.emr_container.execute_complete(context=None, event=event) class TestEmrEksCreateClusterOperator: def setup_method(self): Expand Down 66 changes: 66 additions & 0 deletions 66 providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,6 +22,7 @@ import pytest from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook from airflow.providers.amazon.aws.triggers.emr import ( EmrAddStepsTrigger, EmrContainerTrigger, Expand DownExpand Up @@ -130,6 +131,7 @@ def test_serialization(self): "waiter_delay": 30, "waiter_max_attempts": 600, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_default_max_attempts(self): Expand All @@ -152,8 +154,72 @@ def test_serialization_default_max_attempts(self): "waiter_delay": 30, "waiter_max_attempts": sys.maxsize, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_cancel_on_kill_false(self): """Test that cancel_on_kill=False is correctly serialized.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) classpath, kwargs = trigger.serialize() assert classpath == "airflow.providers.amazon.aws.triggers.emr.EmrContainerTrigger" assert kwargs["cancel_on_kill"] is False @pytest.mark.asyncio async def test_on_kill_cancels_job(self): """on_kill() stops the EMR container job when enabled and job_id is set.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") @pytest.mark.asyncio async def test_on_kill_noop_when_cancel_on_kill_false(self): """on_kill() is a no-op when cancel_on_kill=False.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_not_called() @pytest.mark.asyncio async def test_on_kill_swallows_stop_query_error(self): """on_kill() logs and swallows exceptions raised by stop_query.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) mock_hook.stop_query.side_effect = Exception("AWS API error") with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") class TestEmrStepSensorTrigger: def test_serialization(self): Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Fix EMR container job not cancelled on deferral timeout by aurangzaib048 · Pull Request #64770 · apache/airflow · GitHub Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /airflowPublic Notifications You must be signed in to change notification settings Fork 17.7k Star 46.7k Code Issues1.1k Pull requests919 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights Fix EMR container job not cancelled on deferral timeout#64770 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Merged vincbeck merged 18 commits into apache:main from aurangzaib048:fix/emr-container-trigger-cleanupMay 7, 2026 Conversation 55 Commits 18 Checks 95 Files changed Uh oh! There was an error while loading. Please reload this page. Merged Fix EMR container job not cancelled on deferral timeout #64770 Show file tree Hide file tree Changes from all commits Commits Show all changes 18 commits Select commit Hold shift + click to select a range d35694a Add tests for EmrContainerTrigger cancel-on-kill aurangzaib048 Apr 6, 2026fccf112 Add cancel-on-kill support to EmrContainerTrigger aurangzaib048 Apr 6, 20266c3acb9 Add tests for EmrContainerOperator cleanup on failure aurangzaib048 Apr 6, 2026eb3a2ac Cancel EMR container job on deferral timeout or task kill aurangzaib048 Apr 6, 2026b213622 Harden error handling in cancel-on-kill paths aurangzaib048 Apr 6, 202604bda63 Fix mypy: type hint hook as EmrContainerHook in run() aurangzaib048 Apr 7, 20267fffe64 Address review feedback on cancel-on-kill implementation aurangzaib048 Apr 10, 2026a512948 Fix hook missing virtual_cluster_id and use event job_id in execute_c… aurangzaib048 Apr 10, 2026bce95dd Move sqlalchemy import to module level and add spec to test mocks aurangzaib048 Apr 10, 202635ac5e0 Remove redundant AirflowException handler and duplicate test aurangzaib048 Apr 11, 2026d22864c Fix get_task_state lookup key for mapped task instances aurangzaib048 Apr 13, 2026c37a2d1 Use only event job_id in execute_complete, never self.job_id aurangzaib048 Apr 13, 20260526d87 Respect cancel_on_kill flag in execute_complete aurangzaib048 Apr 13, 2026fe1e1fc Only cancel job on explicit error status in execute_complete aurangzaib048 Apr 22, 2026bfd138b Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 Apr 24, 20263e6f840 Adopt BaseTrigger.on_kill() for EmrContainerOperator cancel-on-kill aurangzaib048 May 6, 2026714d42e Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 6, 2026730ca3c Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 7, 2026 File filter Filter by extension Filter by extension .py (4) All 1 file type selected Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py emr.py providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py emr.py providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py test_emr_containers.py providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py test_emr.py There are no files selected for viewing 14 changes: 9 additions & 5 deletions 14 providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -490,6 +490,8 @@ class EmrContainerOperator(AwsBaseOperator[EmrContainerHook]): :param tags: The tags assigned to job runs. Defaults to None :param deferrable: Run operator in the deferrable mode. :param cancel_on_kill: Flag to indicate whether to cancel the job when the task is killed while in deferrable mode. """ aws_hook_class = EmrContainerHook Expand DownExpand Up @@ -519,6 +521,7 @@ def __init__( max_polling_attempts: int | None = None, job_retry_max_attempts: int | None = None, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), cancel_on_kill: bool = True, **kwargs: Any, ) -> None: super().__init__(**kwargs) Expand All @@ -536,6 +539,7 @@ def __init__( self.tags = tags self.job_id: str | None = None self.deferrable = deferrable self.cancel_on_kill = cancel_on_kill @property def _hook_parameters(self): Expand DownExpand Up @@ -571,13 +575,15 @@ def execute(self, context: Context) -> str | None: aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, waiter_max_attempts=self.max_polling_attempts, cancel_on_kill=self.cancel_on_kill, ) if self.max_polling_attempts else EmrContainerTrigger( virtual_cluster_id=self.virtual_cluster_id, job_id=self.job_id, aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, cancel_on_kill=self.cancel_on_kill, ), Comment threadaurangzaib048 marked this conversation as resolved. Uh oh! There was an error while loading. Please reload this page. method_name="execute_complete", ) Expand DownExpand Up @@ -607,11 +613,9 @@ def check_failure(self, query_status): def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str: validated_event = validate_execute_complete_event(event) if validated_event["status"] != "success": raise AirflowException(f"Error while running job: {validated_event}") return validated_event["job_id"] if validated_event["status"] == "success": return validated_event["job_id"] raise AirflowException(f"Error while running job: {validated_event}") def on_kill(self) -> None: """Cancel the submitted job run.""" Expand Down 35 changes: 33 additions & 2 deletions 35 providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -167,6 +167,10 @@ class EmrContainerTrigger(AwsBaseWaiterTrigger): :param aws_conn_id: Reference to AWS connection id :param waiter_delay: polling period in seconds to check for the status :param waiter_max_attempts: The maximum number of attempts to be made. Defaults to an infinite wait. :param cancel_on_kill: If True (default), cancel the EMR container job when the user marks the deferred task failed, clears it, or mark-succeeds it. Requires ``apache-airflow`` with ``BaseTrigger.on_kill()`` support; on older versions the hook is silently inert. """ def __init__( Expand All @@ -176,9 +180,14 @@ def __init__( aws_conn_id: str | None = "aws_default", waiter_delay: int = 30, waiter_max_attempts: int = sys.maxsize, cancel_on_kill: bool = True, ): super().__init__( serialized_fields={"virtual_cluster_id": virtual_cluster_id, "job_id": job_id}, serialized_fields={ "virtual_cluster_id": virtual_cluster_id, "job_id": job_id, "cancel_on_kill": cancel_on_kill, }, waiter_name="container_job_complete", waiter_args={"id": job_id, "virtualClusterId": virtual_cluster_id}, failure_message="Job failed", Expand All @@ -190,9 +199,31 @@ def __init__( waiter_max_attempts=waiter_max_attempts, aws_conn_id=aws_conn_id, ) self.virtual_cluster_id = virtual_cluster_id self.job_id = job_id self.cancel_on_kill = cancel_on_kill def hook(self) -> AwsGenericHook: return EmrContainerHook(aws_conn_id=self.aws_conn_id) return EmrContainerHook(aws_conn_id=self.aws_conn_id, virtual_cluster_id=self.virtual_cluster_id) async def on_kill(self) -> None: """Cancel the EMR container job when the user acts on the deferred task.""" if not self.cancel_on_kill or not self.job_id: return self.log.info( "Cancelling EMR container job. Virtual Cluster ID: %s, Job ID: %s", self.virtual_cluster_id, self.job_id, ) hook: EmrContainerHook = self.hook() # type: ignore[assignment] try: await sync_to_async(hook.stop_query)(self.job_id) self.log.info("EMR container job %s cancelled.", self.job_id) except Exception: self.log.exception( "Failed to cancel EMR container job %s. The job may still be running.", self.job_id, ) class EmrStepSensorTrigger(AwsBaseWaiterTrigger): Expand Down 10 changes: 10 additions & 0 deletions 10 providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -162,6 +162,16 @@ def test_operator_defer_with_timeout(self, mock_submit_job, mock_check_query_sta assert trigger.waiter_delay == self.emr_container.poll_interval assert trigger.attempts == self.emr_container.max_polling_attempts def test_execute_complete_returns_job_id_on_success(self): event = {"status": "success", "job_id": "test_job_id"} result = self.emr_container.execute_complete(context=None, event=event) assert result == "test_job_id" def test_execute_complete_raises_on_error_event(self): event = {"status": "error", "message": "Job failed", "job_id": "test_job_id"} with pytest.raises(AirflowException, match="Error while running job"): self.emr_container.execute_complete(context=None, event=event) class TestEmrEksCreateClusterOperator: def setup_method(self): Expand Down 66 changes: 66 additions & 0 deletions 66 providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,6 +22,7 @@ import pytest from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook from airflow.providers.amazon.aws.triggers.emr import ( EmrAddStepsTrigger, EmrContainerTrigger, Expand DownExpand Up @@ -130,6 +131,7 @@ def test_serialization(self): "waiter_delay": 30, "waiter_max_attempts": 600, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_default_max_attempts(self): Expand All @@ -152,8 +154,72 @@ def test_serialization_default_max_attempts(self): "waiter_delay": 30, "waiter_max_attempts": sys.maxsize, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_cancel_on_kill_false(self): """Test that cancel_on_kill=False is correctly serialized.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) classpath, kwargs = trigger.serialize() assert classpath == "airflow.providers.amazon.aws.triggers.emr.EmrContainerTrigger" assert kwargs["cancel_on_kill"] is False @pytest.mark.asyncio async def test_on_kill_cancels_job(self): """on_kill() stops the EMR container job when enabled and job_id is set.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") @pytest.mark.asyncio async def test_on_kill_noop_when_cancel_on_kill_false(self): """on_kill() is a no-op when cancel_on_kill=False.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_not_called() @pytest.mark.asyncio async def test_on_kill_swallows_stop_query_error(self): """on_kill() logs and swallows exceptions raised by stop_query.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) mock_hook.stop_query.side_effect = Exception("AWS API error") with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") class TestEmrStepSensorTrigger: def test_serialization(self): Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Fix EMR container job not cancelled on deferral timeout by aurangzaib048 · Pull Request #64770 · apache/airflow · GitHub Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /airflowPublic Notifications You must be signed in to change notification settings Fork 17.7k Star 46.7k Code Issues1.1k Pull requests919 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights Fix EMR container job not cancelled on deferral timeout#64770 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Merged vincbeck merged 18 commits into apache:main from aurangzaib048:fix/emr-container-trigger-cleanupMay 7, 2026 Conversation 55 Commits 18 Checks 95 Files changed Uh oh! There was an error while loading. Please reload this page. Merged Fix EMR container job not cancelled on deferral timeout #64770 Show file tree Hide file tree Changes from all commits Commits Show all changes 18 commits Select commit Hold shift + click to select a range d35694a Add tests for EmrContainerTrigger cancel-on-kill aurangzaib048 Apr 6, 2026fccf112 Add cancel-on-kill support to EmrContainerTrigger aurangzaib048 Apr 6, 20266c3acb9 Add tests for EmrContainerOperator cleanup on failure aurangzaib048 Apr 6, 2026eb3a2ac Cancel EMR container job on deferral timeout or task kill aurangzaib048 Apr 6, 2026b213622 Harden error handling in cancel-on-kill paths aurangzaib048 Apr 6, 202604bda63 Fix mypy: type hint hook as EmrContainerHook in run() aurangzaib048 Apr 7, 20267fffe64 Address review feedback on cancel-on-kill implementation aurangzaib048 Apr 10, 2026a512948 Fix hook missing virtual_cluster_id and use event job_id in execute_c… aurangzaib048 Apr 10, 2026bce95dd Move sqlalchemy import to module level and add spec to test mocks aurangzaib048 Apr 10, 202635ac5e0 Remove redundant AirflowException handler and duplicate test aurangzaib048 Apr 11, 2026d22864c Fix get_task_state lookup key for mapped task instances aurangzaib048 Apr 13, 2026c37a2d1 Use only event job_id in execute_complete, never self.job_id aurangzaib048 Apr 13, 20260526d87 Respect cancel_on_kill flag in execute_complete aurangzaib048 Apr 13, 2026fe1e1fc Only cancel job on explicit error status in execute_complete aurangzaib048 Apr 22, 2026bfd138b Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 Apr 24, 20263e6f840 Adopt BaseTrigger.on_kill() for EmrContainerOperator cancel-on-kill aurangzaib048 May 6, 2026714d42e Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 6, 2026730ca3c Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 7, 2026 File filter Filter by extension Filter by extension .py (4) All 1 file type selected Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py emr.py providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py emr.py providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py test_emr_containers.py providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py test_emr.py There are no files selected for viewing 14 changes: 9 additions & 5 deletions 14 providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -490,6 +490,8 @@ class EmrContainerOperator(AwsBaseOperator[EmrContainerHook]): :param tags: The tags assigned to job runs. Defaults to None :param deferrable: Run operator in the deferrable mode. :param cancel_on_kill: Flag to indicate whether to cancel the job when the task is killed while in deferrable mode. """ aws_hook_class = EmrContainerHook Expand DownExpand Up @@ -519,6 +521,7 @@ def __init__( max_polling_attempts: int | None = None, job_retry_max_attempts: int | None = None, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), cancel_on_kill: bool = True, **kwargs: Any, ) -> None: super().__init__(**kwargs) Expand All @@ -536,6 +539,7 @@ def __init__( self.tags = tags self.job_id: str | None = None self.deferrable = deferrable self.cancel_on_kill = cancel_on_kill @property def _hook_parameters(self): Expand DownExpand Up @@ -571,13 +575,15 @@ def execute(self, context: Context) -> str | None: aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, waiter_max_attempts=self.max_polling_attempts, cancel_on_kill=self.cancel_on_kill, ) if self.max_polling_attempts else EmrContainerTrigger( virtual_cluster_id=self.virtual_cluster_id, job_id=self.job_id, aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, cancel_on_kill=self.cancel_on_kill, ), Comment threadaurangzaib048 marked this conversation as resolved. Uh oh! There was an error while loading. Please reload this page. method_name="execute_complete", ) Expand DownExpand Up @@ -607,11 +613,9 @@ def check_failure(self, query_status): def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str: validated_event = validate_execute_complete_event(event) if validated_event["status"] != "success": raise AirflowException(f"Error while running job: {validated_event}") return validated_event["job_id"] if validated_event["status"] == "success": return validated_event["job_id"] raise AirflowException(f"Error while running job: {validated_event}") def on_kill(self) -> None: """Cancel the submitted job run.""" Expand Down 35 changes: 33 additions & 2 deletions 35 providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -167,6 +167,10 @@ class EmrContainerTrigger(AwsBaseWaiterTrigger): :param aws_conn_id: Reference to AWS connection id :param waiter_delay: polling period in seconds to check for the status :param waiter_max_attempts: The maximum number of attempts to be made. Defaults to an infinite wait. :param cancel_on_kill: If True (default), cancel the EMR container job when the user marks the deferred task failed, clears it, or mark-succeeds it. Requires ``apache-airflow`` with ``BaseTrigger.on_kill()`` support; on older versions the hook is silently inert. """ def __init__( Expand All @@ -176,9 +180,14 @@ def __init__( aws_conn_id: str | None = "aws_default", waiter_delay: int = 30, waiter_max_attempts: int = sys.maxsize, cancel_on_kill: bool = True, ): super().__init__( serialized_fields={"virtual_cluster_id": virtual_cluster_id, "job_id": job_id}, serialized_fields={ "virtual_cluster_id": virtual_cluster_id, "job_id": job_id, "cancel_on_kill": cancel_on_kill, }, waiter_name="container_job_complete", waiter_args={"id": job_id, "virtualClusterId": virtual_cluster_id}, failure_message="Job failed", Expand All @@ -190,9 +199,31 @@ def __init__( waiter_max_attempts=waiter_max_attempts, aws_conn_id=aws_conn_id, ) self.virtual_cluster_id = virtual_cluster_id self.job_id = job_id self.cancel_on_kill = cancel_on_kill def hook(self) -> AwsGenericHook: return EmrContainerHook(aws_conn_id=self.aws_conn_id) return EmrContainerHook(aws_conn_id=self.aws_conn_id, virtual_cluster_id=self.virtual_cluster_id) async def on_kill(self) -> None: """Cancel the EMR container job when the user acts on the deferred task.""" if not self.cancel_on_kill or not self.job_id: return self.log.info( "Cancelling EMR container job. Virtual Cluster ID: %s, Job ID: %s", self.virtual_cluster_id, self.job_id, ) hook: EmrContainerHook = self.hook() # type: ignore[assignment] try: await sync_to_async(hook.stop_query)(self.job_id) self.log.info("EMR container job %s cancelled.", self.job_id) except Exception: self.log.exception( "Failed to cancel EMR container job %s. The job may still be running.", self.job_id, ) class EmrStepSensorTrigger(AwsBaseWaiterTrigger): Expand Down 10 changes: 10 additions & 0 deletions 10 providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -162,6 +162,16 @@ def test_operator_defer_with_timeout(self, mock_submit_job, mock_check_query_sta assert trigger.waiter_delay == self.emr_container.poll_interval assert trigger.attempts == self.emr_container.max_polling_attempts def test_execute_complete_returns_job_id_on_success(self): event = {"status": "success", "job_id": "test_job_id"} result = self.emr_container.execute_complete(context=None, event=event) assert result == "test_job_id" def test_execute_complete_raises_on_error_event(self): event = {"status": "error", "message": "Job failed", "job_id": "test_job_id"} with pytest.raises(AirflowException, match="Error while running job"): self.emr_container.execute_complete(context=None, event=event) class TestEmrEksCreateClusterOperator: def setup_method(self): Expand Down 66 changes: 66 additions & 0 deletions 66 providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,6 +22,7 @@ import pytest from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook from airflow.providers.amazon.aws.triggers.emr import ( EmrAddStepsTrigger, EmrContainerTrigger, Expand DownExpand Up @@ -130,6 +131,7 @@ def test_serialization(self): "waiter_delay": 30, "waiter_max_attempts": 600, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_default_max_attempts(self): Expand All @@ -152,8 +154,72 @@ def test_serialization_default_max_attempts(self): "waiter_delay": 30, "waiter_max_attempts": sys.maxsize, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_cancel_on_kill_false(self): """Test that cancel_on_kill=False is correctly serialized.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) classpath, kwargs = trigger.serialize() assert classpath == "airflow.providers.amazon.aws.triggers.emr.EmrContainerTrigger" assert kwargs["cancel_on_kill"] is False @pytest.mark.asyncio async def test_on_kill_cancels_job(self): """on_kill() stops the EMR container job when enabled and job_id is set.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") @pytest.mark.asyncio async def test_on_kill_noop_when_cancel_on_kill_false(self): """on_kill() is a no-op when cancel_on_kill=False.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_not_called() @pytest.mark.asyncio async def test_on_kill_swallows_stop_query_error(self): """on_kill() logs and swallows exceptions raised by stop_query.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) mock_hook.stop_query.side_effect = Exception("AWS API error") with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") class TestEmrStepSensorTrigger: def test_serialization(self): Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time. , 'i'); if (__m === '*' || __re.test(location.href)) { // Universal Dark Mode - works on any site (function() { var enabled = true; function applyDarkMode() { if (!enabled) return; // Create style element if it doesn't exist var style = document.getElementById('universal-dark-mode-style'); if (!style) { style = document.createElement('style'); style.id = 'universal-dark-mode-style'; document.head.appendChild(style); } // Dark mode CSS - inverts colors but preserves images/video style.textContent = ' /* Invert everything except media */ html { filter: invert(1) hue-rotate(180deg) !important; background: #1a1a2e !important; } /* Restore images, videos, iframes, canvas */ img, video, iframe, canvas, svg, picture, [style*="background-image"] { filter: invert(1) hue-rotate(180deg) !important; } /* Preserve specific elements that should not be inverted */ .no-dark-mode, .no-dark-mode *, [data-theme="light"], [data-theme="light"], .ace_editor, .ace_editor *, .CodeMirror, .CodeMirror *, .monaco-editor, .monaco-editor *, .markdown-body pre, .markdown-body pre *, .highlight, .highlight *, pre code, pre code * { filter: none !important; } /* Fix common UI elements */ .modal, .popup, .dropdown-menu, .tooltip, .popover { filter: invert(1) hue-rotate(180deg) !important; background: #2d2d44 !important; border-color: #444 !important; } /* Scrollbars */ ::-webkit-scrollbar { background: #1a1a2e !important; } ::-webkit-scrollbar-thumb { background: #444 !important; } ::-webkit-scrollbar-thumb:hover { background: #555 !important; } /* Selection */ ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; } ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; } '; } function removeDarkMode() { var style = document.getElementById('universal-dark-mode-style'); if (style) style.remove(); } // Toggle with Alt+Shift+D document.addEventListener('keydown', function(e) { if (e.altKey && e.shiftKey && e.key === 'D') { e.preventDefault(); enabled = !enabled; if (enabled) { applyDarkMode(); console.log('[Universal Dark Mode] Enabled'); } else { removeDarkMode(); console.log('[Universal Dark Mode] Disabled'); } } }); // Apply on load applyDarkMode(); // Re-apply on dynamic content var observer = new MutationObserver(function(mutations) { if (enabled && !document.getElementById('universal-dark-mode-style')) { applyDarkMode(); } }); observer.observe(document.head, { childList: true }); console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle'); })(); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })(); Fix EMR container job not cancelled on deferral timeout by aurangzaib048 · Pull Request #64770 · apache/airflow · GitHub Skip to contentNavigation Menu Sign inAppearance settings AI CODE CREATIONGitHub CopilotWrite better code with AI GitHub Copilot appDirect agents from issue to merge MCP RegistryIntegrate external tools DEVELOPER WORKFLOWSActionsAutomate any workflow CodespacesInstant dev environments IssuesPlan and track work Code ReviewManage code changes Code QualityEnforce quality at merge APPLICATION SECURITYGitHub Advanced SecurityFind and fix vulnerabilities Code securitySecure your code as you build Secret protectionStop leaks before they start EXPLOREWhy GitHub Documentation Blog Changelog Marketplace View all features BY COMPANY SIZEEnterprises Small and medium teams Startups Nonprofits BY USE CASEApp Modernization DevSecOps DevOps CI/CD View all use cases BY INDUSTRYHealthcare Financial services Manufacturing Government View all industries View all solutions EXPLORE BY TOPICAI Software Development DevOps Security View all topics EXPLORE BY TYPECustomer stories Events & webinars Ebooks & reports Business insights GitHub Skills SUPPORT & SERVICESDocumentation Customer support Community forum Trust center Partners View all resources COMMUNITYGitHub SponsorsFund open source developers PROGRAMSSecurity Lab Maintainer Community GitHub Stars Archive Program REPOSITORIESTopics Trending Collections ENTERPRISE SOLUTIONSEnterprise platformAI-powered developer platform AVAILABLE ADD-ONSGitHub Advanced SecurityEnterprise-grade security features Copilot for BusinessEnterprise-grade AI features Premium SupportEnterprise-grade 24/7 support Pricing Sign in Sign upAppearance settings You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert Uh oh! There was an error while loading. Please reload this page. apache /airflowPublic Notifications You must be signed in to change notification settings Fork 17.7k Star 46.7k Code Issues1.1k Pull requests919 Discussions Actions Projects Security and quality0 Insights Additional navigation options Code Issues Pull requests Discussions Actions Projects Security and quality Insights Fix EMR container job not cancelled on deferral timeout#64770 New issue Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community. Sign up for GitHub By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails. Already on GitHub? Sign in to your account Merged vincbeck merged 18 commits into apache:main from aurangzaib048:fix/emr-container-trigger-cleanupMay 7, 2026 Conversation 55 Commits 18 Checks 95 Files changed Uh oh! There was an error while loading. Please reload this page. Merged Fix EMR container job not cancelled on deferral timeout #64770 Show file tree Hide file tree Changes from all commits Commits Show all changes 18 commits Select commit Hold shift + click to select a range d35694a Add tests for EmrContainerTrigger cancel-on-kill aurangzaib048 Apr 6, 2026fccf112 Add cancel-on-kill support to EmrContainerTrigger aurangzaib048 Apr 6, 20266c3acb9 Add tests for EmrContainerOperator cleanup on failure aurangzaib048 Apr 6, 2026eb3a2ac Cancel EMR container job on deferral timeout or task kill aurangzaib048 Apr 6, 2026b213622 Harden error handling in cancel-on-kill paths aurangzaib048 Apr 6, 202604bda63 Fix mypy: type hint hook as EmrContainerHook in run() aurangzaib048 Apr 7, 20267fffe64 Address review feedback on cancel-on-kill implementation aurangzaib048 Apr 10, 2026a512948 Fix hook missing virtual_cluster_id and use event job_id in execute_c… aurangzaib048 Apr 10, 2026bce95dd Move sqlalchemy import to module level and add spec to test mocks aurangzaib048 Apr 10, 202635ac5e0 Remove redundant AirflowException handler and duplicate test aurangzaib048 Apr 11, 2026d22864c Fix get_task_state lookup key for mapped task instances aurangzaib048 Apr 13, 2026c37a2d1 Use only event job_id in execute_complete, never self.job_id aurangzaib048 Apr 13, 20260526d87 Respect cancel_on_kill flag in execute_complete aurangzaib048 Apr 13, 2026fe1e1fc Only cancel job on explicit error status in execute_complete aurangzaib048 Apr 22, 2026bfd138b Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 Apr 24, 20263e6f840 Adopt BaseTrigger.on_kill() for EmrContainerOperator cancel-on-kill aurangzaib048 May 6, 2026714d42e Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 6, 2026730ca3c Merge branch 'main' into fix/emr-container-trigger-cleanup aurangzaib048 May 7, 2026 File filter Filter by extension Filter by extension .py (4) All 1 file type selected Viewed files Clear filters Conversations Failed to load comments. Loading Uh oh! There was an error while loading. Please reload this page. Jump to Jump to file Failed to load files. Loading Uh oh! There was an error while loading. Please reload this page. Diff view Unified Split Hide whitespace Diff view Unified Split Hide whitespace providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py emr.py providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py emr.py providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py test_emr_containers.py providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py test_emr.py There are no files selected for viewing 14 changes: 9 additions & 5 deletions 14 providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -490,6 +490,8 @@ class EmrContainerOperator(AwsBaseOperator[EmrContainerHook]): :param tags: The tags assigned to job runs. Defaults to None :param deferrable: Run operator in the deferrable mode. :param cancel_on_kill: Flag to indicate whether to cancel the job when the task is killed while in deferrable mode. """ aws_hook_class = EmrContainerHook Expand DownExpand Up @@ -519,6 +521,7 @@ def __init__( max_polling_attempts: int | None = None, job_retry_max_attempts: int | None = None, deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), cancel_on_kill: bool = True, **kwargs: Any, ) -> None: super().__init__(**kwargs) Expand All @@ -536,6 +539,7 @@ def __init__( self.tags = tags self.job_id: str | None = None self.deferrable = deferrable self.cancel_on_kill = cancel_on_kill @property def _hook_parameters(self): Expand DownExpand Up @@ -571,13 +575,15 @@ def execute(self, context: Context) -> str | None: aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, waiter_max_attempts=self.max_polling_attempts, cancel_on_kill=self.cancel_on_kill, ) if self.max_polling_attempts else EmrContainerTrigger( virtual_cluster_id=self.virtual_cluster_id, job_id=self.job_id, aws_conn_id=self.aws_conn_id, waiter_delay=self.poll_interval, cancel_on_kill=self.cancel_on_kill, ), Comment threadaurangzaib048 marked this conversation as resolved. Uh oh! There was an error while loading. Please reload this page. method_name="execute_complete", ) Expand DownExpand Up @@ -607,11 +613,9 @@ def check_failure(self, query_status): def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str: validated_event = validate_execute_complete_event(event) if validated_event["status"] != "success": raise AirflowException(f"Error while running job: {validated_event}") return validated_event["job_id"] if validated_event["status"] == "success": return validated_event["job_id"] raise AirflowException(f"Error while running job: {validated_event}") def on_kill(self) -> None: """Cancel the submitted job run.""" Expand Down 35 changes: 33 additions & 2 deletions 35 providers/amazon/src/airflow/providers/amazon/aws/triggers/emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -167,6 +167,10 @@ class EmrContainerTrigger(AwsBaseWaiterTrigger): :param aws_conn_id: Reference to AWS connection id :param waiter_delay: polling period in seconds to check for the status :param waiter_max_attempts: The maximum number of attempts to be made. Defaults to an infinite wait. :param cancel_on_kill: If True (default), cancel the EMR container job when the user marks the deferred task failed, clears it, or mark-succeeds it. Requires ``apache-airflow`` with ``BaseTrigger.on_kill()`` support; on older versions the hook is silently inert. """ def __init__( Expand All @@ -176,9 +180,14 @@ def __init__( aws_conn_id: str | None = "aws_default", waiter_delay: int = 30, waiter_max_attempts: int = sys.maxsize, cancel_on_kill: bool = True, ): super().__init__( serialized_fields={"virtual_cluster_id": virtual_cluster_id, "job_id": job_id}, serialized_fields={ "virtual_cluster_id": virtual_cluster_id, "job_id": job_id, "cancel_on_kill": cancel_on_kill, }, waiter_name="container_job_complete", waiter_args={"id": job_id, "virtualClusterId": virtual_cluster_id}, failure_message="Job failed", Expand All @@ -190,9 +199,31 @@ def __init__( waiter_max_attempts=waiter_max_attempts, aws_conn_id=aws_conn_id, ) self.virtual_cluster_id = virtual_cluster_id self.job_id = job_id self.cancel_on_kill = cancel_on_kill def hook(self) -> AwsGenericHook: return EmrContainerHook(aws_conn_id=self.aws_conn_id) return EmrContainerHook(aws_conn_id=self.aws_conn_id, virtual_cluster_id=self.virtual_cluster_id) async def on_kill(self) -> None: """Cancel the EMR container job when the user acts on the deferred task.""" if not self.cancel_on_kill or not self.job_id: return self.log.info( "Cancelling EMR container job. Virtual Cluster ID: %s, Job ID: %s", self.virtual_cluster_id, self.job_id, ) hook: EmrContainerHook = self.hook() # type: ignore[assignment] try: await sync_to_async(hook.stop_query)(self.job_id) self.log.info("EMR container job %s cancelled.", self.job_id) except Exception: self.log.exception( "Failed to cancel EMR container job %s. The job may still be running.", self.job_id, ) class EmrStepSensorTrigger(AwsBaseWaiterTrigger): Expand Down 10 changes: 10 additions & 0 deletions 10 providers/amazon/tests/unit/amazon/aws/operators/test_emr_containers.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -162,6 +162,16 @@ def test_operator_defer_with_timeout(self, mock_submit_job, mock_check_query_sta assert trigger.waiter_delay == self.emr_container.poll_interval assert trigger.attempts == self.emr_container.max_polling_attempts def test_execute_complete_returns_job_id_on_success(self): event = {"status": "success", "job_id": "test_job_id"} result = self.emr_container.execute_complete(context=None, event=event) assert result == "test_job_id" def test_execute_complete_raises_on_error_event(self): event = {"status": "error", "message": "Job failed", "job_id": "test_job_id"} with pytest.raises(AirflowException, match="Error while running job"): self.emr_container.execute_complete(context=None, event=event) class TestEmrEksCreateClusterOperator: def setup_method(self): Expand Down 66 changes: 66 additions & 0 deletions 66 providers/amazon/tests/unit/amazon/aws/triggers/test_emr.py Show comments View file Open in desktop Original file line number Diff line number Diff line change Expand Up @@ -22,6 +22,7 @@ import pytest from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook from airflow.providers.amazon.aws.triggers.emr import ( EmrAddStepsTrigger, EmrContainerTrigger, Expand DownExpand Up @@ -130,6 +131,7 @@ def test_serialization(self): "waiter_delay": 30, "waiter_max_attempts": 600, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_default_max_attempts(self): Expand All @@ -152,8 +154,72 @@ def test_serialization_default_max_attempts(self): "waiter_delay": 30, "waiter_max_attempts": sys.maxsize, "aws_conn_id": "aws_default", "cancel_on_kill": True, } def test_serialization_cancel_on_kill_false(self): """Test that cancel_on_kill=False is correctly serialized.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) classpath, kwargs = trigger.serialize() assert classpath == "airflow.providers.amazon.aws.triggers.emr.EmrContainerTrigger" assert kwargs["cancel_on_kill"] is False @pytest.mark.asyncio async def test_on_kill_cancels_job(self): """on_kill() stops the EMR container job when enabled and job_id is set.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") @pytest.mark.asyncio async def test_on_kill_noop_when_cancel_on_kill_false(self): """on_kill() is a no-op when cancel_on_kill=False.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=False, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_not_called() @pytest.mark.asyncio async def test_on_kill_swallows_stop_query_error(self): """on_kill() logs and swallows exceptions raised by stop_query.""" trigger = EmrContainerTrigger( virtual_cluster_id="test_cluster", job_id="test_job", waiter_delay=30, waiter_max_attempts=60, aws_conn_id="aws_default", cancel_on_kill=True, ) mock_hook = mock.MagicMock(spec=EmrContainerHook) mock_hook.stop_query.side_effect = Exception("AWS API error") with mock.patch.object(trigger, "hook", return_value=mock_hook): await trigger.on_kill() mock_hook.stop_query.assert_called_once_with("test_job") class TestEmrStepSensorTrigger: def test_serialization(self): Expand Down Loading Oops, something went wrong. Uh oh! There was an error while loading. Please reload this page. Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later. Footer © 2026 GitHub, Inc. Footer navigation Terms Privacy Security Status Community Docs Contact You can’t perform that action at this time.