Add Deferrable Mode to RedshiftDeleteClusterOperator#30870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
syedahsn wants to merge 3 commits into from
Closed

Add Deferrable Mode to RedshiftDeleteClusterOperator #30870

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
37ca318
Change base_aws.py to support async_conn
syedahsn Mar 6, 2023
e60898b
Add deferrable mode to redshift delete cluster
syedahsn Mar 22, 2023
cc1c7dd
Rebase on main
syedahsn Apr 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions airflow/providers/amazon/aws/operators/redshift_cluster.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
RedshiftClusterTrigger,
RedshiftCreateClusterTrigger,
RedshiftDeleteClusterTrigger,
)

if TYPE_CHECKING:
Expand DownExpand Up@@ -595,6 +596,9 @@ class RedshiftDeleteClusterOperator(BaseOperator):
The default value is ``True``
:param aws_conn_id: aws connection to use
:param poll_interval: Time (in seconds) to wait between two consecutive calls to check cluster state
:param max_attempts: Number of attempts the cluster should be polled to determine the cluster
was deleted.
:param deferrable: If True, the operator will run as a deferrable operator.
"""

template_fields: Sequence[str] = ("cluster_identifier",)
Expand All@@ -609,7 +613,9 @@ def __init__(
final_cluster_snapshot_identifier: str | None = None,
wait_for_completion: bool = True,
aws_conn_id: str = "aws_default",
poll_interval: float = 30.0,
poll_interval: int = 30,
max_attempts: int = 20,
deferrable: bool = False,
**kwargs,
):
super().__init__(**kwargs)
Expand All@@ -624,6 +630,9 @@ def __init__(
self._attempts = 10
self._attempt_interval = 15
self.redshift_hook = RedshiftHook(aws_conn_id=aws_conn_id)
self.deferrable = deferrable
self.max_attempts = max_attempts
self.aws_conn_id = aws_conn_id

def execute(self, context: Context):
while self._attempts >= 1:
Expand All@@ -642,10 +651,27 @@ def execute(self, context: Context):
time.sleep(self._attempt_interval)
else:
raise
if self.deferrable:
self.defer(
trigger=RedshiftDeleteClusterTrigger(
cluster_identifier=self.cluster_identifier,
poll_interval=self.poll_interval,
max_attempts=self.max_attempts,
aws_conn_id=self.aws_conn_id,
),
method_name="execute_complete",
)

if self.wait_for_completion:
waiter = self.redshift_hook.get_conn().get_waiter("cluster_deleted")
waiter.wait(
ClusterIdentifier=self.cluster_identifier,
WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": 30},
WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": self.max_attempts},
)

def execute_complete(self, context, event=None):
if event["status"] != "success":
raise AirflowException(f"Error deleting cluster: {event}")
else:
self.log.info("Cluster deleted successfully")
return
40 changes: 39 additions & 1 deletion airflow/providers/amazon/aws/triggers/redshift_cluster.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,9 @@
from airflow.compat.functools import cached_property
from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook, RedshiftHook
from airflow.triggers.base import BaseTrigger, TriggerEvent
from typing import Any

from airflow.compat.functools import cached_property


class RedshiftClusterTrigger(BaseTrigger):
Expand DownExpand Up@@ -87,7 +90,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]:
if self.attempts < 1:
yield TriggerEvent({"status": "error", "message": str(e)})


class RedshiftCreateClusterTrigger(BaseTrigger):
"""
Trigger for RedshiftCreateClusterOperator.
Expand DownExpand Up@@ -137,3 +139,39 @@ async def run(self):
},
)
yield TriggerEvent({"status": "success", "message": "Cluster Created"})

class RedshiftDeleteClusterTrigger(BaseTrigger):
def __init__(
self,
cluster_identifier: str,
poll_interval: int,
max_attempts: int,
aws_conn_id: str,
):
self.cluster_identifier = cluster_identifier
self.poll_interval = poll_interval
self.max_attempts = max_attempts
self.aws_conn_id = aws_conn_id

def serialize(self) -> tuple[str, dict[str, Any]]:
return (
"airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger",
{
"cluster_identifier": str(self.cluster_identifier),
"poll_interval": str(self.poll_interval),
"max_attempts": str(self.max_attempts),
"aws_conn_id": str(self.aws_conn_id),
},
)

async def run(self):
self.redshift_hook = RedshiftHook(aws_conn_id=self.aws_conn_id)
async with self.redshift_hook.async_conn as client:
await client.get_waiter("cluster_deleted").wait(
ClusterIdentifier=self.cluster_identifier,
WaiterConfig={
"Delay": int(self.poll_interval),
"MaxAttempts": int(self.max_attempts),
},
)
yield TriggerEvent({"status": "success", "message": "Cluster deleted"})
23 changes: 22 additions & 1 deletion tests/providers/amazon/aws/operators/test_redshift_cluster.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,7 +31,10 @@
RedshiftPauseClusterOperator,
RedshiftResumeClusterOperator,
)
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
RedshiftClusterTrigger,
RedshiftDeleteClusterTrigger,
)


class TestRedshiftCreateClusterOperator:
Expand DownExpand Up@@ -481,3 +484,21 @@ def test_delete_cluster_multiple_attempts_fail(self, _, mock_conn, mock_delete_c
redshift_operator.execute(None)

assert mock_delete_cluster.call_count == 10

@mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.delete_cluster")
def test_delete_cluster_deferrable(self, mock_delete_cluster):
mock_delete_cluster.return_value = True

redshift_operator = RedshiftDeleteClusterOperator(
task_id="task_test",
cluster_identifier="test_cluster",
aws_conn_id="aws_conn_test",
deferrable=True,
)

with pytest.raises(TaskDeferred) as exc:
redshift_operator.execute(None)

assert isinstance(
exc.value.trigger, RedshiftDeleteClusterTrigger
), "Trigger is not a RedshiftDeleteClusterTrigger"
39 changes: 38 additions & 1 deletion tests/providers/amazon/aws/triggers/test_redshift_cluster.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@

import pytest

from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger, RedshiftDeleteClusterTrigger
from airflow.triggers.base import TriggerEvent

if sys.version_info < (3, 8):
Expand DownExpand Up@@ -72,3 +72,40 @@ async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
response = await generator.asend(None)

assert response == TriggerEvent({"status": "success", "message": "Cluster Created"})

class TestRedshiftDeleteClusterTrigger:
def test_redshift_create_cluster_trigger_serialize(self):
redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
cluster_identifier=TEST_CLUSTER_IDENTIFIER,
poll_interval=TEST_POLL_INTERVAL,
max_attempts=TEST_MAX_ATTEMPT,
aws_conn_id=TEST_AWS_CONN_ID,
)
class_path, args = redshift_delete_cluster_trigger.serialize()
assert (
class_path
== "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger"
)
assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER
assert args["poll_interval"] == str(TEST_POLL_INTERVAL)
assert args["max_attempts"] == str(TEST_MAX_ATTEMPT)
assert args["aws_conn_id"] == TEST_AWS_CONN_ID

@pytest.mark.asyncio
@async_mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.async_conn")
async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
mock = async_mock.MagicMock()
mock_async_conn.__aenter__.return_value = mock
mock.get_waiter().wait = AsyncMock()

redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
cluster_identifier=TEST_CLUSTER_IDENTIFIER,
poll_interval=TEST_POLL_INTERVAL,
max_attempts=TEST_MAX_ATTEMPT,
aws_conn_id=TEST_AWS_CONN_ID,
)

generator = redshift_delete_cluster_trigger.run()
response = await generator.asend(None)

assert response == TriggerEvent({"status": "success", "message": "Cluster deleted"})
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Add Deferrable Mode to RedshiftDeleteClusterOperator#30870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

syedahsn
wants to merge
3
commits into

from

 Closed


Add Deferrable Mode to RedshiftDeleteClusterOperator
#30870
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
37ca318

Change base_aws.py to support async_conn


syedahsn Mar 6, 2023
e60898b

Add deferrable mode to redshift delete cluster


syedahsn Mar 22, 2023
cc1c7dd

Rebase on main


syedahsn Apr 25, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


30 changes: 28 additions & 2 deletions
airflow/providers/amazon/aws/operators/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftCreateClusterTrigger,
 RedshiftDeleteClusterTrigger,
)

if TYPE_CHECKING:
Expand DownExpand Up@@ -595,6 +596,9 @@ class RedshiftDeleteClusterOperator(BaseOperator):
 The default value is ``True``
 :param aws_conn_id: aws connection to use
 :param poll_interval: Time (in seconds) to wait between two consecutive calls to check cluster state
 :param max_attempts: Number of attempts the cluster should be polled to determine the cluster
 was deleted.
 :param deferrable: If True, the operator will run as a deferrable operator.
 """

 template_fields: Sequence[str] = ("cluster_identifier",)
Expand All@@ -609,7 +613,9 @@ def __init__(
 final_cluster_snapshot_identifier: str | None = None,
 wait_for_completion: bool = True,
 aws_conn_id: str = "aws_default",
 poll_interval: float = 30.0,
 poll_interval: int = 30,
 max_attempts: int = 20,
 deferrable: bool = False,
 **kwargs,
 ):
 super().__init__(**kwargs)
Expand All@@ -624,6 +630,9 @@ def __init__(
 self._attempts = 10
 self._attempt_interval = 15
 self.redshift_hook = RedshiftHook(aws_conn_id=aws_conn_id)
 self.deferrable = deferrable
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def execute(self, context: Context):
 while self._attempts >= 1:
Expand All@@ -642,10 +651,27 @@ def execute(self, context: Context):
 time.sleep(self._attempt_interval)
 else:
 raise
 if self.deferrable:
 self.defer(
 trigger=RedshiftDeleteClusterTrigger(
 cluster_identifier=self.cluster_identifier,
 poll_interval=self.poll_interval,
 max_attempts=self.max_attempts,
 aws_conn_id=self.aws_conn_id,
 ),
 method_name="execute_complete",
 )

 if self.wait_for_completion:
 waiter = self.redshift_hook.get_conn().get_waiter("cluster_deleted")
 waiter.wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": 30},
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": self.max_attempts},
 )

 def execute_complete(self, context, event=None):
 if event["status"] != "success":
 raise AirflowException(f"Error deleting cluster: {event}")
 else:
 self.log.info("Cluster deleted successfully")
 return

40 changes: 39 additions & 1 deletion
airflow/providers/amazon/aws/triggers/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,9 @@
from airflow.compat.functools import cached_property
from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook, RedshiftHook
from airflow.triggers.base import BaseTrigger, TriggerEvent
from typing import Any

from airflow.compat.functools import cached_property


class RedshiftClusterTrigger(BaseTrigger):
Expand DownExpand Up@@ -87,7 +90,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]:
 if self.attempts < 1:
 yield TriggerEvent({"status": "error", "message": str(e)})


class RedshiftCreateClusterTrigger(BaseTrigger):
 """
 Trigger for RedshiftCreateClusterOperator.
Expand DownExpand Up@@ -137,3 +139,39 @@ async def run(self):
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster Created"})

class RedshiftDeleteClusterTrigger(BaseTrigger):
 def __init__(
 self,
 cluster_identifier: str,
 poll_interval: int,
 max_attempts: int,
 aws_conn_id: str,
 ):
 self.cluster_identifier = cluster_identifier
 self.poll_interval = poll_interval
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def serialize(self) -> tuple[str, dict[str, Any]]:
 return (
 "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger",
 {
 "cluster_identifier": str(self.cluster_identifier),
 "poll_interval": str(self.poll_interval),
 "max_attempts": str(self.max_attempts),
 "aws_conn_id": str(self.aws_conn_id),
 },
 )

 async def run(self):
 self.redshift_hook = RedshiftHook(aws_conn_id=self.aws_conn_id)
 async with self.redshift_hook.async_conn as client:
 await client.get_waiter("cluster_deleted").wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={
 "Delay": int(self.poll_interval),
 "MaxAttempts": int(self.max_attempts),
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster deleted"})

23 changes: 22 additions & 1 deletion
tests/providers/amazon/aws/operators/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,7 +31,10 @@
 RedshiftPauseClusterOperator,
 RedshiftResumeClusterOperator,
)
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftDeleteClusterTrigger,
)


class TestRedshiftCreateClusterOperator:
Expand DownExpand Up@@ -481,3 +484,21 @@ def test_delete_cluster_multiple_attempts_fail(self, _, mock_conn, mock_delete_c
 redshift_operator.execute(None)

 assert mock_delete_cluster.call_count == 10

 @mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.delete_cluster")
 def test_delete_cluster_deferrable(self, mock_delete_cluster):
 mock_delete_cluster.return_value = True

 redshift_operator = RedshiftDeleteClusterOperator(
 task_id="task_test",
 cluster_identifier="test_cluster",
 aws_conn_id="aws_conn_test",
 deferrable=True,
 )

 with pytest.raises(TaskDeferred) as exc:
 redshift_operator.execute(None)

 assert isinstance(
 exc.value.trigger, RedshiftDeleteClusterTrigger
 ), "Trigger is not a RedshiftDeleteClusterTrigger"

39 changes: 38 additions & 1 deletion
tests/providers/amazon/aws/triggers/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@

import pytest

from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger, RedshiftDeleteClusterTrigger
from airflow.triggers.base import TriggerEvent

if sys.version_info < (3, 8):
Expand DownExpand Up@@ -72,3 +72,40 @@ async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster Created"})

class TestRedshiftDeleteClusterTrigger:
 def test_redshift_create_cluster_trigger_serialize(self):
 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )
 class_path, args = redshift_delete_cluster_trigger.serialize()
 assert (
 class_path
 == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger"
 )
 assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER
 assert args["poll_interval"] == str(TEST_POLL_INTERVAL)
 assert args["max_attempts"] == str(TEST_MAX_ATTEMPT)
 assert args["aws_conn_id"] == TEST_AWS_CONN_ID

 @pytest.mark.asyncio
 @async_mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.async_conn")
 async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 mock = async_mock.MagicMock()
 mock_async_conn.__aenter__.return_value = mock
 mock.get_waiter().wait = AsyncMock()

 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )

 generator = redshift_delete_cluster_trigger.run()
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster deleted"})

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add Deferrable Mode to RedshiftDeleteClusterOperator#30870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

syedahsn
wants to merge
3
commits into

from

 Closed


Add Deferrable Mode to RedshiftDeleteClusterOperator
#30870
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
37ca318

Change base_aws.py to support async_conn


syedahsn Mar 6, 2023
e60898b

Add deferrable mode to redshift delete cluster


syedahsn Mar 22, 2023
cc1c7dd

Rebase on main


syedahsn Apr 25, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


30 changes: 28 additions & 2 deletions
airflow/providers/amazon/aws/operators/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftCreateClusterTrigger,
 RedshiftDeleteClusterTrigger,
)

if TYPE_CHECKING:
Expand DownExpand Up@@ -595,6 +596,9 @@ class RedshiftDeleteClusterOperator(BaseOperator):
 The default value is ``True``
 :param aws_conn_id: aws connection to use
 :param poll_interval: Time (in seconds) to wait between two consecutive calls to check cluster state
 :param max_attempts: Number of attempts the cluster should be polled to determine the cluster
 was deleted.
 :param deferrable: If True, the operator will run as a deferrable operator.
 """

 template_fields: Sequence[str] = ("cluster_identifier",)
Expand All@@ -609,7 +613,9 @@ def __init__(
 final_cluster_snapshot_identifier: str | None = None,
 wait_for_completion: bool = True,
 aws_conn_id: str = "aws_default",
 poll_interval: float = 30.0,
 poll_interval: int = 30,
 max_attempts: int = 20,
 deferrable: bool = False,
 **kwargs,
 ):
 super().__init__(**kwargs)
Expand All@@ -624,6 +630,9 @@ def __init__(
 self._attempts = 10
 self._attempt_interval = 15
 self.redshift_hook = RedshiftHook(aws_conn_id=aws_conn_id)
 self.deferrable = deferrable
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def execute(self, context: Context):
 while self._attempts >= 1:
Expand All@@ -642,10 +651,27 @@ def execute(self, context: Context):
 time.sleep(self._attempt_interval)
 else:
 raise
 if self.deferrable:
 self.defer(
 trigger=RedshiftDeleteClusterTrigger(
 cluster_identifier=self.cluster_identifier,
 poll_interval=self.poll_interval,
 max_attempts=self.max_attempts,
 aws_conn_id=self.aws_conn_id,
 ),
 method_name="execute_complete",
 )

 if self.wait_for_completion:
 waiter = self.redshift_hook.get_conn().get_waiter("cluster_deleted")
 waiter.wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": 30},
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": self.max_attempts},
 )

 def execute_complete(self, context, event=None):
 if event["status"] != "success":
 raise AirflowException(f"Error deleting cluster: {event}")
 else:
 self.log.info("Cluster deleted successfully")
 return

40 changes: 39 additions & 1 deletion
airflow/providers/amazon/aws/triggers/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,9 @@
from airflow.compat.functools import cached_property
from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook, RedshiftHook
from airflow.triggers.base import BaseTrigger, TriggerEvent
from typing import Any

from airflow.compat.functools import cached_property


class RedshiftClusterTrigger(BaseTrigger):
Expand DownExpand Up@@ -87,7 +90,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]:
 if self.attempts < 1:
 yield TriggerEvent({"status": "error", "message": str(e)})


class RedshiftCreateClusterTrigger(BaseTrigger):
 """
 Trigger for RedshiftCreateClusterOperator.
Expand DownExpand Up@@ -137,3 +139,39 @@ async def run(self):
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster Created"})

class RedshiftDeleteClusterTrigger(BaseTrigger):
 def __init__(
 self,
 cluster_identifier: str,
 poll_interval: int,
 max_attempts: int,
 aws_conn_id: str,
 ):
 self.cluster_identifier = cluster_identifier
 self.poll_interval = poll_interval
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def serialize(self) -> tuple[str, dict[str, Any]]:
 return (
 "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger",
 {
 "cluster_identifier": str(self.cluster_identifier),
 "poll_interval": str(self.poll_interval),
 "max_attempts": str(self.max_attempts),
 "aws_conn_id": str(self.aws_conn_id),
 },
 )

 async def run(self):
 self.redshift_hook = RedshiftHook(aws_conn_id=self.aws_conn_id)
 async with self.redshift_hook.async_conn as client:
 await client.get_waiter("cluster_deleted").wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={
 "Delay": int(self.poll_interval),
 "MaxAttempts": int(self.max_attempts),
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster deleted"})

23 changes: 22 additions & 1 deletion
tests/providers/amazon/aws/operators/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,7 +31,10 @@
 RedshiftPauseClusterOperator,
 RedshiftResumeClusterOperator,
)
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftDeleteClusterTrigger,
)


class TestRedshiftCreateClusterOperator:
Expand DownExpand Up@@ -481,3 +484,21 @@ def test_delete_cluster_multiple_attempts_fail(self, _, mock_conn, mock_delete_c
 redshift_operator.execute(None)

 assert mock_delete_cluster.call_count == 10

 @mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.delete_cluster")
 def test_delete_cluster_deferrable(self, mock_delete_cluster):
 mock_delete_cluster.return_value = True

 redshift_operator = RedshiftDeleteClusterOperator(
 task_id="task_test",
 cluster_identifier="test_cluster",
 aws_conn_id="aws_conn_test",
 deferrable=True,
 )

 with pytest.raises(TaskDeferred) as exc:
 redshift_operator.execute(None)

 assert isinstance(
 exc.value.trigger, RedshiftDeleteClusterTrigger
 ), "Trigger is not a RedshiftDeleteClusterTrigger"

39 changes: 38 additions & 1 deletion
tests/providers/amazon/aws/triggers/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@

import pytest

from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger, RedshiftDeleteClusterTrigger
from airflow.triggers.base import TriggerEvent

if sys.version_info < (3, 8):
Expand DownExpand Up@@ -72,3 +72,40 @@ async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster Created"})

class TestRedshiftDeleteClusterTrigger:
 def test_redshift_create_cluster_trigger_serialize(self):
 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )
 class_path, args = redshift_delete_cluster_trigger.serialize()
 assert (
 class_path
 == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger"
 )
 assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER
 assert args["poll_interval"] == str(TEST_POLL_INTERVAL)
 assert args["max_attempts"] == str(TEST_MAX_ATTEMPT)
 assert args["aws_conn_id"] == TEST_AWS_CONN_ID

 @pytest.mark.asyncio
 @async_mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.async_conn")
 async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 mock = async_mock.MagicMock()
 mock_async_conn.__aenter__.return_value = mock
 mock.get_waiter().wait = AsyncMock()

 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )

 generator = redshift_delete_cluster_trigger.run()
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster deleted"})

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add Deferrable Mode to RedshiftDeleteClusterOperator#30870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

syedahsn
wants to merge
3
commits into

from

 Closed


Add Deferrable Mode to RedshiftDeleteClusterOperator
#30870
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
37ca318

Change base_aws.py to support async_conn


syedahsn Mar 6, 2023
e60898b

Add deferrable mode to redshift delete cluster


syedahsn Mar 22, 2023
cc1c7dd

Rebase on main


syedahsn Apr 25, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


30 changes: 28 additions & 2 deletions
airflow/providers/amazon/aws/operators/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftCreateClusterTrigger,
 RedshiftDeleteClusterTrigger,
)

if TYPE_CHECKING:
Expand DownExpand Up@@ -595,6 +596,9 @@ class RedshiftDeleteClusterOperator(BaseOperator):
 The default value is ``True``
 :param aws_conn_id: aws connection to use
 :param poll_interval: Time (in seconds) to wait between two consecutive calls to check cluster state
 :param max_attempts: Number of attempts the cluster should be polled to determine the cluster
 was deleted.
 :param deferrable: If True, the operator will run as a deferrable operator.
 """

 template_fields: Sequence[str] = ("cluster_identifier",)
Expand All@@ -609,7 +613,9 @@ def __init__(
 final_cluster_snapshot_identifier: str | None = None,
 wait_for_completion: bool = True,
 aws_conn_id: str = "aws_default",
 poll_interval: float = 30.0,
 poll_interval: int = 30,
 max_attempts: int = 20,
 deferrable: bool = False,
 **kwargs,
 ):
 super().__init__(**kwargs)
Expand All@@ -624,6 +630,9 @@ def __init__(
 self._attempts = 10
 self._attempt_interval = 15
 self.redshift_hook = RedshiftHook(aws_conn_id=aws_conn_id)
 self.deferrable = deferrable
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def execute(self, context: Context):
 while self._attempts >= 1:
Expand All@@ -642,10 +651,27 @@ def execute(self, context: Context):
 time.sleep(self._attempt_interval)
 else:
 raise
 if self.deferrable:
 self.defer(
 trigger=RedshiftDeleteClusterTrigger(
 cluster_identifier=self.cluster_identifier,
 poll_interval=self.poll_interval,
 max_attempts=self.max_attempts,
 aws_conn_id=self.aws_conn_id,
 ),
 method_name="execute_complete",
 )

 if self.wait_for_completion:
 waiter = self.redshift_hook.get_conn().get_waiter("cluster_deleted")
 waiter.wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": 30},
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": self.max_attempts},
 )

 def execute_complete(self, context, event=None):
 if event["status"] != "success":
 raise AirflowException(f"Error deleting cluster: {event}")
 else:
 self.log.info("Cluster deleted successfully")
 return

40 changes: 39 additions & 1 deletion
airflow/providers/amazon/aws/triggers/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,9 @@
from airflow.compat.functools import cached_property
from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook, RedshiftHook
from airflow.triggers.base import BaseTrigger, TriggerEvent
from typing import Any

from airflow.compat.functools import cached_property


class RedshiftClusterTrigger(BaseTrigger):
Expand DownExpand Up@@ -87,7 +90,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]:
 if self.attempts < 1:
 yield TriggerEvent({"status": "error", "message": str(e)})


class RedshiftCreateClusterTrigger(BaseTrigger):
 """
 Trigger for RedshiftCreateClusterOperator.
Expand DownExpand Up@@ -137,3 +139,39 @@ async def run(self):
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster Created"})

class RedshiftDeleteClusterTrigger(BaseTrigger):
 def __init__(
 self,
 cluster_identifier: str,
 poll_interval: int,
 max_attempts: int,
 aws_conn_id: str,
 ):
 self.cluster_identifier = cluster_identifier
 self.poll_interval = poll_interval
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def serialize(self) -> tuple[str, dict[str, Any]]:
 return (
 "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger",
 {
 "cluster_identifier": str(self.cluster_identifier),
 "poll_interval": str(self.poll_interval),
 "max_attempts": str(self.max_attempts),
 "aws_conn_id": str(self.aws_conn_id),
 },
 )

 async def run(self):
 self.redshift_hook = RedshiftHook(aws_conn_id=self.aws_conn_id)
 async with self.redshift_hook.async_conn as client:
 await client.get_waiter("cluster_deleted").wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={
 "Delay": int(self.poll_interval),
 "MaxAttempts": int(self.max_attempts),
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster deleted"})

23 changes: 22 additions & 1 deletion
tests/providers/amazon/aws/operators/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,7 +31,10 @@
 RedshiftPauseClusterOperator,
 RedshiftResumeClusterOperator,
)
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftDeleteClusterTrigger,
)


class TestRedshiftCreateClusterOperator:
Expand DownExpand Up@@ -481,3 +484,21 @@ def test_delete_cluster_multiple_attempts_fail(self, _, mock_conn, mock_delete_c
 redshift_operator.execute(None)

 assert mock_delete_cluster.call_count == 10

 @mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.delete_cluster")
 def test_delete_cluster_deferrable(self, mock_delete_cluster):
 mock_delete_cluster.return_value = True

 redshift_operator = RedshiftDeleteClusterOperator(
 task_id="task_test",
 cluster_identifier="test_cluster",
 aws_conn_id="aws_conn_test",
 deferrable=True,
 )

 with pytest.raises(TaskDeferred) as exc:
 redshift_operator.execute(None)

 assert isinstance(
 exc.value.trigger, RedshiftDeleteClusterTrigger
 ), "Trigger is not a RedshiftDeleteClusterTrigger"

39 changes: 38 additions & 1 deletion
tests/providers/amazon/aws/triggers/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@

import pytest

from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger, RedshiftDeleteClusterTrigger
from airflow.triggers.base import TriggerEvent

if sys.version_info < (3, 8):
Expand DownExpand Up@@ -72,3 +72,40 @@ async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster Created"})

class TestRedshiftDeleteClusterTrigger:
 def test_redshift_create_cluster_trigger_serialize(self):
 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )
 class_path, args = redshift_delete_cluster_trigger.serialize()
 assert (
 class_path
 == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger"
 )
 assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER
 assert args["poll_interval"] == str(TEST_POLL_INTERVAL)
 assert args["max_attempts"] == str(TEST_MAX_ATTEMPT)
 assert args["aws_conn_id"] == TEST_AWS_CONN_ID

 @pytest.mark.asyncio
 @async_mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.async_conn")
 async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 mock = async_mock.MagicMock()
 mock_async_conn.__aenter__.return_value = mock
 mock.get_waiter().wait = AsyncMock()

 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )

 generator = redshift_delete_cluster_trigger.run()
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster deleted"})

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Add Deferrable Mode to RedshiftDeleteClusterOperator#30870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

syedahsn
wants to merge
3
commits into

from

 Closed


Add Deferrable Mode to RedshiftDeleteClusterOperator
#30870
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
37ca318

Change base_aws.py to support async_conn


syedahsn Mar 6, 2023
e60898b

Add deferrable mode to redshift delete cluster


syedahsn Mar 22, 2023
cc1c7dd

Rebase on main


syedahsn Apr 25, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


30 changes: 28 additions & 2 deletions
airflow/providers/amazon/aws/operators/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftCreateClusterTrigger,
 RedshiftDeleteClusterTrigger,
)

if TYPE_CHECKING:
Expand DownExpand Up@@ -595,6 +596,9 @@ class RedshiftDeleteClusterOperator(BaseOperator):
 The default value is ``True``
 :param aws_conn_id: aws connection to use
 :param poll_interval: Time (in seconds) to wait between two consecutive calls to check cluster state
 :param max_attempts: Number of attempts the cluster should be polled to determine the cluster
 was deleted.
 :param deferrable: If True, the operator will run as a deferrable operator.
 """

 template_fields: Sequence[str] = ("cluster_identifier",)
Expand All@@ -609,7 +613,9 @@ def __init__(
 final_cluster_snapshot_identifier: str | None = None,
 wait_for_completion: bool = True,
 aws_conn_id: str = "aws_default",
 poll_interval: float = 30.0,
 poll_interval: int = 30,
 max_attempts: int = 20,
 deferrable: bool = False,
 **kwargs,
 ):
 super().__init__(**kwargs)
Expand All@@ -624,6 +630,9 @@ def __init__(
 self._attempts = 10
 self._attempt_interval = 15
 self.redshift_hook = RedshiftHook(aws_conn_id=aws_conn_id)
 self.deferrable = deferrable
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def execute(self, context: Context):
 while self._attempts >= 1:
Expand All@@ -642,10 +651,27 @@ def execute(self, context: Context):
 time.sleep(self._attempt_interval)
 else:
 raise
 if self.deferrable:
 self.defer(
 trigger=RedshiftDeleteClusterTrigger(
 cluster_identifier=self.cluster_identifier,
 poll_interval=self.poll_interval,
 max_attempts=self.max_attempts,
 aws_conn_id=self.aws_conn_id,
 ),
 method_name="execute_complete",
 )

 if self.wait_for_completion:
 waiter = self.redshift_hook.get_conn().get_waiter("cluster_deleted")
 waiter.wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": 30},
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": self.max_attempts},
 )

 def execute_complete(self, context, event=None):
 if event["status"] != "success":
 raise AirflowException(f"Error deleting cluster: {event}")
 else:
 self.log.info("Cluster deleted successfully")
 return

40 changes: 39 additions & 1 deletion
airflow/providers/amazon/aws/triggers/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,9 @@
from airflow.compat.functools import cached_property
from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook, RedshiftHook
from airflow.triggers.base import BaseTrigger, TriggerEvent
from typing import Any

from airflow.compat.functools import cached_property


class RedshiftClusterTrigger(BaseTrigger):
Expand DownExpand Up@@ -87,7 +90,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]:
 if self.attempts < 1:
 yield TriggerEvent({"status": "error", "message": str(e)})


class RedshiftCreateClusterTrigger(BaseTrigger):
 """
 Trigger for RedshiftCreateClusterOperator.
Expand DownExpand Up@@ -137,3 +139,39 @@ async def run(self):
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster Created"})

class RedshiftDeleteClusterTrigger(BaseTrigger):
 def __init__(
 self,
 cluster_identifier: str,
 poll_interval: int,
 max_attempts: int,
 aws_conn_id: str,
 ):
 self.cluster_identifier = cluster_identifier
 self.poll_interval = poll_interval
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def serialize(self) -> tuple[str, dict[str, Any]]:
 return (
 "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger",
 {
 "cluster_identifier": str(self.cluster_identifier),
 "poll_interval": str(self.poll_interval),
 "max_attempts": str(self.max_attempts),
 "aws_conn_id": str(self.aws_conn_id),
 },
 )

 async def run(self):
 self.redshift_hook = RedshiftHook(aws_conn_id=self.aws_conn_id)
 async with self.redshift_hook.async_conn as client:
 await client.get_waiter("cluster_deleted").wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={
 "Delay": int(self.poll_interval),
 "MaxAttempts": int(self.max_attempts),
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster deleted"})

23 changes: 22 additions & 1 deletion
tests/providers/amazon/aws/operators/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,7 +31,10 @@
 RedshiftPauseClusterOperator,
 RedshiftResumeClusterOperator,
)
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftDeleteClusterTrigger,
)


class TestRedshiftCreateClusterOperator:
Expand DownExpand Up@@ -481,3 +484,21 @@ def test_delete_cluster_multiple_attempts_fail(self, _, mock_conn, mock_delete_c
 redshift_operator.execute(None)

 assert mock_delete_cluster.call_count == 10

 @mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.delete_cluster")
 def test_delete_cluster_deferrable(self, mock_delete_cluster):
 mock_delete_cluster.return_value = True

 redshift_operator = RedshiftDeleteClusterOperator(
 task_id="task_test",
 cluster_identifier="test_cluster",
 aws_conn_id="aws_conn_test",
 deferrable=True,
 )

 with pytest.raises(TaskDeferred) as exc:
 redshift_operator.execute(None)

 assert isinstance(
 exc.value.trigger, RedshiftDeleteClusterTrigger
 ), "Trigger is not a RedshiftDeleteClusterTrigger"

39 changes: 38 additions & 1 deletion
tests/providers/amazon/aws/triggers/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@

import pytest

from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger, RedshiftDeleteClusterTrigger
from airflow.triggers.base import TriggerEvent

if sys.version_info < (3, 8):
Expand DownExpand Up@@ -72,3 +72,40 @@ async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster Created"})

class TestRedshiftDeleteClusterTrigger:
 def test_redshift_create_cluster_trigger_serialize(self):
 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )
 class_path, args = redshift_delete_cluster_trigger.serialize()
 assert (
 class_path
 == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger"
 )
 assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER
 assert args["poll_interval"] == str(TEST_POLL_INTERVAL)
 assert args["max_attempts"] == str(TEST_MAX_ATTEMPT)
 assert args["aws_conn_id"] == TEST_AWS_CONN_ID

 @pytest.mark.asyncio
 @async_mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.async_conn")
 async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 mock = async_mock.MagicMock()
 mock_async_conn.__aenter__.return_value = mock
 mock.get_waiter().wait = AsyncMock()

 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )

 generator = redshift_delete_cluster_trigger.run()
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster deleted"})

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add Deferrable Mode to RedshiftDeleteClusterOperator#30870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

syedahsn
wants to merge
3
commits into

from

 Closed


Add Deferrable Mode to RedshiftDeleteClusterOperator
#30870
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
37ca318

Change base_aws.py to support async_conn


syedahsn Mar 6, 2023
e60898b

Add deferrable mode to redshift delete cluster


syedahsn Mar 22, 2023
cc1c7dd

Rebase on main


syedahsn Apr 25, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


30 changes: 28 additions & 2 deletions
airflow/providers/amazon/aws/operators/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftCreateClusterTrigger,
 RedshiftDeleteClusterTrigger,
)

if TYPE_CHECKING:
Expand DownExpand Up@@ -595,6 +596,9 @@ class RedshiftDeleteClusterOperator(BaseOperator):
 The default value is ``True``
 :param aws_conn_id: aws connection to use
 :param poll_interval: Time (in seconds) to wait between two consecutive calls to check cluster state
 :param max_attempts: Number of attempts the cluster should be polled to determine the cluster
 was deleted.
 :param deferrable: If True, the operator will run as a deferrable operator.
 """

 template_fields: Sequence[str] = ("cluster_identifier",)
Expand All@@ -609,7 +613,9 @@ def __init__(
 final_cluster_snapshot_identifier: str | None = None,
 wait_for_completion: bool = True,
 aws_conn_id: str = "aws_default",
 poll_interval: float = 30.0,
 poll_interval: int = 30,
 max_attempts: int = 20,
 deferrable: bool = False,
 **kwargs,
 ):
 super().__init__(**kwargs)
Expand All@@ -624,6 +630,9 @@ def __init__(
 self._attempts = 10
 self._attempt_interval = 15
 self.redshift_hook = RedshiftHook(aws_conn_id=aws_conn_id)
 self.deferrable = deferrable
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def execute(self, context: Context):
 while self._attempts >= 1:
Expand All@@ -642,10 +651,27 @@ def execute(self, context: Context):
 time.sleep(self._attempt_interval)
 else:
 raise
 if self.deferrable:
 self.defer(
 trigger=RedshiftDeleteClusterTrigger(
 cluster_identifier=self.cluster_identifier,
 poll_interval=self.poll_interval,
 max_attempts=self.max_attempts,
 aws_conn_id=self.aws_conn_id,
 ),
 method_name="execute_complete",
 )

 if self.wait_for_completion:
 waiter = self.redshift_hook.get_conn().get_waiter("cluster_deleted")
 waiter.wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": 30},
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": self.max_attempts},
 )

 def execute_complete(self, context, event=None):
 if event["status"] != "success":
 raise AirflowException(f"Error deleting cluster: {event}")
 else:
 self.log.info("Cluster deleted successfully")
 return

40 changes: 39 additions & 1 deletion
airflow/providers/amazon/aws/triggers/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,9 @@
from airflow.compat.functools import cached_property
from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook, RedshiftHook
from airflow.triggers.base import BaseTrigger, TriggerEvent
from typing import Any

from airflow.compat.functools import cached_property


class RedshiftClusterTrigger(BaseTrigger):
Expand DownExpand Up@@ -87,7 +90,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]:
 if self.attempts < 1:
 yield TriggerEvent({"status": "error", "message": str(e)})


class RedshiftCreateClusterTrigger(BaseTrigger):
 """
 Trigger for RedshiftCreateClusterOperator.
Expand DownExpand Up@@ -137,3 +139,39 @@ async def run(self):
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster Created"})

class RedshiftDeleteClusterTrigger(BaseTrigger):
 def __init__(
 self,
 cluster_identifier: str,
 poll_interval: int,
 max_attempts: int,
 aws_conn_id: str,
 ):
 self.cluster_identifier = cluster_identifier
 self.poll_interval = poll_interval
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def serialize(self) -> tuple[str, dict[str, Any]]:
 return (
 "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger",
 {
 "cluster_identifier": str(self.cluster_identifier),
 "poll_interval": str(self.poll_interval),
 "max_attempts": str(self.max_attempts),
 "aws_conn_id": str(self.aws_conn_id),
 },
 )

 async def run(self):
 self.redshift_hook = RedshiftHook(aws_conn_id=self.aws_conn_id)
 async with self.redshift_hook.async_conn as client:
 await client.get_waiter("cluster_deleted").wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={
 "Delay": int(self.poll_interval),
 "MaxAttempts": int(self.max_attempts),
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster deleted"})

23 changes: 22 additions & 1 deletion
tests/providers/amazon/aws/operators/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,7 +31,10 @@
 RedshiftPauseClusterOperator,
 RedshiftResumeClusterOperator,
)
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftDeleteClusterTrigger,
)


class TestRedshiftCreateClusterOperator:
Expand DownExpand Up@@ -481,3 +484,21 @@ def test_delete_cluster_multiple_attempts_fail(self, _, mock_conn, mock_delete_c
 redshift_operator.execute(None)

 assert mock_delete_cluster.call_count == 10

 @mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.delete_cluster")
 def test_delete_cluster_deferrable(self, mock_delete_cluster):
 mock_delete_cluster.return_value = True

 redshift_operator = RedshiftDeleteClusterOperator(
 task_id="task_test",
 cluster_identifier="test_cluster",
 aws_conn_id="aws_conn_test",
 deferrable=True,
 )

 with pytest.raises(TaskDeferred) as exc:
 redshift_operator.execute(None)

 assert isinstance(
 exc.value.trigger, RedshiftDeleteClusterTrigger
 ), "Trigger is not a RedshiftDeleteClusterTrigger"

39 changes: 38 additions & 1 deletion
tests/providers/amazon/aws/triggers/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@

import pytest

from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger, RedshiftDeleteClusterTrigger
from airflow.triggers.base import TriggerEvent

if sys.version_info < (3, 8):
Expand DownExpand Up@@ -72,3 +72,40 @@ async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster Created"})

class TestRedshiftDeleteClusterTrigger:
 def test_redshift_create_cluster_trigger_serialize(self):
 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )
 class_path, args = redshift_delete_cluster_trigger.serialize()
 assert (
 class_path
 == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger"
 )
 assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER
 assert args["poll_interval"] == str(TEST_POLL_INTERVAL)
 assert args["max_attempts"] == str(TEST_MAX_ATTEMPT)
 assert args["aws_conn_id"] == TEST_AWS_CONN_ID

 @pytest.mark.asyncio
 @async_mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.async_conn")
 async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 mock = async_mock.MagicMock()
 mock_async_conn.__aenter__.return_value = mock
 mock.get_waiter().wait = AsyncMock()

 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )

 generator = redshift_delete_cluster_trigger.run()
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster deleted"})

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add Deferrable Mode to RedshiftDeleteClusterOperator#30870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

syedahsn
wants to merge
3
commits into

from

 Closed


Add Deferrable Mode to RedshiftDeleteClusterOperator
#30870
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
37ca318

Change base_aws.py to support async_conn


syedahsn Mar 6, 2023
e60898b

Add deferrable mode to redshift delete cluster


syedahsn Mar 22, 2023
cc1c7dd

Rebase on main


syedahsn Apr 25, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


30 changes: 28 additions & 2 deletions
airflow/providers/amazon/aws/operators/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftCreateClusterTrigger,
 RedshiftDeleteClusterTrigger,
)

if TYPE_CHECKING:
Expand DownExpand Up@@ -595,6 +596,9 @@ class RedshiftDeleteClusterOperator(BaseOperator):
 The default value is ``True``
 :param aws_conn_id: aws connection to use
 :param poll_interval: Time (in seconds) to wait between two consecutive calls to check cluster state
 :param max_attempts: Number of attempts the cluster should be polled to determine the cluster
 was deleted.
 :param deferrable: If True, the operator will run as a deferrable operator.
 """

 template_fields: Sequence[str] = ("cluster_identifier",)
Expand All@@ -609,7 +613,9 @@ def __init__(
 final_cluster_snapshot_identifier: str | None = None,
 wait_for_completion: bool = True,
 aws_conn_id: str = "aws_default",
 poll_interval: float = 30.0,
 poll_interval: int = 30,
 max_attempts: int = 20,
 deferrable: bool = False,
 **kwargs,
 ):
 super().__init__(**kwargs)
Expand All@@ -624,6 +630,9 @@ def __init__(
 self._attempts = 10
 self._attempt_interval = 15
 self.redshift_hook = RedshiftHook(aws_conn_id=aws_conn_id)
 self.deferrable = deferrable
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def execute(self, context: Context):
 while self._attempts >= 1:
Expand All@@ -642,10 +651,27 @@ def execute(self, context: Context):
 time.sleep(self._attempt_interval)
 else:
 raise
 if self.deferrable:
 self.defer(
 trigger=RedshiftDeleteClusterTrigger(
 cluster_identifier=self.cluster_identifier,
 poll_interval=self.poll_interval,
 max_attempts=self.max_attempts,
 aws_conn_id=self.aws_conn_id,
 ),
 method_name="execute_complete",
 )

 if self.wait_for_completion:
 waiter = self.redshift_hook.get_conn().get_waiter("cluster_deleted")
 waiter.wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": 30},
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": self.max_attempts},
 )

 def execute_complete(self, context, event=None):
 if event["status"] != "success":
 raise AirflowException(f"Error deleting cluster: {event}")
 else:
 self.log.info("Cluster deleted successfully")
 return

40 changes: 39 additions & 1 deletion
airflow/providers/amazon/aws/triggers/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,9 @@
from airflow.compat.functools import cached_property
from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook, RedshiftHook
from airflow.triggers.base import BaseTrigger, TriggerEvent
from typing import Any

from airflow.compat.functools import cached_property


class RedshiftClusterTrigger(BaseTrigger):
Expand DownExpand Up@@ -87,7 +90,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]:
 if self.attempts < 1:
 yield TriggerEvent({"status": "error", "message": str(e)})


class RedshiftCreateClusterTrigger(BaseTrigger):
 """
 Trigger for RedshiftCreateClusterOperator.
Expand DownExpand Up@@ -137,3 +139,39 @@ async def run(self):
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster Created"})

class RedshiftDeleteClusterTrigger(BaseTrigger):
 def __init__(
 self,
 cluster_identifier: str,
 poll_interval: int,
 max_attempts: int,
 aws_conn_id: str,
 ):
 self.cluster_identifier = cluster_identifier
 self.poll_interval = poll_interval
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def serialize(self) -> tuple[str, dict[str, Any]]:
 return (
 "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger",
 {
 "cluster_identifier": str(self.cluster_identifier),
 "poll_interval": str(self.poll_interval),
 "max_attempts": str(self.max_attempts),
 "aws_conn_id": str(self.aws_conn_id),
 },
 )

 async def run(self):
 self.redshift_hook = RedshiftHook(aws_conn_id=self.aws_conn_id)
 async with self.redshift_hook.async_conn as client:
 await client.get_waiter("cluster_deleted").wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={
 "Delay": int(self.poll_interval),
 "MaxAttempts": int(self.max_attempts),
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster deleted"})

23 changes: 22 additions & 1 deletion
tests/providers/amazon/aws/operators/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,7 +31,10 @@
 RedshiftPauseClusterOperator,
 RedshiftResumeClusterOperator,
)
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftDeleteClusterTrigger,
)


class TestRedshiftCreateClusterOperator:
Expand DownExpand Up@@ -481,3 +484,21 @@ def test_delete_cluster_multiple_attempts_fail(self, _, mock_conn, mock_delete_c
 redshift_operator.execute(None)

 assert mock_delete_cluster.call_count == 10

 @mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.delete_cluster")
 def test_delete_cluster_deferrable(self, mock_delete_cluster):
 mock_delete_cluster.return_value = True

 redshift_operator = RedshiftDeleteClusterOperator(
 task_id="task_test",
 cluster_identifier="test_cluster",
 aws_conn_id="aws_conn_test",
 deferrable=True,
 )

 with pytest.raises(TaskDeferred) as exc:
 redshift_operator.execute(None)

 assert isinstance(
 exc.value.trigger, RedshiftDeleteClusterTrigger
 ), "Trigger is not a RedshiftDeleteClusterTrigger"

39 changes: 38 additions & 1 deletion
tests/providers/amazon/aws/triggers/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@

import pytest

from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger, RedshiftDeleteClusterTrigger
from airflow.triggers.base import TriggerEvent

if sys.version_info < (3, 8):
Expand DownExpand Up@@ -72,3 +72,40 @@ async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster Created"})

class TestRedshiftDeleteClusterTrigger:
 def test_redshift_create_cluster_trigger_serialize(self):
 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )
 class_path, args = redshift_delete_cluster_trigger.serialize()
 assert (
 class_path
 == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger"
 )
 assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER
 assert args["poll_interval"] == str(TEST_POLL_INTERVAL)
 assert args["max_attempts"] == str(TEST_MAX_ATTEMPT)
 assert args["aws_conn_id"] == TEST_AWS_CONN_ID

 @pytest.mark.asyncio
 @async_mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.async_conn")
 async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 mock = async_mock.MagicMock()
 mock_async_conn.__aenter__.return_value = mock
 mock.get_waiter().wait = AsyncMock()

 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )

 generator = redshift_delete_cluster_trigger.run()
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster deleted"})

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Add Deferrable Mode to RedshiftDeleteClusterOperator#30870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

syedahsn
wants to merge
3
commits into

from

 Closed


Add Deferrable Mode to RedshiftDeleteClusterOperator
#30870
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
37ca318

Change base_aws.py to support async_conn


syedahsn Mar 6, 2023
e60898b

Add deferrable mode to redshift delete cluster


syedahsn Mar 22, 2023
cc1c7dd

Rebase on main


syedahsn Apr 25, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


30 changes: 28 additions & 2 deletions
airflow/providers/amazon/aws/operators/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftCreateClusterTrigger,
 RedshiftDeleteClusterTrigger,
)

if TYPE_CHECKING:
Expand DownExpand Up@@ -595,6 +596,9 @@ class RedshiftDeleteClusterOperator(BaseOperator):
 The default value is ``True``
 :param aws_conn_id: aws connection to use
 :param poll_interval: Time (in seconds) to wait between two consecutive calls to check cluster state
 :param max_attempts: Number of attempts the cluster should be polled to determine the cluster
 was deleted.
 :param deferrable: If True, the operator will run as a deferrable operator.
 """

 template_fields: Sequence[str] = ("cluster_identifier",)
Expand All@@ -609,7 +613,9 @@ def __init__(
 final_cluster_snapshot_identifier: str | None = None,
 wait_for_completion: bool = True,
 aws_conn_id: str = "aws_default",
 poll_interval: float = 30.0,
 poll_interval: int = 30,
 max_attempts: int = 20,
 deferrable: bool = False,
 **kwargs,
 ):
 super().__init__(**kwargs)
Expand All@@ -624,6 +630,9 @@ def __init__(
 self._attempts = 10
 self._attempt_interval = 15
 self.redshift_hook = RedshiftHook(aws_conn_id=aws_conn_id)
 self.deferrable = deferrable
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def execute(self, context: Context):
 while self._attempts >= 1:
Expand All@@ -642,10 +651,27 @@ def execute(self, context: Context):
 time.sleep(self._attempt_interval)
 else:
 raise
 if self.deferrable:
 self.defer(
 trigger=RedshiftDeleteClusterTrigger(
 cluster_identifier=self.cluster_identifier,
 poll_interval=self.poll_interval,
 max_attempts=self.max_attempts,
 aws_conn_id=self.aws_conn_id,
 ),
 method_name="execute_complete",
 )

 if self.wait_for_completion:
 waiter = self.redshift_hook.get_conn().get_waiter("cluster_deleted")
 waiter.wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": 30},
 WaiterConfig={"Delay": self.poll_interval, "MaxAttempts": self.max_attempts},
 )

 def execute_complete(self, context, event=None):
 if event["status"] != "success":
 raise AirflowException(f"Error deleting cluster: {event}")
 else:
 self.log.info("Cluster deleted successfully")
 return

40 changes: 39 additions & 1 deletion
airflow/providers/amazon/aws/triggers/redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,9 @@
from airflow.compat.functools import cached_property
from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook, RedshiftHook
from airflow.triggers.base import BaseTrigger, TriggerEvent
from typing import Any

from airflow.compat.functools import cached_property


class RedshiftClusterTrigger(BaseTrigger):
Expand DownExpand Up@@ -87,7 +90,6 @@ async def run(self) -> AsyncIterator[TriggerEvent]:
 if self.attempts < 1:
 yield TriggerEvent({"status": "error", "message": str(e)})


class RedshiftCreateClusterTrigger(BaseTrigger):
 """
 Trigger for RedshiftCreateClusterOperator.
Expand DownExpand Up@@ -137,3 +139,39 @@ async def run(self):
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster Created"})

class RedshiftDeleteClusterTrigger(BaseTrigger):
 def __init__(
 self,
 cluster_identifier: str,
 poll_interval: int,
 max_attempts: int,
 aws_conn_id: str,
 ):
 self.cluster_identifier = cluster_identifier
 self.poll_interval = poll_interval
 self.max_attempts = max_attempts
 self.aws_conn_id = aws_conn_id

 def serialize(self) -> tuple[str, dict[str, Any]]:
 return (
 "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger",
 {
 "cluster_identifier": str(self.cluster_identifier),
 "poll_interval": str(self.poll_interval),
 "max_attempts": str(self.max_attempts),
 "aws_conn_id": str(self.aws_conn_id),
 },
 )

 async def run(self):
 self.redshift_hook = RedshiftHook(aws_conn_id=self.aws_conn_id)
 async with self.redshift_hook.async_conn as client:
 await client.get_waiter("cluster_deleted").wait(
 ClusterIdentifier=self.cluster_identifier,
 WaiterConfig={
 "Delay": int(self.poll_interval),
 "MaxAttempts": int(self.max_attempts),
 },
 )
 yield TriggerEvent({"status": "success", "message": "Cluster deleted"})

23 changes: 22 additions & 1 deletion
tests/providers/amazon/aws/operators/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -31,7 +31,10 @@
 RedshiftPauseClusterOperator,
 RedshiftResumeClusterOperator,
)
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import (
 RedshiftClusterTrigger,
 RedshiftDeleteClusterTrigger,
)


class TestRedshiftCreateClusterOperator:
Expand DownExpand Up@@ -481,3 +484,21 @@ def test_delete_cluster_multiple_attempts_fail(self, _, mock_conn, mock_delete_c
 redshift_operator.execute(None)

 assert mock_delete_cluster.call_count == 10

 @mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.delete_cluster")
 def test_delete_cluster_deferrable(self, mock_delete_cluster):
 mock_delete_cluster.return_value = True

 redshift_operator = RedshiftDeleteClusterOperator(
 task_id="task_test",
 cluster_identifier="test_cluster",
 aws_conn_id="aws_conn_test",
 deferrable=True,
 )

 with pytest.raises(TaskDeferred) as exc:
 redshift_operator.execute(None)

 assert isinstance(
 exc.value.trigger, RedshiftDeleteClusterTrigger
 ), "Trigger is not a RedshiftDeleteClusterTrigger"

39 changes: 38 additions & 1 deletion
tests/providers/amazon/aws/triggers/test_redshift_cluster.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,7 +20,7 @@

import pytest

from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger
from airflow.providers.amazon.aws.triggers.redshift_cluster import RedshiftCreateClusterTrigger, RedshiftDeleteClusterTrigger
from airflow.triggers.base import TriggerEvent

if sys.version_info < (3, 8):
Expand DownExpand Up@@ -72,3 +72,40 @@ async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster Created"})

class TestRedshiftDeleteClusterTrigger:
 def test_redshift_create_cluster_trigger_serialize(self):
 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )
 class_path, args = redshift_delete_cluster_trigger.serialize()
 assert (
 class_path
 == "airflow.providers.amazon.aws.triggers.redshift_cluster.RedshiftDeleteClusterTrigger"
 )
 assert args["cluster_identifier"] == TEST_CLUSTER_IDENTIFIER
 assert args["poll_interval"] == str(TEST_POLL_INTERVAL)
 assert args["max_attempts"] == str(TEST_MAX_ATTEMPT)
 assert args["aws_conn_id"] == TEST_AWS_CONN_ID

 @pytest.mark.asyncio
 @async_mock.patch("airflow.providers.amazon.aws.hooks.redshift_cluster.RedshiftHook.async_conn")
 async def test_redshift_create_cluster_trigger_run(self, mock_async_conn):
 mock = async_mock.MagicMock()
 mock_async_conn.__aenter__.return_value = mock
 mock.get_waiter().wait = AsyncMock()

 redshift_delete_cluster_trigger = RedshiftDeleteClusterTrigger(
 cluster_identifier=TEST_CLUSTER_IDENTIFIER,
 poll_interval=TEST_POLL_INTERVAL,
 max_attempts=TEST_MAX_ATTEMPT,
 aws_conn_id=TEST_AWS_CONN_ID,
 )

 generator = redshift_delete_cluster_trigger.run()
 response = await generator.asend(None)

 assert response == TriggerEvent({"status": "success", "message": "Cluster deleted"})