') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Add deferrable mode in Redshift delete cluster by pankajastro · Pull Request #30244 · apache/airflow · GitHub
Skip to content

Add deferrable mode in Redshift delete cluster - #30244

Merged
potiuk merged 1 commit into
apache:mainfrom
astronomer:async_redshift_delete_cluster
Jun 4, 2023
Merged

Add deferrable mode in Redshift delete cluster#30244
potiuk merged 1 commit into
apache:mainfrom
astronomer:async_redshift_delete_cluster

Conversation

@pankajastro

@pankajastropankajastro commented Mar 22, 2023•
edited
Loading

Copy link
Copy Markdown
Member

Add the deferrable param in RedshiftDeleteClusterOperator.
This will allow running RedshiftDeleteClusterOperator in an async way
that means we only submit a job from the worker to delete a redshift cluster
then defer to the trigger for the polling and waiter for a cluster to get removed
and the worker slot won't be occupied for the whole period of
task execution.

^ Add meaningful description above

Read the Pull Request Guidelines for more information.
In case of fundamental code changes, an Airflow Improvement Proposal (AIP) is needed.
In case of a new dependency, check compliance with the ASF 3rd Party License Policy.
In case of backwards incompatible changes please leave a note in a newsfragment file, named {pr_number}.significant.rst or {issue_number}.significant.rst, in newsfragments.

@pankajastro
pankajastroforce-pushed the async_redshift_delete_cluster branch 5 times, most recently from 1d4814f to 0a97c7bCompareMarch 23, 2023 07:31
@pankajastro
pankajastro marked this pull request as ready for review March 23, 2023 09:08
phanikumv
phanikumv reviewed Mar 23, 2023
View reviewed changes
Comment threaddocs/apache-airflow-providers-amazon/operators/redshift_cluster.rst Outdated
syedahsn
syedahsn reviewed Mar 25, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/operators/redshift_cluster.py Outdated
syedahsn
syedahsn reviewed Mar 25, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/triggers/redshift_cluster.py Outdated
@pankajastro
pankajastroforce-pushed the async_redshift_delete_cluster branch from 1e6937f to f2e353cCompareMarch 30, 2023 18:14
@pankajastro
pankajastroforce-pushed the async_redshift_delete_cluster branch 5 times, most recently from 000f937 to b648783CompareApril 11, 2023 15:32
syedahsn
syedahsn reviewed Apr 11, 2023
View reviewed changes

@syedahsnsyedahsn left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a few more comments. A more general comment on the PR is that the way it is written means that we are forced to duplicate logic in operators to Triggers if we want to have the same functionality between deferrable and non deferrable modes. This is seen in the retry logic that's implemented in the delete_cluster method in the operator, which is now being copied over to the Trigger. I think this approach will lead to more cpmplicated, error-prone code because any change made to one will have to be copied to the other. I'm currently in the middle of writing a PR for supporting deferrable operators as well, and I took a slightly different approach. I'd love to hear what your thoughts are on how I've gone about it.

Comment threadairflow/providers/amazon/aws/hooks/redshift_cluster.py Outdated
Comment threadairflow/providers/amazon/aws/triggers/redshift_cluster.py Outdated
@pankajastro
pankajastroforce-pushed the async_redshift_delete_cluster branch 2 times, most recently from 6d6c376 to 04f8c40CompareApril 26, 2023 09:47
@pankajastropankajastro mentioned this pull request Apr 26, 2023
Closed
@pankajastro

pankajastro commented Apr 26, 2023

Copy link
Copy Markdown
MemberAuthor

Added a few more comments. A more general comment on the PR is that the way it is written means that we are forced to duplicate logic in operators to Triggers if we want to have the same functionality between deferrable and non deferrable modes. This is seen in the retry logic that's implemented in the delete_cluster method in the operator, which is now being copied over to the Trigger. I think this approach will lead to more cpmplicated, error-prone code because any change made to one will have to be copied to the other. I'm currently in the middle of writing a PR for supporting deferrable operators as well, and I took a slightly different approach. I'd love to hear what your thoughts are on how I've gone about it.

Rebased on top of your PR but I see you have also created a PR for the same #30870

syedahsn
syedahsn reviewed Apr 26, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/operators/redshift_cluster.py Outdated
syedahsn
syedahsn reviewed Apr 26, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/triggers/redshift_cluster.py Outdated
@pankajastro
pankajastroforce-pushed the async_redshift_delete_cluster branch from a88789d to a20eae1CompareMay 25, 2023 10:30
@pankajastro
pankajastro marked this pull request as ready for review May 25, 2023 14:00
@pankajastro
pankajastroforce-pushed the async_redshift_delete_cluster branch 3 times, most recently from 41e4bb4 to f19ab65CompareMay 26, 2023 09:39
pankajastro
pankajastro commented May 29, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/triggers/redshift_cluster.py Outdated
@pankajastro
pankajastroforce-pushed the async_redshift_delete_cluster branch 2 times, most recently from 8b91cf1 to e5c4c61CompareMay 29, 2023 18:09
syedahsn
syedahsn reviewed May 29, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/operators/redshift_cluster.py Outdated
syedahsn
syedahsn reviewed May 29, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/triggers/redshift_cluster.py Outdated
syedahsn
syedahsn reviewed May 29, 2023
View reviewed changes
Comment threadtests/providers/amazon/aws/triggers/test_redshift_cluster.py Outdated
syedahsn
syedahsn suggested changes May 29, 2023
View reviewed changes

@syedahsnsyedahsn left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks good, just a few nits, and some more test cases.

@pankajastro
pankajastro requested a review from syedahsnMay 30, 2023 06:36
@pankajastro
pankajastroforce-pushed the async_redshift_delete_cluster branch from dc26d94 to b8cb678CompareMay 30, 2023 07:04
hussein-awala
hussein-awala reviewed May 30, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/operators/redshift_cluster.py Outdated
Comment threadairflow/providers/amazon/aws/triggers/redshift_cluster.py Outdated
Comment on lines 396 to 398

@hussein-awalahussein-awalaMay 30, 2023

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just out of curiosity, why are you using RedshiftHook instead of RedshiftAsyncHook? (I see it used in 3 of the other 4 triggers)

@pankajastropankajastroMay 30, 2023•
edited
Loading

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In main we have RedshiftAsyncHook in one trigger RedshiftClusterTrigger which I'll clean once this PR will be merged
https://github.com/apache/airflow/blob/main/airflow/providers/amazon/aws/triggers/redshift_cluster.py can you please point me to which trigger you are looking at? Also, we decided to go with boto waiter approach and not the async hook for aws async operator #30032

@pankajastropankajastroMay 30, 2023

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll remove it in #31610 PR

pankajastro
pankajastro commented May 30, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/operators/redshift_cluster.py Outdated
@pankajastro

pankajastro commented May 31, 2023

Copy link
Copy Markdown
MemberAuthor

Hi @hussein-awala / @eladkal requesting a re-review on this. thank you!

@hussein-awalahussein-awala mentioned this pull request May 31, 2023
Closed
2 tasks
@pankajastro
Add deferrable mode in Redshift delete cluster
3598464
Add the deferrable param in RedshiftDeleteClusterOperator.
This will allow running RedshiftDeleteClusterOperator in an async way
that means we only submit a job from the worker to delete a redshift cluster
then defer to the trigger for the polling and waiter for a cluster to get removed
and the worker slot won't be occupied for the whole period of
task execution.
@pankajastro
pankajastroforce-pushed the async_redshift_delete_cluster branch from a1a1b43 to 3598464CompareJune 4, 2023 17:52
hussein-awala
hussein-awala reviewed Jun 4, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/triggers/redshift_cluster.py
@hussein-awala
hussein-awala requested a review from ashbJune 4, 2023 20:13
@potiuk
potiuk merged commit a247a8f into apache:mainJun 4, 2023
pankajastro added a commit to astronomer/airflow that referenced this pull request Jun 5, 2023
@pankajastro
Fix static check
501a424
for some reason CI was green in apache#30244 despite some issue.
Most probably because of wrong rebase. This PR remove unimported value to make to consistent with other
and fix the main.
@pankajastropankajastro mentioned this pull request Jun 5, 2023
Closed
@pankajastro
pankajastro deleted the async_redshift_delete_cluster branch June 5, 2023 07:28
@eladkaleladkal mentioned this pull request Jun 20, 2023
Closed
86 tasks
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@phanikumvphanikumvphanikumv approved these changes
@potiukpotiukpotiuk approved these changes
@hussein-awalahussein-awalahussein-awala approved these changes
@eladkaleladkalAwaiting requested review from eladkal
@o-nikolaso-nikolasAwaiting requested review from o-nikolas
@syedahsnsyedahsnAwaiting requested review from syedahsn
@ashbashbAwaiting requested review from ashb

Assignees

No one assigned

Labels

area:providerskind:documentationprovider:amazonAWS/Amazon - related issues

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

6 participants

@pankajastro@ashb@potiuk@hussein-awala@phanikumv@syedahsn