') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Change Deferrable implementation for RedshiftResumeClusterOperator to follow standard by syedahsn · Pull Request #30864 · apache/airflow · GitHub
Skip to content

Change Deferrable implementation for RedshiftResumeClusterOperator to follow standard - #30864

Merged
o-nikolas merged 8 commits into
apache:mainfrom
aws-mwaa:syedahsn/deferrable-redshift-resume-cluster
May 24, 2023
Merged

Change Deferrable implementation for RedshiftResumeClusterOperator to follow standard#30864
o-nikolas merged 8 commits into
apache:mainfrom
aws-mwaa:syedahsn/deferrable-redshift-resume-cluster

Conversation

@syedahsn

@syedahsnsyedahsn commented Apr 25, 2023

Copy link
Copy Markdown
Contributor

Now that #30032 is merged, we can update the deferrable implementation for RedshiftResumeClusterOperator to use custom aiobotocore waiters , as well as the async_conn to simplify deferrable implmentation, and reduce code duplication. The method used here is described in the README.md included in the initial PR.
@pankajastro@phanikumv

^ Add meaningful description above

Read the Pull Request Guidelines for more information.
In case of fundamental code changes, an Airflow Improvement Proposal (AIP) is needed.
In case of a new dependency, check compliance with the ASF 3rd Party License Policy.
In case of backwards incompatible changes please leave a note in a newsfragment file, named {pr_number}.significant.rst or {issue_number}.significant.rst, in newsfragments.

@boring-cyborgboring-cyborgBot added area:providers provider:amazon AWS/Amazon - related issues labels Apr 25, 2023
o-nikolas
o-nikolas reviewed Apr 25, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/operators/redshift_cluster.py Outdated
Comment threadairflow/providers/amazon/aws/operators/redshift_cluster.py Outdated
Comment threadairflow/providers/amazon/aws/operators/redshift_cluster.py Outdated
o-nikolas
o-nikolas reviewed Apr 26, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/operators/redshift_cluster.py Outdated
potiuk
potiuk reviewed Apr 29, 2023
View reviewed changes
Comment threadairflow/providers/amazon/aws/triggers/redshift_cluster.py Outdated

@potiukpotiukApr 29, 2023

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also think about the case where we want to log some events in trigger periodically while waiting ? similar to discussion in #30853 (comment) ?

@potiukpotiukApr 29, 2023•
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similarly here: #30244 - it almost seems that even if the waiter implementation in botocore does not provide any way to get callback while waiitng, we could have a wrapper around the waiter that could be used across many of the Deferrable Amazon operators (this would become super useful in 2.6.0 with the triggerer logs interleaved with task logs).

syedahsn added 6 commits May 9, 2023 17:55
@syedahsn
Change base_aws.py to support async_conn
bd76e07
Add async custom waiter support in get_waiter, and base_waiter.py
Add Deferrable mode to RedshiftCreateClusterOperator
Add RedshiftCreateClusterTrigger and unit test
Add README.md for writing Triggers for AMPP
@syedahsn
Add Deferrable mode to Redshift Resume Cluster Operator
9123386
@syedahsn
Rebase on main
ebd7699
Fix/add a few unit tests
@syedahsn
Add max_attempts to docstring
f181692
Remove duplicate log statement
Remove duplicate docstring
@syedahsn
Add wait for completeion to RedshiftResumeClusterOperator
4a2f51a
@syedahsn
Incorporate comments from RedshiftPauseClusterOperator PR (apache#30853)
15c0a78
@syedahsn
syedahsnforce-pushed the syedahsn/deferrable-redshift-resume-cluster branch from 85bf375 to 15c0a78CompareMay 10, 2023 00:55
@potiuk

potiuk commented May 17, 2023

Copy link
Copy Markdown
Member

LGTM: @o-nikolas ?

@pankajastro

pankajastro commented May 17, 2023

Copy link
Copy Markdown
Member

@potiuk I think we are waiting for #30853 to merge then we will rebase other on top of this

@o-nikolas

o-nikolas commented May 17, 2023

Copy link
Copy Markdown
Contributor

@potiuk I think we are waiting for #30853 to merge then we will rebase other on top of this

Yup, this is correct, and I believe Pause is good to go. There was some feedback, but I think Syed has addressed it (while he is on vacation at that!).

P.S. I am on vacation starting today as well, responses will be delayed or non-existent 😁

@o-nikolas
o-nikolas merged commit 44741f3 into apache:mainMay 24, 2023
@vandonr-amz
vandonr-amz deleted the syedahsn/deferrable-redshift-resume-cluster branch May 24, 2023 23:35
@syedahsnsyedahsn mentioned this pull request Jun 7, 2023
Merged
@eladkaleladkal mentioned this pull request Jun 20, 2023
Closed
86 tasks
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@o-nikolaso-nikolaso-nikolas left review comments
@potiukpotiukpotiuk approved these changes
@eladkaleladkalAwaiting requested review from eladkal

Assignees

No one assigned

Labels

area:providersprovider:amazonAWS/Amazon - related issues

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

5 participants

@syedahsn@potiuk@pankajastro@o-nikolas@vandonr-amz