') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Made get_conn method async for KiotaRequestAdapterHook to fix AsyncToSync error by dabla · Pull Request #54598 · apache/airflow · GitHub
Skip to content

Made get_conn method async for KiotaRequestAdapterHook to fix AsyncToSync error - #54598

Merged
gopidesupavan merged 27 commits into
apache:mainfrom
dabla:feature/msgraph-async-connection
Aug 21, 2025
Merged

Made get_conn method async for KiotaRequestAdapterHook to fix AsyncToSync error#54598
gopidesupavan merged 27 commits into
apache:mainfrom
dabla:feature/msgraph-async-connection

Conversation

@dabla

@dabladabla commented Aug 18, 2025•
edited by eladkal
Loading

Copy link
Copy Markdown
Contributor

This is a fix for "An error occurred: You cannot use AsyncToSync in the same thread as an async event loop - just await the async function directly." in Airflow 3, as get_conn should be async.

closes: #54598

^ Add meaningful description above
Read the Pull Request Guidelines for more information.
In case of fundamental code changes, an Airflow Improvement Proposal (AIP) is needed.
In case of a new dependency, check compliance with the ASF 3rd Party License Policy.
In case of backwards incompatible changes please leave a note in a newsfragment file, named {pr_number}.significant.rst or {issue_number}.significant.rst, in airflow-core/newsfragments.

@dabla
refactor: Made get_conn method of KiotaRequestAdapterHook async to fi…
ce7e430
…x "An error occurred: You cannot use AsyncToSync in the same thread as an async event loop - just await the async function directly." in Airflow 3
@dabladabla mentioned this pull request Aug 18, 2025
Closed
2 tasks
gopidesupavan
gopidesupavan reviewed Aug 18, 2025
View reviewed changes
Comment threadproviders/microsoft/azure/tests/unit/microsoft/azure/hooks/test_msgraph.py Outdated
@gopidesupavan

gopidesupavan commented Aug 18, 2025

Copy link
Copy Markdown
Member

@dabla thanks overall LGTM, some tests needs to be fixed.

BTW this is complete breaking change for AF2 users or anyone using these hooks methods , because get_conn is now async awaitable. so its better to add some compatibility and throw warning to users ?

Offcourse this change must for this operator to work with AF3.

@gopidesupavan

gopidesupavan commented Aug 18, 2025

Copy link
Copy Markdown
Member

cc: @eladkal FYI.

@dabla
refactor: Fixed test_serialize
3016455
@dabla
refactor: Fixed test_throw_failed_responses_with_text_plain_content_t…
318abc5
…ype and test_throw_failed_responses_with_application_json_content_type
@dabla

dabla commented Aug 19, 2025

Copy link
Copy Markdown
ContributorAuthor

@dabla thanks overall LGTM, some tests needs to be fixed.

BTW this is complete breaking change for AF2 users or anyone using these hooks methods , because get_conn is now async awaitable. so its better to add some compatibility and throw warning to users ?

Offcourse this change must for this operator to work with AF3.

We could but normally this is an "internal" method, as user would normally not interact with it when using operator of even the hook as there you have the run method. WDYT as get_conn is expected to be async now.

@eladkal

eladkal commented Aug 19, 2025

Copy link
Copy Markdown
Contributor

BTW this is complete breaking change for AF2 users or anyone using these hooks methods , because get_conn is now async awaitable. so its better to add some compatibility and throw warning to users ?

This is not good. I don't think we can break Airflow 2.10 comparability for this do we?

@dabla

dabla commented Aug 19, 2025•
edited
Loading

Copy link
Copy Markdown
ContributorAuthor

BTW this is complete breaking change for AF2 users or anyone using these hooks methods , because get_conn is now async awaitable. so its better to add some compatibility and throw warning to users ?

This is not good. I don't think we can break Airflow 2.10 comparability for this do we?

I don't think it's broken for Airflow 2 as this is "internal kitchen stuff", I can test it to make sure in Airflow 2.

@dabla

dabla commented Aug 19, 2025

Copy link
Copy Markdown
ContributorAuthor

BTW this is complete breaking change for AF2 users or anyone using these hooks methods , because get_conn is now async awaitable. so its better to add some compatibility and throw warning to users ?

This is not good. I don't think we can break Airflow 2.10 comparability for this do we?

If they directly interact with get_conn then yes it's would be broken, but that would be a bad practise normally you should use the run method. I can add a new get_async_conn method and keep the original for backward compatibility.

@dabla

dabla commented Aug 19, 2025

Copy link
Copy Markdown
ContributorAuthor

BTW this is complete breaking change for AF2 users or anyone using these hooks methods , because get_conn is now async awaitable. so its better to add some compatibility and throw warning to users ?

This is not good. I don't think we can break Airflow 2.10 comparability for this do we?

I'll made get_conn synced again and create new method get_async_conn and deprecate the former one.

@eladkal

eladkal commented Aug 19, 2025•
edited
Loading

Copy link
Copy Markdown
Contributor

If they directly interact with get_conn then yes it's would be broken, but that would be a bad practise normally you should use the run method. I can add a new get_async_conn method and keep the original for backward compatibility.

I'm not into the details of this change. If this works on AF2 and we have reasonable claim to classify it as bug fix then this is fine. If mitigation steps or special instructions into what was changed are needed then we need a log entry to specify this change (a key indication to know if this is required is: if we expect that as a result of releasing this change workflows will behave differently)

@dabla

dabla commented Aug 19, 2025

Copy link
Copy Markdown
ContributorAuthor

If they directly interact with get_conn then yes it's would be broken, but that would be a bad practise normally you should use the run method. I can add a new get_async_conn method and keep the original for backward compatibility.

I'm not into the details of this change. If this works on AF2 and we have reasonable claim to classify it as bug fix then this is fine. If mitigation steps or special instructions into what was changed are needed then we need a log entry to specify this change (a key indication to know if this is required is: if we expect that as a result of releasing this change workflows will behave differently)

Well I just kept original method and added new async one like other providers do, it wasn't big of a deal after all, and so we are certain we don't break anything, @gopidesupavan was right by mentioning this.

@dabla
dabla marked this pull request as draft August 19, 2025 07:44
@eladkal

eladkal commented Aug 19, 2025

Copy link
Copy Markdown
Contributor

Well I just kept original method and added new async one like other providers do, it wasn't big of a deal after all, and so we are certain we don't break anything, @gopidesupavan was right by mentioning this.

Cool

@dabla

dabla commented Aug 19, 2025

Copy link
Copy Markdown
ContributorAuthor

Will adapt PowerBI related triggers also so it's aligned with MSGraph as they use the same code.

@dabla
dabla marked this pull request as ready for review August 19, 2025 09:08
@dabla

dabla commented Aug 19, 2025

Copy link
Copy Markdown
ContributorAuthor

@eladkal Should be ready now

@gopidesupavan

gopidesupavan commented Aug 19, 2025

Copy link
Copy Markdown
Member

Thanks for the update, instead of removing the properties its better to throw depreciation warning. my only concern is all these are public facing methods or properties, not how users interact with these so its better with deprecation warning. WDYT?

@dabla

dabla commented Aug 20, 2025

Copy link
Copy Markdown
ContributorAuthor

Thanks for the update, instead of removing the properties its better to throw depreciation warning. my only concern is all these are public facing methods or properties, not how users interact with these so its better with deprecation warning. WDYT?

I've re-added get_conn with deprecation.

eladkal
eladkal approved these changes Aug 21, 2025
View reviewed changes

@eladkaleladkal left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LGTM
cc @gopidesupavan

gopidesupavan
gopidesupavan approved these changes Aug 21, 2025
View reviewed changes

@gopidesupavangopidesupavan left a comment

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lovely thanks @dabla :)

@gopidesupavan
gopidesupavan merged commit 9c1f368 into apache:mainAug 21, 2025
75 checks passed
mangal-vairalkar pushed a commit to mangal-vairalkar/airflow that referenced this pull request Aug 30, 2025
@dabla@mangal-vairalkar
Made get_conn method async for KiotaRequestAdapterHook to fix AsyncTo…
3ad3660
…Sync error (apache#54598)
* refactor: Made get_conn method of KiotaRequestAdapterHook async to fix "An error occurred: You cannot use AsyncToSync in the same thread as an async event loop - just await the async function directly." in Airflow 3
* refactor: Fixed static checks
* refactor: Re-added missing import HttpxRequestAdapter
* refactor: Reformatted imports
* refactor: Fixed imports
* refactor: Added missing AirflowProviderDeprecationWarning
* refactor: Fixed test_serialize
* refactor: Fixed test_throw_failed_responses_with_text_plain_content_type and test_throw_failed_responses_with_application_json_content_type
* refactor: Renamed async get_conn method to get_async_conn in KiotaRequestAdapterHook
* refactor: Introduced async get_async_conn and keep original synced get_conn method for backward compatibility
* refactor: Introduced async get_async_conn and keep original synced get_conn method for backward compatibility
* refactor: Refactor PowerBI triggers like MSGraph
* refactor: Generate class names dynamically in PowerBI triggers
* refactor: Fixed some mypy issues in PowerBITrigger
* refactor: Fixed test_serialize
* refactor: Reformatted test_get_conn
* refactor: Added docstring to BasePowerBITrigger
* refactor: Replaced DeprecationWarning with AirflowProviderDeprecationWarning
* refactor: Fixed patching of 2 remaining tests
* refactor: Re-added deprecated get_conn methods in triggers
* refactor: Fixed static checks
@eladkaleladkal mentioned this pull request Sep 5, 2025
Closed
This was referenced Sep 12, 2025
Closed
Closed
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@gopidesupavangopidesupavangopidesupavan approved these changes
@eladkaleladkaleladkal approved these changes

Assignees

No one assigned

Labels

area:providersprovider:microsoft-azureAzure-related issues

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

3 participants

@dabla@gopidesupavan@eladkal