') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Support None comparisons for null expressions by zeel2104 · Pull Request #1489 · apache/datafusion-python · GitHub
Skip to content

Support None comparisons for null expressions - #1489

Merged
timsaucer merged 2 commits into
apache:mainfrom
zeel2104:fix-none-null-comparison
Apr 13, 2026
Merged

Support None comparisons for null expressions#1489
timsaucer merged 2 commits into
apache:mainfrom
zeel2104:fix-none-null-comparison

Conversation

@zeel2104

@zeel2104zeel2104 commented Apr 12, 2026

Copy link
Copy Markdown
Contributor

Which issue does this PR close?

Closes#1483.

Rationale for this change

Comparing expressions to None with == currently builds a regular equality comparison against a null literal, which follows SQL null semantics and does not match null values in filters. This is surprising for Python users, especially since comparing against other scalar values works as expected and the equivalent .is_null() expression does return the expected rows.

What changes are included in this PR?

  • Special-case Expr.__eq__ so expr == None maps to expr.is_null()
  • Special-case Expr.__ne__ so expr != None maps to expr.is_not_null()
  • Add regression tests covering == None and != None on nullable integer and string columns

Are there any user-facing changes?

Yes. Python users can now write col("a") == None and col("a") != None as shorthand for is_null() and is_not_null().

is None is not supported because Python identity checks cannot be overloaded.

timsaucer
timsaucer reviewed Apr 12, 2026
View reviewed changes

@timsaucertimsaucer left a comment

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Very nice addition. Thank you for the submission! I have one minor recommendation.

Comment threadpython/tests/test_expr.py Outdated
Comment on lines +176 to +191
def test_relational_expr_none_uses_null_predicates():
ctx = SessionContext()

batch = pa.RecordBatch.from_arrays(
[
pa.array([1, 2, None]),
pa.array(["alpha", None, "gamma"], type=pa.string_view()),
],
names=["a", "b"],
)
df = ctx.create_dataframe([[batch]], name="batch_with_nulls")

assert df.filter(col("a") == None).count() == 1 # noqa: E711
assert df.filter(col("a") != None).count() == 2 # noqa: E711
assert df.filter(col("b") == None).count() == 1 # noqa: E711
assert df.filter(col("b") != None).count() == 2 # noqa: E711

@timsaucertimsaucerApr 12, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you can just update the test_relational_expr to have some null values and incorporate this into the existing test.

@zeel2104zeel2104Apr 12, 2026

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated the tests to fold the None comparison coverage into test_relational_expr and pushed the change. Thanks for the suggestion.

timsaucer
timsaucer approved these changes Apr 13, 2026
View reviewed changes

@timsaucertimsaucer left a comment

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for the contribution! I'll merge after CI finishes.

@timsaucer
timsaucer merged commit 398980d into apache:mainApr 13, 2026
21 checks passed
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@timsaucertimsaucertimsaucer approved these changes

Assignees

No one assigned

Labels

None yet

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

Support None comparison for nulls

2 participants

@zeel2104@timsaucer