') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Enable ruff B008 (function-call-in-default-argument) and fix violations by shahar1 · Pull Request #66979 · apache/airflow · GitHub
Skip to content

Enable ruff B008 (function-call-in-default-argument) and fix violations - #66979

Merged
shahar1 merged 9 commits into
apache:mainfrom
shahar1:enable-ruff-B008-function-call-in-default-argument
May 16, 2026
Merged

Enable ruff B008 (function-call-in-default-argument) and fix violations#66979
shahar1 merged 9 commits into
apache:mainfrom
shahar1:enable-ruff-B008-function-call-in-default-argument

Conversation

@shahar1

@shahar1shahar1 commented May 15, 2026

Copy link
Copy Markdown
Contributor

Summary

Enables ruff rule B008 (function-call-in-default-argument) which catches function calls used as default argument values — a common source of bugs where mutable objects or config reads are shared across all calls instead of being fresh per call.

  • Adds [tool.ruff.lint.flake8-bugbear] extend-immutable-calls to exempt FastAPI DI callables (Depends, Query, Path, Body, Security) and the stateless cryptography SHA256 descriptor, which are intentionally used in argument defaults
  • Fixes 13 real violations across core, task-sdk, shared libs, and providers:
    • task-sdk BaseSensorMixin: conf.getfloat("sensors", "default_timeout") was read once at import time; now read at instantiation
    • task-sdk Resources: conf.getint(...) for cpus/ram/disk/gpus read at import time; now read at instantiation
    • providers/google S3ToGCSOperator: conf.getboolean("operators", "default_deferrable") read at import time
    • providers/google Vertex AI Ray hook and operator: resources.Resources() mutable default shared across all calls
    • shared/observability (SafeStatsdLogger, SafeDogStatsdLogger, SafeOtelLogger): PatternAllowListValidator() mutable instances shared across all logger instances
    • openlineage system test: setup_jinja() mutable Jinja Environment shared across operator instances
    • api_fastapi LimitFilter.depends: conf.getint("api", "fallback_page_limit") in FastAPI dependency method moved to module-level constant _FALLBACK_PAGE_LIMIT, preserving the OpenAPI schema default value while making the intent explicit
    • providers/fab test conftest: types.SimpleNamespace(...) mutable default in context manager helper
Was generative AI tooling used to co-author this PR?
  • Yes — Claude Sonnet 4.6 (claude-sonnet-4-6)

Generated-by: Claude Sonnet 4.6 (claude-sonnet-4-6) following the guidelines

@boring-cyborgboring-cyborgBot added area:API Airflow's REST/HTTP API area:providers area:task-sdk provider:fab provider:google Google (including GCP) related issues provider:openlineage AIP-53 labels May 15, 2026
@shahar1
shahar1 marked this pull request as draft May 15, 2026 05:41
@shahar1
shahar1 marked this pull request as ready for review May 15, 2026 05:51
@shahar1
shahar1 requested a review from CopilotMay 15, 2026 05:51
CopilotAI reviewed May 15, 2026
View reviewed changes

CopilotAI left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pull request overview

This PR enables Ruff B008 across the Airflow repo and updates several function signatures to avoid function calls or mutable objects in default arguments.

Changes:

  • Enables B008 and configures immutable-call exceptions for FastAPI DI and SHA256.
  • Moves several config-derived or mutable defaults into function bodies.
  • Updates a test helper to avoid a mutable SimpleNamespace default.

Reviewed changes

Copilot reviewed 12 out of 12 changed files in this pull request and generated 5 comments.

Show a summary per file
FileDescription
pyproject.tomlEnables B008 and adds bugbear immutable-call exceptions.
task-sdk/src/airflow/sdk/definitions/operator_resources.pyDefers resource config defaults to Resources construction.
task-sdk/src/airflow/sdk/bases/sensor.pyDefers sensor timeout config default to construction.
shared/observability/src/airflow_shared/observability/metrics/statsd_logger.pyReplaces shared validator defaults with per-instance defaults.
shared/observability/src/airflow_shared/observability/metrics/otel_logger.pyReplaces shared validator default with per-instance default.
shared/observability/src/airflow_shared/observability/metrics/datadog_logger.pyReplaces shared validator defaults with per-instance defaults.
providers/openlineage/tests/system/openlineage/operator.pyCreates the Jinja environment per operator instance.
providers/google/src/airflow/providers/google/cloud/transfers/s3_to_gcs.pyDefers deferrable config default to operator construction.
providers/google/src/airflow/providers/google/cloud/operators/vertex_ai/ray.pyCreates default Ray head node resources per operator instance.
providers/google/src/airflow/providers/google/cloud/hooks/vertex_ai/ray.pyCreates default Ray head node resources inside the hook call.
providers/fab/tests/unit/fab/auth_manager/api_fastapi/conftest.pyCreates the default test user per context-manager call.
airflow-core/src/airflow/api_fastapi/common/parameters.pyReplaces a FastAPI default config call with a module constant and adds a B008 suppression.

Comment threadpyproject.toml
Comment threadproviders/google/src/airflow/providers/google/cloud/hooks/vertex_ai/ray.py
Comment threadtask-sdk/src/airflow/sdk/bases/sensor.py
Comment threadtask-sdk/src/airflow/sdk/definitions/operator_resources.py
Comment threadproviders/google/src/airflow/providers/google/cloud/transfers/s3_to_gcs.py Outdated
@shahar1
shahar1 marked this pull request as draft May 15, 2026 05:59
@shahar1
shahar1 requested a review from CopilotMay 15, 2026 06:11
@shahar1
shahar1 marked this pull request as ready for review May 15, 2026 06:11
CopilotAI reviewed May 15, 2026
View reviewed changes

CopilotAI left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pull request overview

Copilot reviewed 15 out of 15 changed files in this pull request and generated 2 comments.

Comment threadproviders/google/src/airflow/providers/google/cloud/hooks/vertex_ai/ray.py Outdated
Comment threadproviders/google/src/airflow/providers/google/cloud/operators/vertex_ai/ray.py Outdated
shahar1 added a commit to shahar1/airflow that referenced this pull request May 15, 2026
@shahar1
Add regression tests for Ray cluster default head_node_type fix
9078e2a
Addresses remaining Copilot review comments on apache#66979:
- Hook test: verify calling create_ray_cluster with head_node_type=None
passes a fresh Resources() instance to vertex_ray.create_ray_cluster
- New operator test file for vertex_ai/ray.py: verify omitting
head_node_type uses a fresh Resources() per instance (not shared),
and that execute() forwards a Resources() instance to the hook
Lee-W
Lee-W reviewed May 15, 2026
View reviewed changes
Comment threadairflow-core/src/airflow/api_fastapi/common/parameters.py Outdated
Comment threadproviders/google/src/airflow/providers/google/cloud/transfers/s3_to_gcs.py Outdated
shahar1 added a commit to shahar1/airflow that referenced this pull request May 15, 2026
@shahar1
Add regression tests for Ray cluster default head_node_type fix
8b3c329
Addresses remaining Copilot review comments on apache#66979:
- Hook test: verify calling create_ray_cluster with head_node_type=None
passes a fresh Resources() instance to vertex_ray.create_ray_cluster
- New operator test file for vertex_ai/ray.py: verify omitting
head_node_type uses a fresh Resources() per instance (not shared),
and that execute() forwards a Resources() instance to the hook
@shahar1
shahar1force-pushed the enable-ruff-B008-function-call-in-default-argument branch from 9078e2a to 35adb98CompareMay 15, 2026 16:17
@shahar1
shahar1 requested review from Lee-W and jschefflMay 16, 2026 05:25
jscheffl
jscheffl reviewed May 16, 2026
View reviewed changes
Comment threadproviders/google/src/airflow/providers/google/cloud/transfers/s3_to_gcs.py
jscheffl
jscheffl reviewed May 16, 2026
View reviewed changes
Comment threadproviders/google/tests/unit/google/cloud/hooks/vertex_ai/test_ray.py
jscheffl
jscheffl reviewed May 16, 2026
View reviewed changes
Comment threadproviders/openlineage/tests/system/openlineage/operator.py Outdated
jscheffl
jscheffl reviewed May 16, 2026
View reviewed changes
Comment threadshared/observability/src/airflow_shared/observability/metrics/datadog_logger.py Outdated
jscheffl
jscheffl reviewed May 16, 2026
View reviewed changes
Comment threadpyproject.toml
shahar1and others added 9 commits May 16, 2026 18:05
@shahar1
Enable ruff B008 (function-call-in-default-argument) and fix violations
1752941
Adds [tool.ruff.lint.flake8-bugbear] extend-immutable-calls to exempt
FastAPI DI callables (Depends, Query, Path, Body, Security) and the
stateless cryptography SHA256 descriptor from B008, then fixes all
remaining violations where conf.get*() or mutable objects were
evaluated once at import time rather than at call time:
- task-sdk: sensor timeout and Resources cpus/ram/disk/gpus now read
from config at instantiation
- providers/google: s3_to_gcs deferrable default, Vertex AI Ray
head_node_type mutable default
- shared/observability: SafeStatsdLogger, SafeDogStatsdLogger, and
SafeOtelLogger metrics_validator/metric_tags_validator mutable
defaults (PatternAllowListValidator shared across instances)
- openlineage system test: setup_jinja() mutable Jinja Environment default
- api_fastapi/common/parameters: LimitFilter.depends conf.getint moved
to module-level constant (_FALLBACK_PAGE_LIMIT) to preserve OpenAPI
schema default while making the intent explicit
- providers/fab tests: types.SimpleNamespace mutable default in conftest
@shahar1
Address Copilot review feedback on B008 PR
13323d3
- Make RayHook.create_ray_cluster use `is not None` for head_node_type
fallback so an explicit falsy value is forwarded instead of being
replaced; matches the operator's behavior.
- Add regression tests for the conf-default-at-instantiation behavior:
* Sensor `timeout` reads `sensors.default_timeout` at construction
* `Resources()` reads `operators.default_*` at construction and
preserves explicit 0 instead of falling back to config
* `S3ToGCSOperator.deferrable` reads `operators.default_deferrable`
at construction; explicit `False` overrides a truthy config value
@shahar1
Add regression tests for Ray cluster default head_node_type fix
81b1a4b
Addresses remaining Copilot review comments on apache#66979:
- Hook test: verify calling create_ray_cluster with head_node_type=None
passes a fresh Resources() instance to vertex_ray.create_ray_cluster
- New operator test file for vertex_ai/ray.py: verify omitting
head_node_type uses a fresh Resources() per instance (not shared),
and that execute() forwards a Resources() instance to the hook
@shahar1
Address Wei Lee review comments
d011b83
- parameters.py: compute Query() before inner() instead of suppressing
B008 with noqa - cleaner and avoids the linter exception
- s3_to_gcs.py: revert deferrable=None pattern; restore the canonical
conf.getboolean() default enforced by check_deferrable_default checker;
add type annotation so B008 doesn't flag the unannotated call
@shahar1@jscheffl
Update providers/google/src/airflow/providers/google/cloud/hooks/vert…
5b855ef
…ex_ai/ray.py
Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com>
@shahar1@jscheffl
Update providers/openlineage/tests/system/openlineage/operator.py
62c2be0
Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com>
@shahar1@jscheffl
Update shared/observability/src/airflow_shared/observability/metrics/…
4cc6857
…datadog_logger.py
Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com>
@shahar1@jscheffl
Update providers/google/src/airflow/providers/google/cloud/operators/…
5858903
…vertex_ai/ray.py
Co-authored-by: Jens Scheffler <95105677+jscheffl@users.noreply.github.com>
@shahar1
Use or instead of is not None else for validator defaults
bec9bea
Address Jens's review comment on PR apache#66979 — the ListValidator
instances assigned to `metrics_validator` / `metric_tags_validator`
are never falsy except when `None`, so the shorter `or` form is
equivalent and easier to read.
@shahar1
shahar1force-pushed the enable-ruff-B008-function-call-in-default-argument branch from f2669d8 to bec9beaCompareMay 16, 2026 15:18
@shahar1
shahar1 requested a review from jschefflMay 16, 2026 15:18
@shahar1
shahar1 merged commit 74ec32f into apache:mainMay 16, 2026
292 checks passed
@dkranchiidkranchii mentioned this pull request Jul 28, 2026
Merged
1 task
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

Copilot code reviewCopilot
Copilot review effort
Applies to this pull request for everyone.Learn more about Copilot code review.
Copilot left review comments
@jscheffljscheffljscheffl approved these changes
@ashbashbAwaiting requested review from ashbashb is a code owner
@amoghrajeshamoghrajeshAwaiting requested review from amoghrajeshamoghrajesh is a code owner
@potiukpotiukAwaiting requested review from potiukpotiuk is a code owner
@kaxilkaxilAwaiting requested review from kaxilkaxil is a code owner
@mobuchowskimobuchowskiAwaiting requested review from mobuchowskimobuchowski is a code owner
@vincbeckvincbeckAwaiting requested review from vincbeckvincbeck is a code owner
@ephraimbuddyephraimbuddyAwaiting requested review from ephraimbuddyephraimbuddy is a code owner
@pierrejeambrunpierrejeambrunAwaiting requested review from pierrejeambrunpierrejeambrun is a code owner
@rawwarrawwarAwaiting requested review from rawwarrawwar is a code owner
@jason810496jason810496Awaiting requested review from jason810496jason810496 is a code owner
@bugraoz93bugraoz93Awaiting requested review from bugraoz93bugraoz93 is a code owner
@shubhamraj-gitshubhamraj-gitAwaiting requested review from shubhamraj-gitshubhamraj-git is a code owner
@choo121600choo121600Awaiting requested review from choo121600choo121600 is a code owner
@vatsrahul1001vatsrahul1001Awaiting requested review from vatsrahul1001
@Lee-WLee-WAwaiting requested review from Lee-W

Assignees

No one assigned

Labels

area:APIAirflow's REST/HTTP APIarea:providersarea:task-sdkprovider:fabprovider:googleGoogle (including GCP) related issuesprovider:openlineageAIP-53

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

4 participants

@shahar1@Lee-W@jscheffl