Replace Ruff's known-third-party setting with namespace-packages#32873

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
uranusjr merged 1 commit into from
Jul 27, 2023
+36 −53
Merged

Replace Ruff's known-third-party setting with namespace-packages#32873

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions airflow/__init__.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -49,8 +49,7 @@
# very easily cause import cycles in the conf init/validate code (since downstream code from
# those functions likely import settings).
# configuration is therefore initted early here, simply by importing it.
from airflow import configuration
from airflow import settings
from airflow import configuration, settings

__all__ = ["__version__", "login", "DAG", "PY36", "PY37", "PY38", "PY39", "PY310", "XComArg"]

Expand DownExpand Up@@ -125,7 +124,7 @@ def __getattr__(name: str):
STATICA_HACK = True
globals()["kcah_acitats"[::-1].upper()] = False
if STATICA_HACK: # pragma: no cover
from airflow.models.dag import DAG
from airflow.models.xcom_arg import XComArg
from airflow.exceptions import AirflowException
from airflow.models.dag import DAG
from airflow.models.dataset import Dataset
from airflow.models.xcom_arg import XComArg
3 changes: 2 additions & 1 deletion airflow/providers/apprise/hooks/apprise.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,9 +21,10 @@
from typing import Any, Iterable

import apprise
from airflow.hooks.base import BaseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType

@charliermarshcharliermarshJul 27, 2023

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an example: these apprise imports were incorrectly marked as first-party, since apprise was missing from the known-third-party.


from airflow.hooks.base import BaseHook


class AppriseHook(BaseHook):
"""
Expand Down
3 changes: 2 additions & 1 deletion airflow/providers/apprise/notifications/apprise.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -29,9 +29,10 @@
"Failed to import BaseNotifier. This feature is only available in Airflow versions >= 2.6.0"
)

from airflow.providers.apprise.hooks.apprise import AppriseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType

from airflow.providers.apprise.hooks.apprise import AppriseHook


class AppriseNotifier(BaseNotifier):
"""
Expand Down
4 changes: 2 additions & 2 deletions airflow/providers/openlineage/extractors/base.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,11 @@
from abc import ABC, abstractmethod

from attrs import Factory, define
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset

from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.state import TaskInstanceState
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset


@define
Expand Down
3 changes: 2 additions & 1 deletion airflow/providers/openlineage/extractors/bash.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

from __future__ import annotations

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
UnknownOperatorAttributeRunFacet,
UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down
3 changes: 2 additions & 1 deletion airflow/providers/openlineage/extractors/manager.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -173,9 +173,10 @@ def extract_inlets_and_outlets(

@staticmethod
def convert_to_ol_dataset(obj):
from airflow.lineage.entities import Table
from openlineage.client.run import Dataset

from airflow.lineage.entities import Table

if isinstance(obj, Dataset):
return obj
elif isinstance(obj, Table):
Expand Down
3 changes: 2 additions & 1 deletion airflow/providers/openlineage/extractors/python.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,13 +20,14 @@
import inspect
from typing import Callable

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
UnknownOperatorAttributeRunFacet,
UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down
12 changes: 6 additions & 6 deletions airflow/providers/openlineage/plugins/adapter.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,6 @@

import requests.exceptions
import yaml

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin
from openlineage.client import OpenLineageClient, set_producer
from openlineage.client.facet import (
BaseFacet,
Expand All@@ -42,6 +36,12 @@
)
from openlineage.client.run import Job, Run, RunEvent, RunState

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin

if TYPE_CHECKING:
from airflow.models.dagrun import DagRun
from airflow.utils.log.secrets_masker import SecretsMasker
Expand Down
1 change: 0 additions & 1 deletion airflow/providers/openlineage/plugins/facets.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,7 +17,6 @@
from __future__ import annotations

from attrs import define

from openlineage.client.facet import BaseFacet
from openlineage.client.utils import RedactMixin

Expand Down
6 changes: 3 additions & 3 deletions airflow/providers/openlineage/sqlparser.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,9 @@

import sqlparse
from attrs import define
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

from airflow.providers.openlineage.extractors.base import OperatorLineage
from airflow.providers.openlineage.utils.sql import (
Expand All@@ -28,9 +31,6 @@
get_table_schemas,
)
from airflow.typing_compat import TypedDict
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

if TYPE_CHECKING:
from sqlalchemy.engine import Engine
Expand Down
3 changes: 1 addition & 2 deletions airflow/providers/openlineage/utils/sql.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,9 @@
from typing import TYPE_CHECKING, Dict, List, Optional

from attrs import define
from sqlalchemy import Column, MetaData, Table, and_, union_all

from openlineage.client.facet import SchemaDatasetFacet, SchemaField
from openlineage.client.run import Dataset
from sqlalchemy import Column, MetaData, Table, and_, union_all

if TYPE_CHECKING:
from sqlalchemy.engine import Engine
Expand Down
6 changes: 3 additions & 3 deletions airflow/providers/openlineage/utils/utils.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@
import attrs
from attrs import asdict

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

from airflow.compat.functools import cache
from airflow.configuration import conf
from airflow.providers.openlineage.plugins.facets import (
Expand All@@ -37,9 +40,6 @@
)
from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

if TYPE_CHECKING:
from airflow.models import DAG, BaseOperator, Connection, DagRun, TaskInstance

Expand Down
4 changes: 4 additions & 0 deletions dev/breeze/pyproject.toml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -33,3 +33,7 @@ python_files = [
testpaths = [
"tests",
]

[tool.ruff]
extend = "../../pyproject.toml"
src = ["src"]
31 changes: 4 additions & 27 deletions pyproject.toml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,8 @@ extend-exclude = [
# The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can
# ignore them in ruff
"airflow/providers/common/sql/*/*.pyi",
"airflow/migrations/versions/*.py"
"airflow/migrations/versions/*.py",
"tests/dags/test_imports.py",
]

extend-select = [
Expand DownExpand Up@@ -73,6 +74,8 @@ extend-ignore = [
"E731",
]

namespace-packages = ["airflow/providers"]

[tool.pytest.ini_options]
# * Disable `flaky` plugin for pytest. This plugin conflicts with `rerunfailures` because provide same marker.
# * Disable `nose` builtin plugin for pytest. This feature deprecated in 7.2 and will be removed in pytest>=8
Expand DownExpand Up@@ -106,39 +109,13 @@ testpaths = [
]

[tool.ruff.isort]
known-first-party = ["airflow", "airflow_breeze", "docker_tests", "docs", "kubernetes_tests", "tests"]
required-imports = ["from __future__ import annotations"]
combine-as-imports = true

# TODO: for now, https://github.com/charliermarsh/ruff/issues/1817
known-third-party = [
"asana",
"atlassian",
"celery",
"cloudant",
"databricks",
"datadog",
"docker",
"elasticsearch",
"github",
"google",
"grpc",
"jenkins",
"mysql",
"neo4j",
"papermill",
"redis",
"sendgrid",
"snowflake",
"telegram",
"trino",
]

[tool.ruff.per-file-ignores]
"airflow/models/__init__.py" = ["F401"]
"airflow/models/sqla_models.py" = ["F401"]


# The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
# needed for the test to work
"tests/decorators/test_python.py" = ["I002"]
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Replace Ruff's known-third-party setting with namespace-packages#32873

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

uranusjr
merged 1 commit into

from

Jul 27, 2023

+36

−53

 Merged


Replace Ruff's known-third-party setting with namespace-packages#32873
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 3 additions & 4 deletions
airflow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,8 +49,7 @@
# very easily cause import cycles in the conf init/validate code (since downstream code from
# those functions likely import settings).
# configuration is therefore initted early here, simply by importing it.
from airflow import configuration
from airflow import settings
from airflow import configuration, settings

__all__ = ["__version__", "login", "DAG", "PY36", "PY37", "PY38", "PY39", "PY310", "XComArg"]

Expand DownExpand Up@@ -125,7 +124,7 @@ def __getattr__(name: str):
STATICA_HACK = True
globals()["kcah_acitats"[::-1].upper()] = False
if STATICA_HACK: # pragma: no cover
 from airflow.models.dag import DAG
 from airflow.models.xcom_arg import XComArg
 from airflow.exceptions import AirflowException
 from airflow.models.dag import DAG
 from airflow.models.dataset import Dataset
 from airflow.models.xcom_arg import XComArg

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/hooks/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,9 +21,10 @@
from typing import Any, Iterable

import apprise
from airflow.hooks.base import BaseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType
@charliermarshcharliermarshJul 27, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an example: these apprise imports were incorrectly marked as first-party, since apprise was missing from the known-third-party.

from airflow.hooks.base import BaseHook


class AppriseHook(BaseHook):
 """
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/notifications/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,9 +29,10 @@
 "Failed to import BaseNotifier. This feature is only available in Airflow versions >= 2.6.0"
 )

from airflow.providers.apprise.hooks.apprise import AppriseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType

from airflow.providers.apprise.hooks.apprise import AppriseHook


class AppriseNotifier(BaseNotifier):
 """
Expand Down

4 changes: 2 additions & 2 deletions
airflow/providers/openlineage/extractors/base.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,11 @@
from abc import ABC, abstractmethod

from attrs import Factory, define
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset

from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.state import TaskInstanceState
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset


@define
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

from __future__ import annotations

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/manager.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -173,9 +173,10 @@ def extract_inlets_and_outlets(

 @staticmethod
 def convert_to_ol_dataset(obj):
 from airflow.lineage.entities import Table
 from openlineage.client.run import Dataset

 from airflow.lineage.entities import Table

 if isinstance(obj, Dataset):
 return obj
 elif isinstance(obj, Table):
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/python.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,13 +20,14 @@
import inspect
from typing import Callable

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

12 changes: 6 additions & 6 deletions
airflow/providers/openlineage/plugins/adapter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,6 @@

import requests.exceptions
import yaml

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin
from openlineage.client import OpenLineageClient, set_producer
from openlineage.client.facet import (
 BaseFacet,
Expand All@@ -42,6 +36,12 @@
)
from openlineage.client.run import Job, Run, RunEvent, RunState

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin

if TYPE_CHECKING:
 from airflow.models.dagrun import DagRun
 from airflow.utils.log.secrets_masker import SecretsMasker
Expand Down

1 change: 0 additions & 1 deletion
airflow/providers/openlineage/plugins/facets.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,7 +17,6 @@
from __future__ import annotations

from attrs import define

from openlineage.client.facet import BaseFacet
from openlineage.client.utils import RedactMixin

Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/sqlparser.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,9 @@

import sqlparse
from attrs import define
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

from airflow.providers.openlineage.extractors.base import OperatorLineage
from airflow.providers.openlineage.utils.sql import (
Expand All@@ -28,9 +31,6 @@
 get_table_schemas,
)
from airflow.typing_compat import TypedDict
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

3 changes: 1 addition & 2 deletions
airflow/providers/openlineage/utils/sql.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,9 @@
from typing import TYPE_CHECKING, Dict, List, Optional

from attrs import define
from sqlalchemy import Column, MetaData, Table, and_, union_all

from openlineage.client.facet import SchemaDatasetFacet, SchemaField
from openlineage.client.run import Dataset
from sqlalchemy import Column, MetaData, Table, and_, union_all

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/utils/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@
import attrs
from attrs import asdict

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

from airflow.compat.functools import cache
from airflow.configuration import conf
from airflow.providers.openlineage.plugins.facets import (
Expand All@@ -37,9 +40,6 @@
)
from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

if TYPE_CHECKING:
 from airflow.models import DAG, BaseOperator, Connection, DagRun, TaskInstance

Expand Down

4 changes: 4 additions & 0 deletions
dev/breeze/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,3 +33,7 @@ python_files = [
testpaths = [
 "tests",
]

[tool.ruff]
extend = "../../pyproject.toml"
src = ["src"]

31 changes: 4 additions & 27 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,8 @@ extend-exclude = [
 # The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can
 # ignore them in ruff
 "airflow/providers/common/sql/*/*.pyi",
 "airflow/migrations/versions/*.py"
 "airflow/migrations/versions/*.py",
 "tests/dags/test_imports.py",
]

extend-select = [
Expand DownExpand Up@@ -73,6 +74,8 @@ extend-ignore = [
 "E731",
]

namespace-packages = ["airflow/providers"]

[tool.pytest.ini_options]
# * Disable `flaky` plugin for pytest. This plugin conflicts with `rerunfailures` because provide same marker.
# * Disable `nose` builtin plugin for pytest. This feature deprecated in 7.2 and will be removed in pytest>=8
Expand DownExpand Up@@ -106,39 +109,13 @@ testpaths = [
]

[tool.ruff.isort]
known-first-party = ["airflow", "airflow_breeze", "docker_tests", "docs", "kubernetes_tests", "tests"]
required-imports = ["from __future__ import annotations"]
combine-as-imports = true

# TODO: for now, https://github.com/charliermarsh/ruff/issues/1817
known-third-party = [
 "asana",
 "atlassian",
 "celery",
 "cloudant",
 "databricks",
 "datadog",
 "docker",
 "elasticsearch",
 "github",
 "google",
 "grpc",
 "jenkins",
 "mysql",
 "neo4j",
 "papermill",
 "redis",
 "sendgrid",
 "snowflake",
 "telegram",
 "trino",
]

[tool.ruff.per-file-ignores]
"airflow/models/__init__.py" = ["F401"]
"airflow/models/sqla_models.py" = ["F401"]


# The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
# needed for the test to work
"tests/decorators/test_python.py" = ["I002"]
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Replace Ruff's known-third-party setting with namespace-packages#32873

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

uranusjr
merged 1 commit into

from

Jul 27, 2023

+36

−53

 Merged


Replace Ruff's known-third-party setting with namespace-packages#32873
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 3 additions & 4 deletions
airflow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,8 +49,7 @@
# very easily cause import cycles in the conf init/validate code (since downstream code from
# those functions likely import settings).
# configuration is therefore initted early here, simply by importing it.
from airflow import configuration
from airflow import settings
from airflow import configuration, settings

__all__ = ["__version__", "login", "DAG", "PY36", "PY37", "PY38", "PY39", "PY310", "XComArg"]

Expand DownExpand Up@@ -125,7 +124,7 @@ def __getattr__(name: str):
STATICA_HACK = True
globals()["kcah_acitats"[::-1].upper()] = False
if STATICA_HACK: # pragma: no cover
 from airflow.models.dag import DAG
 from airflow.models.xcom_arg import XComArg
 from airflow.exceptions import AirflowException
 from airflow.models.dag import DAG
 from airflow.models.dataset import Dataset
 from airflow.models.xcom_arg import XComArg

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/hooks/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,9 +21,10 @@
from typing import Any, Iterable

import apprise
from airflow.hooks.base import BaseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType
@charliermarshcharliermarshJul 27, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an example: these apprise imports were incorrectly marked as first-party, since apprise was missing from the known-third-party.

from airflow.hooks.base import BaseHook


class AppriseHook(BaseHook):
 """
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/notifications/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,9 +29,10 @@
 "Failed to import BaseNotifier. This feature is only available in Airflow versions >= 2.6.0"
 )

from airflow.providers.apprise.hooks.apprise import AppriseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType

from airflow.providers.apprise.hooks.apprise import AppriseHook


class AppriseNotifier(BaseNotifier):
 """
Expand Down

4 changes: 2 additions & 2 deletions
airflow/providers/openlineage/extractors/base.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,11 @@
from abc import ABC, abstractmethod

from attrs import Factory, define
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset

from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.state import TaskInstanceState
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset


@define
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

from __future__ import annotations

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/manager.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -173,9 +173,10 @@ def extract_inlets_and_outlets(

 @staticmethod
 def convert_to_ol_dataset(obj):
 from airflow.lineage.entities import Table
 from openlineage.client.run import Dataset

 from airflow.lineage.entities import Table

 if isinstance(obj, Dataset):
 return obj
 elif isinstance(obj, Table):
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/python.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,13 +20,14 @@
import inspect
from typing import Callable

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

12 changes: 6 additions & 6 deletions
airflow/providers/openlineage/plugins/adapter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,6 @@

import requests.exceptions
import yaml

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin
from openlineage.client import OpenLineageClient, set_producer
from openlineage.client.facet import (
 BaseFacet,
Expand All@@ -42,6 +36,12 @@
)
from openlineage.client.run import Job, Run, RunEvent, RunState

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin

if TYPE_CHECKING:
 from airflow.models.dagrun import DagRun
 from airflow.utils.log.secrets_masker import SecretsMasker
Expand Down

1 change: 0 additions & 1 deletion
airflow/providers/openlineage/plugins/facets.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,7 +17,6 @@
from __future__ import annotations

from attrs import define

from openlineage.client.facet import BaseFacet
from openlineage.client.utils import RedactMixin

Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/sqlparser.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,9 @@

import sqlparse
from attrs import define
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

from airflow.providers.openlineage.extractors.base import OperatorLineage
from airflow.providers.openlineage.utils.sql import (
Expand All@@ -28,9 +31,6 @@
 get_table_schemas,
)
from airflow.typing_compat import TypedDict
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

3 changes: 1 addition & 2 deletions
airflow/providers/openlineage/utils/sql.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,9 @@
from typing import TYPE_CHECKING, Dict, List, Optional

from attrs import define
from sqlalchemy import Column, MetaData, Table, and_, union_all

from openlineage.client.facet import SchemaDatasetFacet, SchemaField
from openlineage.client.run import Dataset
from sqlalchemy import Column, MetaData, Table, and_, union_all

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/utils/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@
import attrs
from attrs import asdict

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

from airflow.compat.functools import cache
from airflow.configuration import conf
from airflow.providers.openlineage.plugins.facets import (
Expand All@@ -37,9 +40,6 @@
)
from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

if TYPE_CHECKING:
 from airflow.models import DAG, BaseOperator, Connection, DagRun, TaskInstance

Expand Down

4 changes: 4 additions & 0 deletions
dev/breeze/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,3 +33,7 @@ python_files = [
testpaths = [
 "tests",
]

[tool.ruff]
extend = "../../pyproject.toml"
src = ["src"]

31 changes: 4 additions & 27 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,8 @@ extend-exclude = [
 # The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can
 # ignore them in ruff
 "airflow/providers/common/sql/*/*.pyi",
 "airflow/migrations/versions/*.py"
 "airflow/migrations/versions/*.py",
 "tests/dags/test_imports.py",
]

extend-select = [
Expand DownExpand Up@@ -73,6 +74,8 @@ extend-ignore = [
 "E731",
]

namespace-packages = ["airflow/providers"]

[tool.pytest.ini_options]
# * Disable `flaky` plugin for pytest. This plugin conflicts with `rerunfailures` because provide same marker.
# * Disable `nose` builtin plugin for pytest. This feature deprecated in 7.2 and will be removed in pytest>=8
Expand DownExpand Up@@ -106,39 +109,13 @@ testpaths = [
]

[tool.ruff.isort]
known-first-party = ["airflow", "airflow_breeze", "docker_tests", "docs", "kubernetes_tests", "tests"]
required-imports = ["from __future__ import annotations"]
combine-as-imports = true

# TODO: for now, https://github.com/charliermarsh/ruff/issues/1817
known-third-party = [
 "asana",
 "atlassian",
 "celery",
 "cloudant",
 "databricks",
 "datadog",
 "docker",
 "elasticsearch",
 "github",
 "google",
 "grpc",
 "jenkins",
 "mysql",
 "neo4j",
 "papermill",
 "redis",
 "sendgrid",
 "snowflake",
 "telegram",
 "trino",
]

[tool.ruff.per-file-ignores]
"airflow/models/__init__.py" = ["F401"]
"airflow/models/sqla_models.py" = ["F401"]


# The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
# needed for the test to work
"tests/decorators/test_python.py" = ["I002"]
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Replace Ruff's known-third-party setting with namespace-packages#32873

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

uranusjr
merged 1 commit into

from

Jul 27, 2023

+36

−53

 Merged


Replace Ruff's known-third-party setting with namespace-packages#32873
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 3 additions & 4 deletions
airflow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,8 +49,7 @@
# very easily cause import cycles in the conf init/validate code (since downstream code from
# those functions likely import settings).
# configuration is therefore initted early here, simply by importing it.
from airflow import configuration
from airflow import settings
from airflow import configuration, settings

__all__ = ["__version__", "login", "DAG", "PY36", "PY37", "PY38", "PY39", "PY310", "XComArg"]

Expand DownExpand Up@@ -125,7 +124,7 @@ def __getattr__(name: str):
STATICA_HACK = True
globals()["kcah_acitats"[::-1].upper()] = False
if STATICA_HACK: # pragma: no cover
 from airflow.models.dag import DAG
 from airflow.models.xcom_arg import XComArg
 from airflow.exceptions import AirflowException
 from airflow.models.dag import DAG
 from airflow.models.dataset import Dataset
 from airflow.models.xcom_arg import XComArg

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/hooks/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,9 +21,10 @@
from typing import Any, Iterable

import apprise
from airflow.hooks.base import BaseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType
@charliermarshcharliermarshJul 27, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an example: these apprise imports were incorrectly marked as first-party, since apprise was missing from the known-third-party.

from airflow.hooks.base import BaseHook


class AppriseHook(BaseHook):
 """
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/notifications/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,9 +29,10 @@
 "Failed to import BaseNotifier. This feature is only available in Airflow versions >= 2.6.0"
 )

from airflow.providers.apprise.hooks.apprise import AppriseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType

from airflow.providers.apprise.hooks.apprise import AppriseHook


class AppriseNotifier(BaseNotifier):
 """
Expand Down

4 changes: 2 additions & 2 deletions
airflow/providers/openlineage/extractors/base.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,11 @@
from abc import ABC, abstractmethod

from attrs import Factory, define
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset

from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.state import TaskInstanceState
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset


@define
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

from __future__ import annotations

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/manager.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -173,9 +173,10 @@ def extract_inlets_and_outlets(

 @staticmethod
 def convert_to_ol_dataset(obj):
 from airflow.lineage.entities import Table
 from openlineage.client.run import Dataset

 from airflow.lineage.entities import Table

 if isinstance(obj, Dataset):
 return obj
 elif isinstance(obj, Table):
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/python.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,13 +20,14 @@
import inspect
from typing import Callable

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

12 changes: 6 additions & 6 deletions
airflow/providers/openlineage/plugins/adapter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,6 @@

import requests.exceptions
import yaml

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin
from openlineage.client import OpenLineageClient, set_producer
from openlineage.client.facet import (
 BaseFacet,
Expand All@@ -42,6 +36,12 @@
)
from openlineage.client.run import Job, Run, RunEvent, RunState

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin

if TYPE_CHECKING:
 from airflow.models.dagrun import DagRun
 from airflow.utils.log.secrets_masker import SecretsMasker
Expand Down

1 change: 0 additions & 1 deletion
airflow/providers/openlineage/plugins/facets.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,7 +17,6 @@
from __future__ import annotations

from attrs import define

from openlineage.client.facet import BaseFacet
from openlineage.client.utils import RedactMixin

Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/sqlparser.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,9 @@

import sqlparse
from attrs import define
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

from airflow.providers.openlineage.extractors.base import OperatorLineage
from airflow.providers.openlineage.utils.sql import (
Expand All@@ -28,9 +31,6 @@
 get_table_schemas,
)
from airflow.typing_compat import TypedDict
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

3 changes: 1 addition & 2 deletions
airflow/providers/openlineage/utils/sql.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,9 @@
from typing import TYPE_CHECKING, Dict, List, Optional

from attrs import define
from sqlalchemy import Column, MetaData, Table, and_, union_all

from openlineage.client.facet import SchemaDatasetFacet, SchemaField
from openlineage.client.run import Dataset
from sqlalchemy import Column, MetaData, Table, and_, union_all

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/utils/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@
import attrs
from attrs import asdict

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

from airflow.compat.functools import cache
from airflow.configuration import conf
from airflow.providers.openlineage.plugins.facets import (
Expand All@@ -37,9 +40,6 @@
)
from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

if TYPE_CHECKING:
 from airflow.models import DAG, BaseOperator, Connection, DagRun, TaskInstance

Expand Down

4 changes: 4 additions & 0 deletions
dev/breeze/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,3 +33,7 @@ python_files = [
testpaths = [
 "tests",
]

[tool.ruff]
extend = "../../pyproject.toml"
src = ["src"]

31 changes: 4 additions & 27 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,8 @@ extend-exclude = [
 # The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can
 # ignore them in ruff
 "airflow/providers/common/sql/*/*.pyi",
 "airflow/migrations/versions/*.py"
 "airflow/migrations/versions/*.py",
 "tests/dags/test_imports.py",
]

extend-select = [
Expand DownExpand Up@@ -73,6 +74,8 @@ extend-ignore = [
 "E731",
]

namespace-packages = ["airflow/providers"]

[tool.pytest.ini_options]
# * Disable `flaky` plugin for pytest. This plugin conflicts with `rerunfailures` because provide same marker.
# * Disable `nose` builtin plugin for pytest. This feature deprecated in 7.2 and will be removed in pytest>=8
Expand DownExpand Up@@ -106,39 +109,13 @@ testpaths = [
]

[tool.ruff.isort]
known-first-party = ["airflow", "airflow_breeze", "docker_tests", "docs", "kubernetes_tests", "tests"]
required-imports = ["from __future__ import annotations"]
combine-as-imports = true

# TODO: for now, https://github.com/charliermarsh/ruff/issues/1817
known-third-party = [
 "asana",
 "atlassian",
 "celery",
 "cloudant",
 "databricks",
 "datadog",
 "docker",
 "elasticsearch",
 "github",
 "google",
 "grpc",
 "jenkins",
 "mysql",
 "neo4j",
 "papermill",
 "redis",
 "sendgrid",
 "snowflake",
 "telegram",
 "trino",
]

[tool.ruff.per-file-ignores]
"airflow/models/__init__.py" = ["F401"]
"airflow/models/sqla_models.py" = ["F401"]


# The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
# needed for the test to work
"tests/decorators/test_python.py" = ["I002"]
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Replace Ruff's known-third-party setting with namespace-packages#32873

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

uranusjr
merged 1 commit into

from

Jul 27, 2023

+36

−53

 Merged


Replace Ruff's known-third-party setting with namespace-packages#32873
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 3 additions & 4 deletions
airflow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,8 +49,7 @@
# very easily cause import cycles in the conf init/validate code (since downstream code from
# those functions likely import settings).
# configuration is therefore initted early here, simply by importing it.
from airflow import configuration
from airflow import settings
from airflow import configuration, settings

__all__ = ["__version__", "login", "DAG", "PY36", "PY37", "PY38", "PY39", "PY310", "XComArg"]

Expand DownExpand Up@@ -125,7 +124,7 @@ def __getattr__(name: str):
STATICA_HACK = True
globals()["kcah_acitats"[::-1].upper()] = False
if STATICA_HACK: # pragma: no cover
 from airflow.models.dag import DAG
 from airflow.models.xcom_arg import XComArg
 from airflow.exceptions import AirflowException
 from airflow.models.dag import DAG
 from airflow.models.dataset import Dataset
 from airflow.models.xcom_arg import XComArg

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/hooks/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,9 +21,10 @@
from typing import Any, Iterable

import apprise
from airflow.hooks.base import BaseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType
@charliermarshcharliermarshJul 27, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an example: these apprise imports were incorrectly marked as first-party, since apprise was missing from the known-third-party.

from airflow.hooks.base import BaseHook


class AppriseHook(BaseHook):
 """
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/notifications/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,9 +29,10 @@
 "Failed to import BaseNotifier. This feature is only available in Airflow versions >= 2.6.0"
 )

from airflow.providers.apprise.hooks.apprise import AppriseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType

from airflow.providers.apprise.hooks.apprise import AppriseHook


class AppriseNotifier(BaseNotifier):
 """
Expand Down

4 changes: 2 additions & 2 deletions
airflow/providers/openlineage/extractors/base.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,11 @@
from abc import ABC, abstractmethod

from attrs import Factory, define
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset

from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.state import TaskInstanceState
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset


@define
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

from __future__ import annotations

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/manager.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -173,9 +173,10 @@ def extract_inlets_and_outlets(

 @staticmethod
 def convert_to_ol_dataset(obj):
 from airflow.lineage.entities import Table
 from openlineage.client.run import Dataset

 from airflow.lineage.entities import Table

 if isinstance(obj, Dataset):
 return obj
 elif isinstance(obj, Table):
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/python.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,13 +20,14 @@
import inspect
from typing import Callable

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

12 changes: 6 additions & 6 deletions
airflow/providers/openlineage/plugins/adapter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,6 @@

import requests.exceptions
import yaml

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin
from openlineage.client import OpenLineageClient, set_producer
from openlineage.client.facet import (
 BaseFacet,
Expand All@@ -42,6 +36,12 @@
)
from openlineage.client.run import Job, Run, RunEvent, RunState

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin

if TYPE_CHECKING:
 from airflow.models.dagrun import DagRun
 from airflow.utils.log.secrets_masker import SecretsMasker
Expand Down

1 change: 0 additions & 1 deletion
airflow/providers/openlineage/plugins/facets.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,7 +17,6 @@
from __future__ import annotations

from attrs import define

from openlineage.client.facet import BaseFacet
from openlineage.client.utils import RedactMixin

Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/sqlparser.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,9 @@

import sqlparse
from attrs import define
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

from airflow.providers.openlineage.extractors.base import OperatorLineage
from airflow.providers.openlineage.utils.sql import (
Expand All@@ -28,9 +31,6 @@
 get_table_schemas,
)
from airflow.typing_compat import TypedDict
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

3 changes: 1 addition & 2 deletions
airflow/providers/openlineage/utils/sql.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,9 @@
from typing import TYPE_CHECKING, Dict, List, Optional

from attrs import define
from sqlalchemy import Column, MetaData, Table, and_, union_all

from openlineage.client.facet import SchemaDatasetFacet, SchemaField
from openlineage.client.run import Dataset
from sqlalchemy import Column, MetaData, Table, and_, union_all

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/utils/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@
import attrs
from attrs import asdict

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

from airflow.compat.functools import cache
from airflow.configuration import conf
from airflow.providers.openlineage.plugins.facets import (
Expand All@@ -37,9 +40,6 @@
)
from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

if TYPE_CHECKING:
 from airflow.models import DAG, BaseOperator, Connection, DagRun, TaskInstance

Expand Down

4 changes: 4 additions & 0 deletions
dev/breeze/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,3 +33,7 @@ python_files = [
testpaths = [
 "tests",
]

[tool.ruff]
extend = "../../pyproject.toml"
src = ["src"]

31 changes: 4 additions & 27 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,8 @@ extend-exclude = [
 # The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can
 # ignore them in ruff
 "airflow/providers/common/sql/*/*.pyi",
 "airflow/migrations/versions/*.py"
 "airflow/migrations/versions/*.py",
 "tests/dags/test_imports.py",
]

extend-select = [
Expand DownExpand Up@@ -73,6 +74,8 @@ extend-ignore = [
 "E731",
]

namespace-packages = ["airflow/providers"]

[tool.pytest.ini_options]
# * Disable `flaky` plugin for pytest. This plugin conflicts with `rerunfailures` because provide same marker.
# * Disable `nose` builtin plugin for pytest. This feature deprecated in 7.2 and will be removed in pytest>=8
Expand DownExpand Up@@ -106,39 +109,13 @@ testpaths = [
]

[tool.ruff.isort]
known-first-party = ["airflow", "airflow_breeze", "docker_tests", "docs", "kubernetes_tests", "tests"]
required-imports = ["from __future__ import annotations"]
combine-as-imports = true

# TODO: for now, https://github.com/charliermarsh/ruff/issues/1817
known-third-party = [
 "asana",
 "atlassian",
 "celery",
 "cloudant",
 "databricks",
 "datadog",
 "docker",
 "elasticsearch",
 "github",
 "google",
 "grpc",
 "jenkins",
 "mysql",
 "neo4j",
 "papermill",
 "redis",
 "sendgrid",
 "snowflake",
 "telegram",
 "trino",
]

[tool.ruff.per-file-ignores]
"airflow/models/__init__.py" = ["F401"]
"airflow/models/sqla_models.py" = ["F401"]


# The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
# needed for the test to work
"tests/decorators/test_python.py" = ["I002"]
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Replace Ruff's known-third-party setting with namespace-packages#32873

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

uranusjr
merged 1 commit into

from

Jul 27, 2023

+36

−53

 Merged


Replace Ruff's known-third-party setting with namespace-packages#32873
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 3 additions & 4 deletions
airflow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,8 +49,7 @@
# very easily cause import cycles in the conf init/validate code (since downstream code from
# those functions likely import settings).
# configuration is therefore initted early here, simply by importing it.
from airflow import configuration
from airflow import settings
from airflow import configuration, settings

__all__ = ["__version__", "login", "DAG", "PY36", "PY37", "PY38", "PY39", "PY310", "XComArg"]

Expand DownExpand Up@@ -125,7 +124,7 @@ def __getattr__(name: str):
STATICA_HACK = True
globals()["kcah_acitats"[::-1].upper()] = False
if STATICA_HACK: # pragma: no cover
 from airflow.models.dag import DAG
 from airflow.models.xcom_arg import XComArg
 from airflow.exceptions import AirflowException
 from airflow.models.dag import DAG
 from airflow.models.dataset import Dataset
 from airflow.models.xcom_arg import XComArg

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/hooks/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,9 +21,10 @@
from typing import Any, Iterable

import apprise
from airflow.hooks.base import BaseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType
@charliermarshcharliermarshJul 27, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an example: these apprise imports were incorrectly marked as first-party, since apprise was missing from the known-third-party.

from airflow.hooks.base import BaseHook


class AppriseHook(BaseHook):
 """
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/notifications/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,9 +29,10 @@
 "Failed to import BaseNotifier. This feature is only available in Airflow versions >= 2.6.0"
 )

from airflow.providers.apprise.hooks.apprise import AppriseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType

from airflow.providers.apprise.hooks.apprise import AppriseHook


class AppriseNotifier(BaseNotifier):
 """
Expand Down

4 changes: 2 additions & 2 deletions
airflow/providers/openlineage/extractors/base.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,11 @@
from abc import ABC, abstractmethod

from attrs import Factory, define
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset

from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.state import TaskInstanceState
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset


@define
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

from __future__ import annotations

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/manager.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -173,9 +173,10 @@ def extract_inlets_and_outlets(

 @staticmethod
 def convert_to_ol_dataset(obj):
 from airflow.lineage.entities import Table
 from openlineage.client.run import Dataset

 from airflow.lineage.entities import Table

 if isinstance(obj, Dataset):
 return obj
 elif isinstance(obj, Table):
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/python.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,13 +20,14 @@
import inspect
from typing import Callable

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

12 changes: 6 additions & 6 deletions
airflow/providers/openlineage/plugins/adapter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,6 @@

import requests.exceptions
import yaml

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin
from openlineage.client import OpenLineageClient, set_producer
from openlineage.client.facet import (
 BaseFacet,
Expand All@@ -42,6 +36,12 @@
)
from openlineage.client.run import Job, Run, RunEvent, RunState

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin

if TYPE_CHECKING:
 from airflow.models.dagrun import DagRun
 from airflow.utils.log.secrets_masker import SecretsMasker
Expand Down

1 change: 0 additions & 1 deletion
airflow/providers/openlineage/plugins/facets.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,7 +17,6 @@
from __future__ import annotations

from attrs import define

from openlineage.client.facet import BaseFacet
from openlineage.client.utils import RedactMixin

Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/sqlparser.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,9 @@

import sqlparse
from attrs import define
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

from airflow.providers.openlineage.extractors.base import OperatorLineage
from airflow.providers.openlineage.utils.sql import (
Expand All@@ -28,9 +31,6 @@
 get_table_schemas,
)
from airflow.typing_compat import TypedDict
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

3 changes: 1 addition & 2 deletions
airflow/providers/openlineage/utils/sql.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,9 @@
from typing import TYPE_CHECKING, Dict, List, Optional

from attrs import define
from sqlalchemy import Column, MetaData, Table, and_, union_all

from openlineage.client.facet import SchemaDatasetFacet, SchemaField
from openlineage.client.run import Dataset
from sqlalchemy import Column, MetaData, Table, and_, union_all

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/utils/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@
import attrs
from attrs import asdict

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

from airflow.compat.functools import cache
from airflow.configuration import conf
from airflow.providers.openlineage.plugins.facets import (
Expand All@@ -37,9 +40,6 @@
)
from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

if TYPE_CHECKING:
 from airflow.models import DAG, BaseOperator, Connection, DagRun, TaskInstance

Expand Down

4 changes: 4 additions & 0 deletions
dev/breeze/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,3 +33,7 @@ python_files = [
testpaths = [
 "tests",
]

[tool.ruff]
extend = "../../pyproject.toml"
src = ["src"]

31 changes: 4 additions & 27 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,8 @@ extend-exclude = [
 # The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can
 # ignore them in ruff
 "airflow/providers/common/sql/*/*.pyi",
 "airflow/migrations/versions/*.py"
 "airflow/migrations/versions/*.py",
 "tests/dags/test_imports.py",
]

extend-select = [
Expand DownExpand Up@@ -73,6 +74,8 @@ extend-ignore = [
 "E731",
]

namespace-packages = ["airflow/providers"]

[tool.pytest.ini_options]
# * Disable `flaky` plugin for pytest. This plugin conflicts with `rerunfailures` because provide same marker.
# * Disable `nose` builtin plugin for pytest. This feature deprecated in 7.2 and will be removed in pytest>=8
Expand DownExpand Up@@ -106,39 +109,13 @@ testpaths = [
]

[tool.ruff.isort]
known-first-party = ["airflow", "airflow_breeze", "docker_tests", "docs", "kubernetes_tests", "tests"]
required-imports = ["from __future__ import annotations"]
combine-as-imports = true

# TODO: for now, https://github.com/charliermarsh/ruff/issues/1817
known-third-party = [
 "asana",
 "atlassian",
 "celery",
 "cloudant",
 "databricks",
 "datadog",
 "docker",
 "elasticsearch",
 "github",
 "google",
 "grpc",
 "jenkins",
 "mysql",
 "neo4j",
 "papermill",
 "redis",
 "sendgrid",
 "snowflake",
 "telegram",
 "trino",
]

[tool.ruff.per-file-ignores]
"airflow/models/__init__.py" = ["F401"]
"airflow/models/sqla_models.py" = ["F401"]


# The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
# needed for the test to work
"tests/decorators/test_python.py" = ["I002"]
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Replace Ruff's known-third-party setting with namespace-packages#32873

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

uranusjr
merged 1 commit into

from

Jul 27, 2023

+36

−53

 Merged


Replace Ruff's known-third-party setting with namespace-packages#32873
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 3 additions & 4 deletions
airflow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,8 +49,7 @@
# very easily cause import cycles in the conf init/validate code (since downstream code from
# those functions likely import settings).
# configuration is therefore initted early here, simply by importing it.
from airflow import configuration
from airflow import settings
from airflow import configuration, settings

__all__ = ["__version__", "login", "DAG", "PY36", "PY37", "PY38", "PY39", "PY310", "XComArg"]

Expand DownExpand Up@@ -125,7 +124,7 @@ def __getattr__(name: str):
STATICA_HACK = True
globals()["kcah_acitats"[::-1].upper()] = False
if STATICA_HACK: # pragma: no cover
 from airflow.models.dag import DAG
 from airflow.models.xcom_arg import XComArg
 from airflow.exceptions import AirflowException
 from airflow.models.dag import DAG
 from airflow.models.dataset import Dataset
 from airflow.models.xcom_arg import XComArg

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/hooks/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,9 +21,10 @@
from typing import Any, Iterable

import apprise
from airflow.hooks.base import BaseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType
@charliermarshcharliermarshJul 27, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an example: these apprise imports were incorrectly marked as first-party, since apprise was missing from the known-third-party.

from airflow.hooks.base import BaseHook


class AppriseHook(BaseHook):
 """
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/notifications/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,9 +29,10 @@
 "Failed to import BaseNotifier. This feature is only available in Airflow versions >= 2.6.0"
 )

from airflow.providers.apprise.hooks.apprise import AppriseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType

from airflow.providers.apprise.hooks.apprise import AppriseHook


class AppriseNotifier(BaseNotifier):
 """
Expand Down

4 changes: 2 additions & 2 deletions
airflow/providers/openlineage/extractors/base.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,11 @@
from abc import ABC, abstractmethod

from attrs import Factory, define
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset

from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.state import TaskInstanceState
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset


@define
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

from __future__ import annotations

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/manager.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -173,9 +173,10 @@ def extract_inlets_and_outlets(

 @staticmethod
 def convert_to_ol_dataset(obj):
 from airflow.lineage.entities import Table
 from openlineage.client.run import Dataset

 from airflow.lineage.entities import Table

 if isinstance(obj, Dataset):
 return obj
 elif isinstance(obj, Table):
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/python.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,13 +20,14 @@
import inspect
from typing import Callable

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

12 changes: 6 additions & 6 deletions
airflow/providers/openlineage/plugins/adapter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,6 @@

import requests.exceptions
import yaml

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin
from openlineage.client import OpenLineageClient, set_producer
from openlineage.client.facet import (
 BaseFacet,
Expand All@@ -42,6 +36,12 @@
)
from openlineage.client.run import Job, Run, RunEvent, RunState

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin

if TYPE_CHECKING:
 from airflow.models.dagrun import DagRun
 from airflow.utils.log.secrets_masker import SecretsMasker
Expand Down

1 change: 0 additions & 1 deletion
airflow/providers/openlineage/plugins/facets.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,7 +17,6 @@
from __future__ import annotations

from attrs import define

from openlineage.client.facet import BaseFacet
from openlineage.client.utils import RedactMixin

Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/sqlparser.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,9 @@

import sqlparse
from attrs import define
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

from airflow.providers.openlineage.extractors.base import OperatorLineage
from airflow.providers.openlineage.utils.sql import (
Expand All@@ -28,9 +31,6 @@
 get_table_schemas,
)
from airflow.typing_compat import TypedDict
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

3 changes: 1 addition & 2 deletions
airflow/providers/openlineage/utils/sql.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,9 @@
from typing import TYPE_CHECKING, Dict, List, Optional

from attrs import define
from sqlalchemy import Column, MetaData, Table, and_, union_all

from openlineage.client.facet import SchemaDatasetFacet, SchemaField
from openlineage.client.run import Dataset
from sqlalchemy import Column, MetaData, Table, and_, union_all

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/utils/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@
import attrs
from attrs import asdict

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

from airflow.compat.functools import cache
from airflow.configuration import conf
from airflow.providers.openlineage.plugins.facets import (
Expand All@@ -37,9 +40,6 @@
)
from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

if TYPE_CHECKING:
 from airflow.models import DAG, BaseOperator, Connection, DagRun, TaskInstance

Expand Down

4 changes: 4 additions & 0 deletions
dev/breeze/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,3 +33,7 @@ python_files = [
testpaths = [
 "tests",
]

[tool.ruff]
extend = "../../pyproject.toml"
src = ["src"]

31 changes: 4 additions & 27 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,8 @@ extend-exclude = [
 # The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can
 # ignore them in ruff
 "airflow/providers/common/sql/*/*.pyi",
 "airflow/migrations/versions/*.py"
 "airflow/migrations/versions/*.py",
 "tests/dags/test_imports.py",
]

extend-select = [
Expand DownExpand Up@@ -73,6 +74,8 @@ extend-ignore = [
 "E731",
]

namespace-packages = ["airflow/providers"]

[tool.pytest.ini_options]
# * Disable `flaky` plugin for pytest. This plugin conflicts with `rerunfailures` because provide same marker.
# * Disable `nose` builtin plugin for pytest. This feature deprecated in 7.2 and will be removed in pytest>=8
Expand DownExpand Up@@ -106,39 +109,13 @@ testpaths = [
]

[tool.ruff.isort]
known-first-party = ["airflow", "airflow_breeze", "docker_tests", "docs", "kubernetes_tests", "tests"]
required-imports = ["from __future__ import annotations"]
combine-as-imports = true

# TODO: for now, https://github.com/charliermarsh/ruff/issues/1817
known-third-party = [
 "asana",
 "atlassian",
 "celery",
 "cloudant",
 "databricks",
 "datadog",
 "docker",
 "elasticsearch",
 "github",
 "google",
 "grpc",
 "jenkins",
 "mysql",
 "neo4j",
 "papermill",
 "redis",
 "sendgrid",
 "snowflake",
 "telegram",
 "trino",
]

[tool.ruff.per-file-ignores]
"airflow/models/__init__.py" = ["F401"]
"airflow/models/sqla_models.py" = ["F401"]


# The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
# needed for the test to work
"tests/decorators/test_python.py" = ["I002"]
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Replace Ruff's known-third-party setting with namespace-packages#32873

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

uranusjr
merged 1 commit into

from

Jul 27, 2023

+36

−53

 Merged


Replace Ruff's known-third-party setting with namespace-packages#32873
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


7 changes: 3 additions & 4 deletions
airflow/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -49,8 +49,7 @@
# very easily cause import cycles in the conf init/validate code (since downstream code from
# those functions likely import settings).
# configuration is therefore initted early here, simply by importing it.
from airflow import configuration
from airflow import settings
from airflow import configuration, settings

__all__ = ["__version__", "login", "DAG", "PY36", "PY37", "PY38", "PY39", "PY310", "XComArg"]

Expand DownExpand Up@@ -125,7 +124,7 @@ def __getattr__(name: str):
STATICA_HACK = True
globals()["kcah_acitats"[::-1].upper()] = False
if STATICA_HACK: # pragma: no cover
 from airflow.models.dag import DAG
 from airflow.models.xcom_arg import XComArg
 from airflow.exceptions import AirflowException
 from airflow.models.dag import DAG
 from airflow.models.dataset import Dataset
 from airflow.models.xcom_arg import XComArg

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/hooks/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -21,9 +21,10 @@
from typing import Any, Iterable

import apprise
from airflow.hooks.base import BaseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType
@charliermarshcharliermarshJul 27, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an example: these apprise imports were incorrectly marked as first-party, since apprise was missing from the known-third-party.

from airflow.hooks.base import BaseHook


class AppriseHook(BaseHook):
 """
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/apprise/notifications/apprise.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,9 +29,10 @@
 "Failed to import BaseNotifier. This feature is only available in Airflow versions >= 2.6.0"
 )

from airflow.providers.apprise.hooks.apprise import AppriseHook
from apprise import AppriseConfig, NotifyFormat, NotifyType

from airflow.providers.apprise.hooks.apprise import AppriseHook


class AppriseNotifier(BaseNotifier):
 """
Expand Down

4 changes: 2 additions & 2 deletions
airflow/providers/openlineage/extractors/base.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,11 @@
from abc import ABC, abstractmethod

from attrs import Factory, define
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset

from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.state import TaskInstanceState
from openlineage.client.facet import BaseFacet
from openlineage.client.run import Dataset


@define
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/bash.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,13 +17,14 @@

from __future__ import annotations

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/manager.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -173,9 +173,10 @@ def extract_inlets_and_outlets(

 @staticmethod
 def convert_to_ol_dataset(obj):
 from airflow.lineage.entities import Table
 from openlineage.client.run import Dataset

 from airflow.lineage.entities import Table

 if isinstance(obj, Dataset):
 return obj
 elif isinstance(obj, Table):
Expand Down

3 changes: 2 additions & 1 deletion
airflow/providers/openlineage/extractors/python.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,13 +20,14 @@
import inspect
from typing import Callable

from openlineage.client.facet import SourceCodeJobFacet

from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
from airflow.providers.openlineage.plugins.facets import (
 UnknownOperatorAttributeRunFacet,
 UnknownOperatorInstance,
)
from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys, is_source_enabled
from openlineage.client.facet import SourceCodeJobFacet

"""
:meta private:
Expand Down

12 changes: 6 additions & 6 deletions
airflow/providers/openlineage/plugins/adapter.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,12 +22,6 @@

import requests.exceptions
import yaml

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin
from openlineage.client import OpenLineageClient, set_producer
from openlineage.client.facet import (
 BaseFacet,
Expand All@@ -42,6 +36,12 @@
)
from openlineage.client.run import Job, Run, RunEvent, RunState

from airflow.configuration import conf
from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
from airflow.utils.log.logging_mixin import LoggingMixin

if TYPE_CHECKING:
 from airflow.models.dagrun import DagRun
 from airflow.utils.log.secrets_masker import SecretsMasker
Expand Down

1 change: 0 additions & 1 deletion
airflow/providers/openlineage/plugins/facets.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,7 +17,6 @@
from __future__ import annotations

from attrs import define

from openlineage.client.facet import BaseFacet
from openlineage.client.utils import RedactMixin

Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/sqlparser.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,9 @@

import sqlparse
from attrs import define
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

from airflow.providers.openlineage.extractors.base import OperatorLineage
from airflow.providers.openlineage.utils.sql import (
Expand All@@ -28,9 +31,6 @@
 get_table_schemas,
)
from airflow.typing_compat import TypedDict
from openlineage.client.facet import BaseFacet, ExtractionError, ExtractionErrorRunFacet, SqlJobFacet
from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta, SqlMeta, parse

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

3 changes: 1 addition & 2 deletions
airflow/providers/openlineage/utils/sql.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,10 +23,9 @@
from typing import TYPE_CHECKING, Dict, List, Optional

from attrs import define
from sqlalchemy import Column, MetaData, Table, and_, union_all

from openlineage.client.facet import SchemaDatasetFacet, SchemaField
from openlineage.client.run import Dataset
from sqlalchemy import Column, MetaData, Table, and_, union_all

if TYPE_CHECKING:
 from sqlalchemy.engine import Engine
Expand Down

6 changes: 3 additions & 3 deletions
airflow/providers/openlineage/utils/utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,9 @@
import attrs
from attrs import asdict

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

from airflow.compat.functools import cache
from airflow.configuration import conf
from airflow.providers.openlineage.plugins.facets import (
Expand All@@ -37,9 +40,6 @@
)
from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key

# TODO: move this maybe to Airflow's logic?
from openlineage.client.utils import RedactMixin

if TYPE_CHECKING:
 from airflow.models import DAG, BaseOperator, Connection, DagRun, TaskInstance

Expand Down

4 changes: 4 additions & 0 deletions
dev/breeze/pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,3 +33,7 @@ python_files = [
testpaths = [
 "tests",
]

[tool.ruff]
extend = "../../pyproject.toml"
src = ["src"]

31 changes: 4 additions & 27 deletions
pyproject.toml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,7 +41,8 @@ extend-exclude = [
 # The files generated by stubgen aren't 100% valid syntax it turns out, and we don't ship them, so we can
 # ignore them in ruff
 "airflow/providers/common/sql/*/*.pyi",
 "airflow/migrations/versions/*.py"
 "airflow/migrations/versions/*.py",
 "tests/dags/test_imports.py",
]

extend-select = [
Expand DownExpand Up@@ -73,6 +74,8 @@ extend-ignore = [
 "E731",
]

namespace-packages = ["airflow/providers"]

[tool.pytest.ini_options]
# * Disable `flaky` plugin for pytest. This plugin conflicts with `rerunfailures` because provide same marker.
# * Disable `nose` builtin plugin for pytest. This feature deprecated in 7.2 and will be removed in pytest>=8
Expand DownExpand Up@@ -106,39 +109,13 @@ testpaths = [
]

[tool.ruff.isort]
known-first-party = ["airflow", "airflow_breeze", "docker_tests", "docs", "kubernetes_tests", "tests"]
required-imports = ["from __future__ import annotations"]
combine-as-imports = true

# TODO: for now, https://github.com/charliermarsh/ruff/issues/1817
known-third-party = [
 "asana",
 "atlassian",
 "celery",
 "cloudant",
 "databricks",
 "datadog",
 "docker",
 "elasticsearch",
 "github",
 "google",
 "grpc",
 "jenkins",
 "mysql",
 "neo4j",
 "papermill",
 "redis",
 "sendgrid",
 "snowflake",
 "telegram",
 "trino",
]

[tool.ruff.per-file-ignores]
"airflow/models/__init__.py" = ["F401"]
"airflow/models/sqla_models.py" = ["F401"]


# The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
# needed for the test to work
"tests/decorators/test_python.py" = ["I002"]
Expand Down