Optimize DAG serialization by excluding schema default values#55849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kaxil merged 2 commits into from
Sep 18, 2025
Merged

Optimize DAG serialization by excluding schema default values #55849

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
589a4e6
Optimize DAG serialization by excluding schema default values
kaxil Sep 18, 2025
a8fd7ea
fixup! Optimize DAG serialization by excluding schema default values
kaxil Sep 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions airflow-core/src/airflow/serialization/schema.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -175,8 +175,8 @@
"value": { "$ref": "#/definitions/dict" }
}
},
"catchup": { "type": "boolean" },
"fail_fast": { "type": "boolean" },
"catchup": { "type": "boolean", "default": false },
"fail_fast": { "type": "boolean", "default": false },
"fileloc": { "type" : "string"},
"relative_fileloc": { "type" : "string"},
"_processor_dags_folder": {
Expand All@@ -198,27 +198,27 @@
]
},
"_concurrency": { "type" : "number"},
"max_active_tasks": { "type" : "number"},
"max_active_runs": { "type" : "number"},
"max_consecutive_failed_dag_runs": { "type" : "number"},
"max_active_tasks": { "type" : "number", "default": 16},
"max_active_runs": { "type" : "number", "default": 16},
"max_consecutive_failed_dag_runs": { "type" : "number", "default": 0},
"default_args": { "$ref": "#/definitions/dict" },
"start_date": { "$ref": "#/definitions/datetime" },
"end_date": { "$ref": "#/definitions/datetime" },
"dagrun_timeout": { "$ref": "#/definitions/timedelta" },
"doc_md": { "type" : "string"},
"access_control": {"$ref": "#/definitions/dict" },
"is_paused_upon_creation": { "type": "boolean" },
"has_on_success_callback": { "type": "boolean" },
"has_on_failure_callback": { "type": "boolean" },
"render_template_as_native_obj": { "type": "boolean" },
"has_on_success_callback": { "type": "boolean", "default": false },
"has_on_failure_callback": { "type": "boolean", "default": false },
"render_template_as_native_obj": { "type": "boolean", "default": false },
"tags": { "type": "array" },
"task_group": {"anyOf": [
{ "type": "null" },
{ "$ref": "#/definitions/task_group" }
]},
"edge_info": { "$ref": "#/definitions/edge_info" },
"dag_dependencies": { "$ref": "#/definitions/dag_dependencies" },
"disable_bundle_versioning": {"type": "boolean"}
"disable_bundle_versioning": {"type": "boolean", "default": false }
},
"required": [
"dag_id",
Expand Down
80 changes: 55 additions & 25 deletions airflow-core/src/airflow/serialization/serialized_objects.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1094,21 +1094,7 @@ def _deserialize_params_dict(cls, encoded_params: list[tuple[str, dict]]) -> Par
return ParamsDict(op_params)

@classmethod
Comment thread
kaxil marked this conversation as resolved.
def get_operator_optional_fields_from_schema(cls) -> set[str]:
schema_loader = cls._json_schema

if schema_loader is None:
return set()

schema_data = schema_loader.schema
operator_def = schema_data.get("definitions", {}).get("operator", {})
operator_fields = set(operator_def.get("properties", {}).keys())
required_fields = set(operator_def.get("required", []))

optional_fields = operator_fields - required_fields
return optional_fields

@classmethod
@lru_cache(maxsize=4) # Cache for "operator", "dag", and a few others
def get_schema_defaults(cls, object_type: str) -> dict[str, Any]:
"""
Extract default values from JSON schema for any object type.
Expand DownExpand Up@@ -1713,6 +1699,22 @@ def set_task_dag_references(task: SerializedOperator | MappedOperator, dag: Seri
# Bypass set_upstream etc here - it does more than we want
dag.task_dict[task_id].upstream_task_ids.add(task.task_id)

@classmethod
@lru_cache(maxsize=1) # Only one type: "operator"
def get_operator_optional_fields_from_schema(cls) -> set[str]:
schema_loader = cls._json_schema

if schema_loader is None:
return set()

schema_data = schema_loader.schema
operator_def = schema_data.get("definitions", {}).get("operator", {})
operator_fields = set(operator_def.get("properties", {}).keys())
required_fields = set(operator_def.get("required", []))

optional_fields = operator_fields - required_fields
return optional_fields

@classmethod
def deserialize_operator(
cls,
Expand DownExpand Up@@ -1814,7 +1816,7 @@ def detect_dependencies(cls, op: SdkOperator) -> set[DagDependency]:
return deps

@classmethod
def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:
def _matches_client_defaults(cls, var: Any, attrname: str) -> bool:
"""
Check if a field value matches client_defaults and should be excluded.

Expand All@@ -1823,7 +1825,6 @@ def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:

:param var: The value to check
:param attrname: The attribute name
:param op: The operator instance
:return: True if value matches client_defaults and should be excluded
"""
try:
Expand DownExpand Up@@ -1851,7 +1852,7 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
:return: True if a variable is excluded, False otherwise.
"""
# Check if value matches client_defaults (hierarchical defaults optimization)
if cls._matches_client_defaults(var, attrname, op):
if cls._matches_client_defaults(var, attrname):
return True
schema_defaults = cls.get_schema_defaults("operator")

Expand DownExpand Up@@ -2384,25 +2385,23 @@ class SerializedDAG(BaseSerialization):
_processor_dags_folder: str

def __init__(self, *, dag_id: str) -> None:
self.catchup = airflow_conf.getboolean("scheduler", "catchup_by_default")
self.catchup = False # Schema default
self.dag_id = self.dag_display_name = dag_id
self.dagrun_timeout = None
self.deadline = None
self.default_args = {}
self.description = None
self.disable_bundle_versioning = airflow_conf.getboolean("dag_processor", "disable_bundle_versioning")
self.disable_bundle_versioning = False
self.doc_md = None
self.edge_info = {}
self.end_date = None
self.fail_fast = False
self.has_on_failure_callback = False
self.has_on_success_callback = False
self.is_paused_upon_creation = None
self.max_active_runs = airflow_conf.getint("core", "max_active_runs_per_dag")
Comment thread
kaxil marked this conversation as resolved.
self.max_active_tasks = airflow_conf.getint("core", "max_active_tasks_per_dag")
self.max_consecutive_failed_dag_runs = airflow_conf.getint(
"core", "max_consecutive_failed_dag_runs_per_dag"
)
self.max_active_runs = 16 # Schema default
self.max_active_tasks = 16 # Schema default
self.max_consecutive_failed_dag_runs = 0 # Schema default
self.owner_links = {}
self.params = ParamsDict()
self.partial = False
Expand DownExpand Up@@ -2624,8 +2623,38 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
return False
if attrname == "dag_display_name" and var == op.dag_id:
return True

# DAG schema defaults exclusion (same pattern as SerializedBaseOperator)
dag_schema_defaults = cls.get_schema_defaults("dag")
if attrname in dag_schema_defaults:
if dag_schema_defaults[attrname] == var:
return True

optional_fields = cls.get_dag_optional_fields_from_schema()
if var is None:
return True
if attrname in optional_fields:
if var in [[], (), set(), {}]:
return True

return super()._is_excluded(var, attrname, op)

@classmethod
@lru_cache(maxsize=1) # Only one type: "dag"
def get_dag_optional_fields_from_schema(cls) -> set[str]:
schema_loader = cls._json_schema

if schema_loader is None:
return set()

schema_data = schema_loader.schema
operator_def = schema_data.get("definitions", {}).get("dag", {})
operator_fields = set(operator_def.get("properties", {}).keys())
required_fields = set(operator_def.get("required", []))

optional_fields = operator_fields - required_fields
return optional_fields

@classmethod
def to_dict(cls, var: Any) -> dict:
"""Stringifies DAGs and operators contained by var and returns a dict of var."""
Expand DownExpand Up@@ -3798,6 +3827,7 @@ class LazyDeserializedDAG(pydantic.BaseModel):
"dag_display_name",
"has_on_success_callback",
"has_on_failure_callback",
"tags",
# Attr properties that are nullable, or have a default that loads from config
"description",
"start_date",
Expand Down
110 changes: 94 additions & 16 deletions airflow-core/tests/unit/serialization/test_dag_serialization.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -152,13 +152,8 @@
"downstream_task_ids": [],
},
"is_paused_upon_creation": False,
"max_active_runs": 16,
"max_active_tasks": 16,
"max_consecutive_failed_dag_runs": 0,
"dag_id": "simple_dag",
"deadline": None,
"catchup": False,
"disable_bundle_versioning": False,
"doc_md": "### DAG Tutorial Documentation",
"fileloc": None,
"_processor_dags_folder": (
Expand DownExpand Up@@ -269,7 +264,6 @@
},
],
"params": [],
"tags": [],
},
}

Expand DownExpand Up@@ -2177,7 +2171,8 @@ def test_dag_disable_bundle_versioning_roundtrip(self, dag_arg, conf_arg, expect
"""
with conf_vars({("dag_processor", "disable_bundle_versioning"): conf_arg}):
kwargs = {}
kwargs["disable_bundle_versioning"] = dag_arg
if dag_arg is not None:
kwargs["disable_bundle_versioning"] = dag_arg
dag = DAG(
dag_id="test_dag_disable_bundle_versioning_roundtrip",
schedule=None,
Expand DownExpand Up@@ -3299,17 +3294,34 @@ def test_handle_v1_serdag():
SerializedDAG.conversion_v1_to_v2(v1)
SerializedDAG.conversion_v2_to_v3(v1)

# Update a few subtle differences
v1["dag"]["tags"] = []
v1["dag"]["catchup"] = False
v1["dag"]["disable_bundle_versioning"] = False
dag = SerializedDAG.from_dict(v1)

expected = copy.deepcopy(serialized_simple_dag_ground_truth)
expected["dag"]["dag_dependencies"] = expected_dag_dependencies
del expected["dag"]["tasks"][1]["__var"]["_operator_extra_links"]
expected_sdag = copy.deepcopy(serialized_simple_dag_ground_truth)
expected = SerializedDAG.from_dict(expected_sdag)

fields_to_verify = set(vars(expected).keys()) - {
"task_group", # Tested separately
"dag_dependencies", # Tested separately
"last_loaded", # Dynamically set to utcnow
}

for f in fields_to_verify:
dag_value = getattr(dag, f)
expected_value = getattr(expected, f)

assert dag_value == expected_value, (
f"V2 DAG field '{f}' differs from V3: V2={dag_value!r} != V3={expected_value!r}"
)

for f in set(vars(expected.task_group).keys()) - {"dag"}:
dag_tg_value = getattr(dag.task_group, f)
expected_tg_value = getattr(expected.task_group, f)

assert dag_tg_value == expected_tg_value, (
f"V2 task_group field '{f}' differs: V2={dag_tg_value!r} != V3={expected_tg_value!r}"
)

del expected["client_defaults"]
assert v1 == expected
assert getattr(dag, "dag_dependencies") == expected_dag_dependencies


def test_handle_v2_serdag():
Expand DownExpand Up@@ -3514,6 +3526,72 @@ def test_handle_v2_serdag():
)


def test_dag_schema_defaults_optimization():
"""Test that DAG fields matching schema defaults are excluded from serialization."""

# Create DAG with all schema default values
dag_with_defaults = DAG(
dag_id="test_defaults_dag",
start_date=datetime(2023, 1, 1),
# These should match schema defaults and be excluded
catchup=False,
fail_fast=False,
max_active_runs=16,
max_active_tasks=16,
max_consecutive_failed_dag_runs=0,
render_template_as_native_obj=False,
disable_bundle_versioning=False,
# These should be excluded as None
description=None,
doc_md=None,
)

# Serialize and check exclusions
serialized = SerializedDAG.to_dict(dag_with_defaults)
dag_data = serialized["dag"]

# Schema default fields should be excluded
for field in SerializedDAG.get_schema_defaults("dag").keys():
assert field not in dag_data, f"Schema default field '{field}' should be excluded"

# None fields should also be excluded
none_fields = ["description", "doc_md"]
for field in none_fields:
assert field not in dag_data, f"None field '{field}' should be excluded"

# Test deserialization restores defaults correctly
deserialized_dag = SerializedDAG.from_dict(serialized)

# Verify schema defaults are restored
assert deserialized_dag.catchup is False
assert deserialized_dag.fail_fast is False
assert deserialized_dag.max_active_runs == 16
assert deserialized_dag.max_active_tasks == 16
assert deserialized_dag.max_consecutive_failed_dag_runs == 0
assert deserialized_dag.render_template_as_native_obj is False
assert deserialized_dag.disable_bundle_versioning is False

# Test with non-default values (should be included)
dag_non_defaults = DAG(
dag_id="test_non_defaults_dag",
start_date=datetime(2023, 1, 1),
catchup=True, # Non-default
max_active_runs=32, # Non-default
description="Test description", # Non-None
)

serialized_non_defaults = SerializedDAG.to_dict(dag_non_defaults)
dag_non_defaults_data = serialized_non_defaults["dag"]

# Non-default values should be included
assert "catchup" in dag_non_defaults_data
assert dag_non_defaults_data["catchup"] is True
assert "max_active_runs" in dag_non_defaults_data
assert dag_non_defaults_data["max_active_runs"] == 32
assert "description" in dag_non_defaults_data
assert dag_non_defaults_data["description"] == "Test description"


def test_email_optimization_removes_email_attrs_when_email_empty():
"""Test that email_on_failure and email_on_retry are removed when email is empty."""
with DAG(dag_id="test_email_optimization") as dag:
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Optimize DAG serialization by excluding schema default values#55849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

kaxil
merged 2 commits into

from

Sep 18, 2025
 Merged


Optimize DAG serialization by excluding schema default values
#55849
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
589a4e6

Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025
a8fd7ea

fixup! Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 9 additions & 9 deletions
airflow-core/src/airflow/serialization/schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,8 +175,8 @@
 "value": { "$ref": "#/definitions/dict" }
 }
 },
 "catchup": { "type": "boolean" },
 "fail_fast": { "type": "boolean" },
 "catchup": { "type": "boolean", "default": false },
 "fail_fast": { "type": "boolean", "default": false },
 "fileloc": { "type" : "string"},
 "relative_fileloc": { "type" : "string"},
 "_processor_dags_folder": {
Expand All@@ -198,27 +198,27 @@
 ]
 },
 "_concurrency": { "type" : "number"},
 "max_active_tasks": { "type" : "number"},
 "max_active_runs": { "type" : "number"},
 "max_consecutive_failed_dag_runs": { "type" : "number"},
 "max_active_tasks": { "type" : "number", "default": 16},
 "max_active_runs": { "type" : "number", "default": 16},
 "max_consecutive_failed_dag_runs": { "type" : "number", "default": 0},
 "default_args": { "$ref": "#/definitions/dict" },
 "start_date": { "$ref": "#/definitions/datetime" },
 "end_date": { "$ref": "#/definitions/datetime" },
 "dagrun_timeout": { "$ref": "#/definitions/timedelta" },
 "doc_md": { "type" : "string"},
 "access_control": {"$ref": "#/definitions/dict" },
 "is_paused_upon_creation": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean" },
 "has_on_failure_callback": { "type": "boolean" },
 "render_template_as_native_obj": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean", "default": false },
 "has_on_failure_callback": { "type": "boolean", "default": false },
 "render_template_as_native_obj": { "type": "boolean", "default": false },
 "tags": { "type": "array" },
 "task_group": {"anyOf": [
 { "type": "null" },
 { "$ref": "#/definitions/task_group" }
 ]},
 "edge_info": { "$ref": "#/definitions/edge_info" },
 "dag_dependencies": { "$ref": "#/definitions/dag_dependencies" },
 "disable_bundle_versioning": {"type": "boolean"}
 "disable_bundle_versioning": {"type": "boolean", "default": false }
 },
 "required": [
 "dag_id",
Expand Down

80 changes: 55 additions & 25 deletions
airflow-core/src/airflow/serialization/serialized_objects.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1094,21 +1094,7 @@ def _deserialize_params_dict(cls, encoded_params: list[tuple[str, dict]]) -> Par
 return ParamsDict(op_params)

 @classmethod
Comment thread
kaxil marked this conversation as resolved.

 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 @lru_cache(maxsize=4) # Cache for "operator", "dag", and a few others
 def get_schema_defaults(cls, object_type: str) -> dict[str, Any]:
 """
 Extract default values from JSON schema for any object type.
Expand DownExpand Up@@ -1713,6 +1699,22 @@ def set_task_dag_references(task: SerializedOperator | MappedOperator, dag: Seri
 # Bypass set_upstream etc here - it does more than we want
 dag.task_dict[task_id].upstream_task_ids.add(task.task_id)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "operator"
 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def deserialize_operator(
 cls,
Expand DownExpand Up@@ -1814,7 +1816,7 @@ def detect_dependencies(cls, op: SdkOperator) -> set[DagDependency]:
 return deps

 @classmethod
 def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:
 def _matches_client_defaults(cls, var: Any, attrname: str) -> bool:
 """
 Check if a field value matches client_defaults and should be excluded.

Expand All@@ -1823,7 +1825,6 @@ def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:

 :param var: The value to check
 :param attrname: The attribute name
 :param op: The operator instance
 :return: True if value matches client_defaults and should be excluded
 """
 try:
Expand DownExpand Up@@ -1851,7 +1852,7 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 :return: True if a variable is excluded, False otherwise.
 """
 # Check if value matches client_defaults (hierarchical defaults optimization)
 if cls._matches_client_defaults(var, attrname, op):
 if cls._matches_client_defaults(var, attrname):
 return True
 schema_defaults = cls.get_schema_defaults("operator")

Expand DownExpand Up@@ -2384,25 +2385,23 @@ class SerializedDAG(BaseSerialization):
 _processor_dags_folder: str

 def __init__(self, *, dag_id: str) -> None:
 self.catchup = airflow_conf.getboolean("scheduler", "catchup_by_default")
 self.catchup = False # Schema default
 self.dag_id = self.dag_display_name = dag_id
 self.dagrun_timeout = None
 self.deadline = None
 self.default_args = {}
 self.description = None
 self.disable_bundle_versioning = airflow_conf.getboolean("dag_processor", "disable_bundle_versioning")
 self.disable_bundle_versioning = False
 self.doc_md = None
 self.edge_info = {}
 self.end_date = None
 self.fail_fast = False
 self.has_on_failure_callback = False
 self.has_on_success_callback = False
 self.is_paused_upon_creation = None
 self.max_active_runs = airflow_conf.getint("core", "max_active_runs_per_dag")
Comment thread
kaxil marked this conversation as resolved.

 self.max_active_tasks = airflow_conf.getint("core", "max_active_tasks_per_dag")
 self.max_consecutive_failed_dag_runs = airflow_conf.getint(
 "core", "max_consecutive_failed_dag_runs_per_dag"
 )
 self.max_active_runs = 16 # Schema default
 self.max_active_tasks = 16 # Schema default
 self.max_consecutive_failed_dag_runs = 0 # Schema default
 self.owner_links = {}
 self.params = ParamsDict()
 self.partial = False
Expand DownExpand Up@@ -2624,8 +2623,38 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 return False
 if attrname == "dag_display_name" and var == op.dag_id:
 return True

 # DAG schema defaults exclusion (same pattern as SerializedBaseOperator)
 dag_schema_defaults = cls.get_schema_defaults("dag")
 if attrname in dag_schema_defaults:
 if dag_schema_defaults[attrname] == var:
 return True

 optional_fields = cls.get_dag_optional_fields_from_schema()
 if var is None:
 return True
 if attrname in optional_fields:
 if var in [[], (), set(), {}]:
 return True

 return super()._is_excluded(var, attrname, op)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "dag"
 def get_dag_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("dag", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def to_dict(cls, var: Any) -> dict:
 """Stringifies DAGs and operators contained by var and returns a dict of var."""
Expand DownExpand Up@@ -3798,6 +3827,7 @@ class LazyDeserializedDAG(pydantic.BaseModel):
 "dag_display_name",
 "has_on_success_callback",
 "has_on_failure_callback",
 "tags",
 # Attr properties that are nullable, or have a default that loads from config
 "description",
 "start_date",
Expand Down

110 changes: 94 additions & 16 deletions
airflow-core/tests/unit/serialization/test_dag_serialization.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,13 +152,8 @@
 "downstream_task_ids": [],
 },
 "is_paused_upon_creation": False,
 "max_active_runs": 16,
 "max_active_tasks": 16,
 "max_consecutive_failed_dag_runs": 0,
 "dag_id": "simple_dag",
 "deadline": None,
 "catchup": False,
 "disable_bundle_versioning": False,
 "doc_md": "### DAG Tutorial Documentation",
 "fileloc": None,
 "_processor_dags_folder": (
Expand DownExpand Up@@ -269,7 +264,6 @@
 },
 ],
 "params": [],
 "tags": [],
 },
}

Expand DownExpand Up@@ -2177,7 +2171,8 @@ def test_dag_disable_bundle_versioning_roundtrip(self, dag_arg, conf_arg, expect
 """
 with conf_vars({("dag_processor", "disable_bundle_versioning"): conf_arg}):
 kwargs = {}
 kwargs["disable_bundle_versioning"] = dag_arg
 if dag_arg is not None:
 kwargs["disable_bundle_versioning"] = dag_arg
 dag = DAG(
 dag_id="test_dag_disable_bundle_versioning_roundtrip",
 schedule=None,
Expand DownExpand Up@@ -3299,17 +3294,34 @@ def test_handle_v1_serdag():
 SerializedDAG.conversion_v1_to_v2(v1)
 SerializedDAG.conversion_v2_to_v3(v1)

 # Update a few subtle differences
 v1["dag"]["tags"] = []
 v1["dag"]["catchup"] = False
 v1["dag"]["disable_bundle_versioning"] = False
 dag = SerializedDAG.from_dict(v1)

 expected = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected["dag"]["dag_dependencies"] = expected_dag_dependencies
 del expected["dag"]["tasks"][1]["__var"]["_operator_extra_links"]
 expected_sdag = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected = SerializedDAG.from_dict(expected_sdag)

 fields_to_verify = set(vars(expected).keys()) - {
 "task_group", # Tested separately
 "dag_dependencies", # Tested separately
 "last_loaded", # Dynamically set to utcnow
 }

 for f in fields_to_verify:
 dag_value = getattr(dag, f)
 expected_value = getattr(expected, f)

 assert dag_value == expected_value, (
 f"V2 DAG field '{f}' differs from V3: V2={dag_value!r} != V3={expected_value!r}"
 )

 for f in set(vars(expected.task_group).keys()) - {"dag"}:
 dag_tg_value = getattr(dag.task_group, f)
 expected_tg_value = getattr(expected.task_group, f)

 assert dag_tg_value == expected_tg_value, (
 f"V2 task_group field '{f}' differs: V2={dag_tg_value!r} != V3={expected_tg_value!r}"
 )

 del expected["client_defaults"]
 assert v1 == expected
 assert getattr(dag, "dag_dependencies") == expected_dag_dependencies


def test_handle_v2_serdag():
Expand DownExpand Up@@ -3514,6 +3526,72 @@ def test_handle_v2_serdag():
 )


def test_dag_schema_defaults_optimization():
 """Test that DAG fields matching schema defaults are excluded from serialization."""

 # Create DAG with all schema default values
 dag_with_defaults = DAG(
 dag_id="test_defaults_dag",
 start_date=datetime(2023, 1, 1),
 # These should match schema defaults and be excluded
 catchup=False,
 fail_fast=False,
 max_active_runs=16,
 max_active_tasks=16,
 max_consecutive_failed_dag_runs=0,
 render_template_as_native_obj=False,
 disable_bundle_versioning=False,
 # These should be excluded as None
 description=None,
 doc_md=None,
 )

 # Serialize and check exclusions
 serialized = SerializedDAG.to_dict(dag_with_defaults)
 dag_data = serialized["dag"]

 # Schema default fields should be excluded
 for field in SerializedDAG.get_schema_defaults("dag").keys():
 assert field not in dag_data, f"Schema default field '{field}' should be excluded"

 # None fields should also be excluded
 none_fields = ["description", "doc_md"]
 for field in none_fields:
 assert field not in dag_data, f"None field '{field}' should be excluded"

 # Test deserialization restores defaults correctly
 deserialized_dag = SerializedDAG.from_dict(serialized)

 # Verify schema defaults are restored
 assert deserialized_dag.catchup is False
 assert deserialized_dag.fail_fast is False
 assert deserialized_dag.max_active_runs == 16
 assert deserialized_dag.max_active_tasks == 16
 assert deserialized_dag.max_consecutive_failed_dag_runs == 0
 assert deserialized_dag.render_template_as_native_obj is False
 assert deserialized_dag.disable_bundle_versioning is False

 # Test with non-default values (should be included)
 dag_non_defaults = DAG(
 dag_id="test_non_defaults_dag",
 start_date=datetime(2023, 1, 1),
 catchup=True, # Non-default
 max_active_runs=32, # Non-default
 description="Test description", # Non-None
 )

 serialized_non_defaults = SerializedDAG.to_dict(dag_non_defaults)
 dag_non_defaults_data = serialized_non_defaults["dag"]

 # Non-default values should be included
 assert "catchup" in dag_non_defaults_data
 assert dag_non_defaults_data["catchup"] is True
 assert "max_active_runs" in dag_non_defaults_data
 assert dag_non_defaults_data["max_active_runs"] == 32
 assert "description" in dag_non_defaults_data
 assert dag_non_defaults_data["description"] == "Test description"


def test_email_optimization_removes_email_attrs_when_email_empty():
 """Test that email_on_failure and email_on_retry are removed when email is empty."""
 with DAG(dag_id="test_email_optimization") as dag:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Optimize DAG serialization by excluding schema default values#55849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

kaxil
merged 2 commits into

from

Sep 18, 2025
 Merged


Optimize DAG serialization by excluding schema default values
#55849
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
589a4e6

Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025
a8fd7ea

fixup! Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 9 additions & 9 deletions
airflow-core/src/airflow/serialization/schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,8 +175,8 @@
 "value": { "$ref": "#/definitions/dict" }
 }
 },
 "catchup": { "type": "boolean" },
 "fail_fast": { "type": "boolean" },
 "catchup": { "type": "boolean", "default": false },
 "fail_fast": { "type": "boolean", "default": false },
 "fileloc": { "type" : "string"},
 "relative_fileloc": { "type" : "string"},
 "_processor_dags_folder": {
Expand All@@ -198,27 +198,27 @@
 ]
 },
 "_concurrency": { "type" : "number"},
 "max_active_tasks": { "type" : "number"},
 "max_active_runs": { "type" : "number"},
 "max_consecutive_failed_dag_runs": { "type" : "number"},
 "max_active_tasks": { "type" : "number", "default": 16},
 "max_active_runs": { "type" : "number", "default": 16},
 "max_consecutive_failed_dag_runs": { "type" : "number", "default": 0},
 "default_args": { "$ref": "#/definitions/dict" },
 "start_date": { "$ref": "#/definitions/datetime" },
 "end_date": { "$ref": "#/definitions/datetime" },
 "dagrun_timeout": { "$ref": "#/definitions/timedelta" },
 "doc_md": { "type" : "string"},
 "access_control": {"$ref": "#/definitions/dict" },
 "is_paused_upon_creation": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean" },
 "has_on_failure_callback": { "type": "boolean" },
 "render_template_as_native_obj": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean", "default": false },
 "has_on_failure_callback": { "type": "boolean", "default": false },
 "render_template_as_native_obj": { "type": "boolean", "default": false },
 "tags": { "type": "array" },
 "task_group": {"anyOf": [
 { "type": "null" },
 { "$ref": "#/definitions/task_group" }
 ]},
 "edge_info": { "$ref": "#/definitions/edge_info" },
 "dag_dependencies": { "$ref": "#/definitions/dag_dependencies" },
 "disable_bundle_versioning": {"type": "boolean"}
 "disable_bundle_versioning": {"type": "boolean", "default": false }
 },
 "required": [
 "dag_id",
Expand Down

80 changes: 55 additions & 25 deletions
airflow-core/src/airflow/serialization/serialized_objects.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1094,21 +1094,7 @@ def _deserialize_params_dict(cls, encoded_params: list[tuple[str, dict]]) -> Par
 return ParamsDict(op_params)

 @classmethod
Comment thread
kaxil marked this conversation as resolved.

 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 @lru_cache(maxsize=4) # Cache for "operator", "dag", and a few others
 def get_schema_defaults(cls, object_type: str) -> dict[str, Any]:
 """
 Extract default values from JSON schema for any object type.
Expand DownExpand Up@@ -1713,6 +1699,22 @@ def set_task_dag_references(task: SerializedOperator | MappedOperator, dag: Seri
 # Bypass set_upstream etc here - it does more than we want
 dag.task_dict[task_id].upstream_task_ids.add(task.task_id)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "operator"
 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def deserialize_operator(
 cls,
Expand DownExpand Up@@ -1814,7 +1816,7 @@ def detect_dependencies(cls, op: SdkOperator) -> set[DagDependency]:
 return deps

 @classmethod
 def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:
 def _matches_client_defaults(cls, var: Any, attrname: str) -> bool:
 """
 Check if a field value matches client_defaults and should be excluded.

Expand All@@ -1823,7 +1825,6 @@ def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:

 :param var: The value to check
 :param attrname: The attribute name
 :param op: The operator instance
 :return: True if value matches client_defaults and should be excluded
 """
 try:
Expand DownExpand Up@@ -1851,7 +1852,7 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 :return: True if a variable is excluded, False otherwise.
 """
 # Check if value matches client_defaults (hierarchical defaults optimization)
 if cls._matches_client_defaults(var, attrname, op):
 if cls._matches_client_defaults(var, attrname):
 return True
 schema_defaults = cls.get_schema_defaults("operator")

Expand DownExpand Up@@ -2384,25 +2385,23 @@ class SerializedDAG(BaseSerialization):
 _processor_dags_folder: str

 def __init__(self, *, dag_id: str) -> None:
 self.catchup = airflow_conf.getboolean("scheduler", "catchup_by_default")
 self.catchup = False # Schema default
 self.dag_id = self.dag_display_name = dag_id
 self.dagrun_timeout = None
 self.deadline = None
 self.default_args = {}
 self.description = None
 self.disable_bundle_versioning = airflow_conf.getboolean("dag_processor", "disable_bundle_versioning")
 self.disable_bundle_versioning = False
 self.doc_md = None
 self.edge_info = {}
 self.end_date = None
 self.fail_fast = False
 self.has_on_failure_callback = False
 self.has_on_success_callback = False
 self.is_paused_upon_creation = None
 self.max_active_runs = airflow_conf.getint("core", "max_active_runs_per_dag")
Comment thread
kaxil marked this conversation as resolved.

 self.max_active_tasks = airflow_conf.getint("core", "max_active_tasks_per_dag")
 self.max_consecutive_failed_dag_runs = airflow_conf.getint(
 "core", "max_consecutive_failed_dag_runs_per_dag"
 )
 self.max_active_runs = 16 # Schema default
 self.max_active_tasks = 16 # Schema default
 self.max_consecutive_failed_dag_runs = 0 # Schema default
 self.owner_links = {}
 self.params = ParamsDict()
 self.partial = False
Expand DownExpand Up@@ -2624,8 +2623,38 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 return False
 if attrname == "dag_display_name" and var == op.dag_id:
 return True

 # DAG schema defaults exclusion (same pattern as SerializedBaseOperator)
 dag_schema_defaults = cls.get_schema_defaults("dag")
 if attrname in dag_schema_defaults:
 if dag_schema_defaults[attrname] == var:
 return True

 optional_fields = cls.get_dag_optional_fields_from_schema()
 if var is None:
 return True
 if attrname in optional_fields:
 if var in [[], (), set(), {}]:
 return True

 return super()._is_excluded(var, attrname, op)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "dag"
 def get_dag_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("dag", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def to_dict(cls, var: Any) -> dict:
 """Stringifies DAGs and operators contained by var and returns a dict of var."""
Expand DownExpand Up@@ -3798,6 +3827,7 @@ class LazyDeserializedDAG(pydantic.BaseModel):
 "dag_display_name",
 "has_on_success_callback",
 "has_on_failure_callback",
 "tags",
 # Attr properties that are nullable, or have a default that loads from config
 "description",
 "start_date",
Expand Down

110 changes: 94 additions & 16 deletions
airflow-core/tests/unit/serialization/test_dag_serialization.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,13 +152,8 @@
 "downstream_task_ids": [],
 },
 "is_paused_upon_creation": False,
 "max_active_runs": 16,
 "max_active_tasks": 16,
 "max_consecutive_failed_dag_runs": 0,
 "dag_id": "simple_dag",
 "deadline": None,
 "catchup": False,
 "disable_bundle_versioning": False,
 "doc_md": "### DAG Tutorial Documentation",
 "fileloc": None,
 "_processor_dags_folder": (
Expand DownExpand Up@@ -269,7 +264,6 @@
 },
 ],
 "params": [],
 "tags": [],
 },
}

Expand DownExpand Up@@ -2177,7 +2171,8 @@ def test_dag_disable_bundle_versioning_roundtrip(self, dag_arg, conf_arg, expect
 """
 with conf_vars({("dag_processor", "disable_bundle_versioning"): conf_arg}):
 kwargs = {}
 kwargs["disable_bundle_versioning"] = dag_arg
 if dag_arg is not None:
 kwargs["disable_bundle_versioning"] = dag_arg
 dag = DAG(
 dag_id="test_dag_disable_bundle_versioning_roundtrip",
 schedule=None,
Expand DownExpand Up@@ -3299,17 +3294,34 @@ def test_handle_v1_serdag():
 SerializedDAG.conversion_v1_to_v2(v1)
 SerializedDAG.conversion_v2_to_v3(v1)

 # Update a few subtle differences
 v1["dag"]["tags"] = []
 v1["dag"]["catchup"] = False
 v1["dag"]["disable_bundle_versioning"] = False
 dag = SerializedDAG.from_dict(v1)

 expected = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected["dag"]["dag_dependencies"] = expected_dag_dependencies
 del expected["dag"]["tasks"][1]["__var"]["_operator_extra_links"]
 expected_sdag = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected = SerializedDAG.from_dict(expected_sdag)

 fields_to_verify = set(vars(expected).keys()) - {
 "task_group", # Tested separately
 "dag_dependencies", # Tested separately
 "last_loaded", # Dynamically set to utcnow
 }

 for f in fields_to_verify:
 dag_value = getattr(dag, f)
 expected_value = getattr(expected, f)

 assert dag_value == expected_value, (
 f"V2 DAG field '{f}' differs from V3: V2={dag_value!r} != V3={expected_value!r}"
 )

 for f in set(vars(expected.task_group).keys()) - {"dag"}:
 dag_tg_value = getattr(dag.task_group, f)
 expected_tg_value = getattr(expected.task_group, f)

 assert dag_tg_value == expected_tg_value, (
 f"V2 task_group field '{f}' differs: V2={dag_tg_value!r} != V3={expected_tg_value!r}"
 )

 del expected["client_defaults"]
 assert v1 == expected
 assert getattr(dag, "dag_dependencies") == expected_dag_dependencies


def test_handle_v2_serdag():
Expand DownExpand Up@@ -3514,6 +3526,72 @@ def test_handle_v2_serdag():
 )


def test_dag_schema_defaults_optimization():
 """Test that DAG fields matching schema defaults are excluded from serialization."""

 # Create DAG with all schema default values
 dag_with_defaults = DAG(
 dag_id="test_defaults_dag",
 start_date=datetime(2023, 1, 1),
 # These should match schema defaults and be excluded
 catchup=False,
 fail_fast=False,
 max_active_runs=16,
 max_active_tasks=16,
 max_consecutive_failed_dag_runs=0,
 render_template_as_native_obj=False,
 disable_bundle_versioning=False,
 # These should be excluded as None
 description=None,
 doc_md=None,
 )

 # Serialize and check exclusions
 serialized = SerializedDAG.to_dict(dag_with_defaults)
 dag_data = serialized["dag"]

 # Schema default fields should be excluded
 for field in SerializedDAG.get_schema_defaults("dag").keys():
 assert field not in dag_data, f"Schema default field '{field}' should be excluded"

 # None fields should also be excluded
 none_fields = ["description", "doc_md"]
 for field in none_fields:
 assert field not in dag_data, f"None field '{field}' should be excluded"

 # Test deserialization restores defaults correctly
 deserialized_dag = SerializedDAG.from_dict(serialized)

 # Verify schema defaults are restored
 assert deserialized_dag.catchup is False
 assert deserialized_dag.fail_fast is False
 assert deserialized_dag.max_active_runs == 16
 assert deserialized_dag.max_active_tasks == 16
 assert deserialized_dag.max_consecutive_failed_dag_runs == 0
 assert deserialized_dag.render_template_as_native_obj is False
 assert deserialized_dag.disable_bundle_versioning is False

 # Test with non-default values (should be included)
 dag_non_defaults = DAG(
 dag_id="test_non_defaults_dag",
 start_date=datetime(2023, 1, 1),
 catchup=True, # Non-default
 max_active_runs=32, # Non-default
 description="Test description", # Non-None
 )

 serialized_non_defaults = SerializedDAG.to_dict(dag_non_defaults)
 dag_non_defaults_data = serialized_non_defaults["dag"]

 # Non-default values should be included
 assert "catchup" in dag_non_defaults_data
 assert dag_non_defaults_data["catchup"] is True
 assert "max_active_runs" in dag_non_defaults_data
 assert dag_non_defaults_data["max_active_runs"] == 32
 assert "description" in dag_non_defaults_data
 assert dag_non_defaults_data["description"] == "Test description"


def test_email_optimization_removes_email_attrs_when_email_empty():
 """Test that email_on_failure and email_on_retry are removed when email is empty."""
 with DAG(dag_id="test_email_optimization") as dag:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Optimize DAG serialization by excluding schema default values#55849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

kaxil
merged 2 commits into

from

Sep 18, 2025
 Merged


Optimize DAG serialization by excluding schema default values
#55849
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
589a4e6

Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025
a8fd7ea

fixup! Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 9 additions & 9 deletions
airflow-core/src/airflow/serialization/schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,8 +175,8 @@
 "value": { "$ref": "#/definitions/dict" }
 }
 },
 "catchup": { "type": "boolean" },
 "fail_fast": { "type": "boolean" },
 "catchup": { "type": "boolean", "default": false },
 "fail_fast": { "type": "boolean", "default": false },
 "fileloc": { "type" : "string"},
 "relative_fileloc": { "type" : "string"},
 "_processor_dags_folder": {
Expand All@@ -198,27 +198,27 @@
 ]
 },
 "_concurrency": { "type" : "number"},
 "max_active_tasks": { "type" : "number"},
 "max_active_runs": { "type" : "number"},
 "max_consecutive_failed_dag_runs": { "type" : "number"},
 "max_active_tasks": { "type" : "number", "default": 16},
 "max_active_runs": { "type" : "number", "default": 16},
 "max_consecutive_failed_dag_runs": { "type" : "number", "default": 0},
 "default_args": { "$ref": "#/definitions/dict" },
 "start_date": { "$ref": "#/definitions/datetime" },
 "end_date": { "$ref": "#/definitions/datetime" },
 "dagrun_timeout": { "$ref": "#/definitions/timedelta" },
 "doc_md": { "type" : "string"},
 "access_control": {"$ref": "#/definitions/dict" },
 "is_paused_upon_creation": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean" },
 "has_on_failure_callback": { "type": "boolean" },
 "render_template_as_native_obj": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean", "default": false },
 "has_on_failure_callback": { "type": "boolean", "default": false },
 "render_template_as_native_obj": { "type": "boolean", "default": false },
 "tags": { "type": "array" },
 "task_group": {"anyOf": [
 { "type": "null" },
 { "$ref": "#/definitions/task_group" }
 ]},
 "edge_info": { "$ref": "#/definitions/edge_info" },
 "dag_dependencies": { "$ref": "#/definitions/dag_dependencies" },
 "disable_bundle_versioning": {"type": "boolean"}
 "disable_bundle_versioning": {"type": "boolean", "default": false }
 },
 "required": [
 "dag_id",
Expand Down

80 changes: 55 additions & 25 deletions
airflow-core/src/airflow/serialization/serialized_objects.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1094,21 +1094,7 @@ def _deserialize_params_dict(cls, encoded_params: list[tuple[str, dict]]) -> Par
 return ParamsDict(op_params)

 @classmethod
Comment thread
kaxil marked this conversation as resolved.

 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 @lru_cache(maxsize=4) # Cache for "operator", "dag", and a few others
 def get_schema_defaults(cls, object_type: str) -> dict[str, Any]:
 """
 Extract default values from JSON schema for any object type.
Expand DownExpand Up@@ -1713,6 +1699,22 @@ def set_task_dag_references(task: SerializedOperator | MappedOperator, dag: Seri
 # Bypass set_upstream etc here - it does more than we want
 dag.task_dict[task_id].upstream_task_ids.add(task.task_id)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "operator"
 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def deserialize_operator(
 cls,
Expand DownExpand Up@@ -1814,7 +1816,7 @@ def detect_dependencies(cls, op: SdkOperator) -> set[DagDependency]:
 return deps

 @classmethod
 def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:
 def _matches_client_defaults(cls, var: Any, attrname: str) -> bool:
 """
 Check if a field value matches client_defaults and should be excluded.

Expand All@@ -1823,7 +1825,6 @@ def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:

 :param var: The value to check
 :param attrname: The attribute name
 :param op: The operator instance
 :return: True if value matches client_defaults and should be excluded
 """
 try:
Expand DownExpand Up@@ -1851,7 +1852,7 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 :return: True if a variable is excluded, False otherwise.
 """
 # Check if value matches client_defaults (hierarchical defaults optimization)
 if cls._matches_client_defaults(var, attrname, op):
 if cls._matches_client_defaults(var, attrname):
 return True
 schema_defaults = cls.get_schema_defaults("operator")

Expand DownExpand Up@@ -2384,25 +2385,23 @@ class SerializedDAG(BaseSerialization):
 _processor_dags_folder: str

 def __init__(self, *, dag_id: str) -> None:
 self.catchup = airflow_conf.getboolean("scheduler", "catchup_by_default")
 self.catchup = False # Schema default
 self.dag_id = self.dag_display_name = dag_id
 self.dagrun_timeout = None
 self.deadline = None
 self.default_args = {}
 self.description = None
 self.disable_bundle_versioning = airflow_conf.getboolean("dag_processor", "disable_bundle_versioning")
 self.disable_bundle_versioning = False
 self.doc_md = None
 self.edge_info = {}
 self.end_date = None
 self.fail_fast = False
 self.has_on_failure_callback = False
 self.has_on_success_callback = False
 self.is_paused_upon_creation = None
 self.max_active_runs = airflow_conf.getint("core", "max_active_runs_per_dag")
Comment thread
kaxil marked this conversation as resolved.

 self.max_active_tasks = airflow_conf.getint("core", "max_active_tasks_per_dag")
 self.max_consecutive_failed_dag_runs = airflow_conf.getint(
 "core", "max_consecutive_failed_dag_runs_per_dag"
 )
 self.max_active_runs = 16 # Schema default
 self.max_active_tasks = 16 # Schema default
 self.max_consecutive_failed_dag_runs = 0 # Schema default
 self.owner_links = {}
 self.params = ParamsDict()
 self.partial = False
Expand DownExpand Up@@ -2624,8 +2623,38 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 return False
 if attrname == "dag_display_name" and var == op.dag_id:
 return True

 # DAG schema defaults exclusion (same pattern as SerializedBaseOperator)
 dag_schema_defaults = cls.get_schema_defaults("dag")
 if attrname in dag_schema_defaults:
 if dag_schema_defaults[attrname] == var:
 return True

 optional_fields = cls.get_dag_optional_fields_from_schema()
 if var is None:
 return True
 if attrname in optional_fields:
 if var in [[], (), set(), {}]:
 return True

 return super()._is_excluded(var, attrname, op)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "dag"
 def get_dag_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("dag", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def to_dict(cls, var: Any) -> dict:
 """Stringifies DAGs and operators contained by var and returns a dict of var."""
Expand DownExpand Up@@ -3798,6 +3827,7 @@ class LazyDeserializedDAG(pydantic.BaseModel):
 "dag_display_name",
 "has_on_success_callback",
 "has_on_failure_callback",
 "tags",
 # Attr properties that are nullable, or have a default that loads from config
 "description",
 "start_date",
Expand Down

110 changes: 94 additions & 16 deletions
airflow-core/tests/unit/serialization/test_dag_serialization.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,13 +152,8 @@
 "downstream_task_ids": [],
 },
 "is_paused_upon_creation": False,
 "max_active_runs": 16,
 "max_active_tasks": 16,
 "max_consecutive_failed_dag_runs": 0,
 "dag_id": "simple_dag",
 "deadline": None,
 "catchup": False,
 "disable_bundle_versioning": False,
 "doc_md": "### DAG Tutorial Documentation",
 "fileloc": None,
 "_processor_dags_folder": (
Expand DownExpand Up@@ -269,7 +264,6 @@
 },
 ],
 "params": [],
 "tags": [],
 },
}

Expand DownExpand Up@@ -2177,7 +2171,8 @@ def test_dag_disable_bundle_versioning_roundtrip(self, dag_arg, conf_arg, expect
 """
 with conf_vars({("dag_processor", "disable_bundle_versioning"): conf_arg}):
 kwargs = {}
 kwargs["disable_bundle_versioning"] = dag_arg
 if dag_arg is not None:
 kwargs["disable_bundle_versioning"] = dag_arg
 dag = DAG(
 dag_id="test_dag_disable_bundle_versioning_roundtrip",
 schedule=None,
Expand DownExpand Up@@ -3299,17 +3294,34 @@ def test_handle_v1_serdag():
 SerializedDAG.conversion_v1_to_v2(v1)
 SerializedDAG.conversion_v2_to_v3(v1)

 # Update a few subtle differences
 v1["dag"]["tags"] = []
 v1["dag"]["catchup"] = False
 v1["dag"]["disable_bundle_versioning"] = False
 dag = SerializedDAG.from_dict(v1)

 expected = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected["dag"]["dag_dependencies"] = expected_dag_dependencies
 del expected["dag"]["tasks"][1]["__var"]["_operator_extra_links"]
 expected_sdag = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected = SerializedDAG.from_dict(expected_sdag)

 fields_to_verify = set(vars(expected).keys()) - {
 "task_group", # Tested separately
 "dag_dependencies", # Tested separately
 "last_loaded", # Dynamically set to utcnow
 }

 for f in fields_to_verify:
 dag_value = getattr(dag, f)
 expected_value = getattr(expected, f)

 assert dag_value == expected_value, (
 f"V2 DAG field '{f}' differs from V3: V2={dag_value!r} != V3={expected_value!r}"
 )

 for f in set(vars(expected.task_group).keys()) - {"dag"}:
 dag_tg_value = getattr(dag.task_group, f)
 expected_tg_value = getattr(expected.task_group, f)

 assert dag_tg_value == expected_tg_value, (
 f"V2 task_group field '{f}' differs: V2={dag_tg_value!r} != V3={expected_tg_value!r}"
 )

 del expected["client_defaults"]
 assert v1 == expected
 assert getattr(dag, "dag_dependencies") == expected_dag_dependencies


def test_handle_v2_serdag():
Expand DownExpand Up@@ -3514,6 +3526,72 @@ def test_handle_v2_serdag():
 )


def test_dag_schema_defaults_optimization():
 """Test that DAG fields matching schema defaults are excluded from serialization."""

 # Create DAG with all schema default values
 dag_with_defaults = DAG(
 dag_id="test_defaults_dag",
 start_date=datetime(2023, 1, 1),
 # These should match schema defaults and be excluded
 catchup=False,
 fail_fast=False,
 max_active_runs=16,
 max_active_tasks=16,
 max_consecutive_failed_dag_runs=0,
 render_template_as_native_obj=False,
 disable_bundle_versioning=False,
 # These should be excluded as None
 description=None,
 doc_md=None,
 )

 # Serialize and check exclusions
 serialized = SerializedDAG.to_dict(dag_with_defaults)
 dag_data = serialized["dag"]

 # Schema default fields should be excluded
 for field in SerializedDAG.get_schema_defaults("dag").keys():
 assert field not in dag_data, f"Schema default field '{field}' should be excluded"

 # None fields should also be excluded
 none_fields = ["description", "doc_md"]
 for field in none_fields:
 assert field not in dag_data, f"None field '{field}' should be excluded"

 # Test deserialization restores defaults correctly
 deserialized_dag = SerializedDAG.from_dict(serialized)

 # Verify schema defaults are restored
 assert deserialized_dag.catchup is False
 assert deserialized_dag.fail_fast is False
 assert deserialized_dag.max_active_runs == 16
 assert deserialized_dag.max_active_tasks == 16
 assert deserialized_dag.max_consecutive_failed_dag_runs == 0
 assert deserialized_dag.render_template_as_native_obj is False
 assert deserialized_dag.disable_bundle_versioning is False

 # Test with non-default values (should be included)
 dag_non_defaults = DAG(
 dag_id="test_non_defaults_dag",
 start_date=datetime(2023, 1, 1),
 catchup=True, # Non-default
 max_active_runs=32, # Non-default
 description="Test description", # Non-None
 )

 serialized_non_defaults = SerializedDAG.to_dict(dag_non_defaults)
 dag_non_defaults_data = serialized_non_defaults["dag"]

 # Non-default values should be included
 assert "catchup" in dag_non_defaults_data
 assert dag_non_defaults_data["catchup"] is True
 assert "max_active_runs" in dag_non_defaults_data
 assert dag_non_defaults_data["max_active_runs"] == 32
 assert "description" in dag_non_defaults_data
 assert dag_non_defaults_data["description"] == "Test description"


def test_email_optimization_removes_email_attrs_when_email_empty():
 """Test that email_on_failure and email_on_retry are removed when email is empty."""
 with DAG(dag_id="test_email_optimization") as dag:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Optimize DAG serialization by excluding schema default values#55849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

kaxil
merged 2 commits into

from

Sep 18, 2025
 Merged


Optimize DAG serialization by excluding schema default values
#55849
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
589a4e6

Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025
a8fd7ea

fixup! Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 9 additions & 9 deletions
airflow-core/src/airflow/serialization/schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,8 +175,8 @@
 "value": { "$ref": "#/definitions/dict" }
 }
 },
 "catchup": { "type": "boolean" },
 "fail_fast": { "type": "boolean" },
 "catchup": { "type": "boolean", "default": false },
 "fail_fast": { "type": "boolean", "default": false },
 "fileloc": { "type" : "string"},
 "relative_fileloc": { "type" : "string"},
 "_processor_dags_folder": {
Expand All@@ -198,27 +198,27 @@
 ]
 },
 "_concurrency": { "type" : "number"},
 "max_active_tasks": { "type" : "number"},
 "max_active_runs": { "type" : "number"},
 "max_consecutive_failed_dag_runs": { "type" : "number"},
 "max_active_tasks": { "type" : "number", "default": 16},
 "max_active_runs": { "type" : "number", "default": 16},
 "max_consecutive_failed_dag_runs": { "type" : "number", "default": 0},
 "default_args": { "$ref": "#/definitions/dict" },
 "start_date": { "$ref": "#/definitions/datetime" },
 "end_date": { "$ref": "#/definitions/datetime" },
 "dagrun_timeout": { "$ref": "#/definitions/timedelta" },
 "doc_md": { "type" : "string"},
 "access_control": {"$ref": "#/definitions/dict" },
 "is_paused_upon_creation": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean" },
 "has_on_failure_callback": { "type": "boolean" },
 "render_template_as_native_obj": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean", "default": false },
 "has_on_failure_callback": { "type": "boolean", "default": false },
 "render_template_as_native_obj": { "type": "boolean", "default": false },
 "tags": { "type": "array" },
 "task_group": {"anyOf": [
 { "type": "null" },
 { "$ref": "#/definitions/task_group" }
 ]},
 "edge_info": { "$ref": "#/definitions/edge_info" },
 "dag_dependencies": { "$ref": "#/definitions/dag_dependencies" },
 "disable_bundle_versioning": {"type": "boolean"}
 "disable_bundle_versioning": {"type": "boolean", "default": false }
 },
 "required": [
 "dag_id",
Expand Down

80 changes: 55 additions & 25 deletions
airflow-core/src/airflow/serialization/serialized_objects.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1094,21 +1094,7 @@ def _deserialize_params_dict(cls, encoded_params: list[tuple[str, dict]]) -> Par
 return ParamsDict(op_params)

 @classmethod
Comment thread
kaxil marked this conversation as resolved.

 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 @lru_cache(maxsize=4) # Cache for "operator", "dag", and a few others
 def get_schema_defaults(cls, object_type: str) -> dict[str, Any]:
 """
 Extract default values from JSON schema for any object type.
Expand DownExpand Up@@ -1713,6 +1699,22 @@ def set_task_dag_references(task: SerializedOperator | MappedOperator, dag: Seri
 # Bypass set_upstream etc here - it does more than we want
 dag.task_dict[task_id].upstream_task_ids.add(task.task_id)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "operator"
 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def deserialize_operator(
 cls,
Expand DownExpand Up@@ -1814,7 +1816,7 @@ def detect_dependencies(cls, op: SdkOperator) -> set[DagDependency]:
 return deps

 @classmethod
 def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:
 def _matches_client_defaults(cls, var: Any, attrname: str) -> bool:
 """
 Check if a field value matches client_defaults and should be excluded.

Expand All@@ -1823,7 +1825,6 @@ def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:

 :param var: The value to check
 :param attrname: The attribute name
 :param op: The operator instance
 :return: True if value matches client_defaults and should be excluded
 """
 try:
Expand DownExpand Up@@ -1851,7 +1852,7 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 :return: True if a variable is excluded, False otherwise.
 """
 # Check if value matches client_defaults (hierarchical defaults optimization)
 if cls._matches_client_defaults(var, attrname, op):
 if cls._matches_client_defaults(var, attrname):
 return True
 schema_defaults = cls.get_schema_defaults("operator")

Expand DownExpand Up@@ -2384,25 +2385,23 @@ class SerializedDAG(BaseSerialization):
 _processor_dags_folder: str

 def __init__(self, *, dag_id: str) -> None:
 self.catchup = airflow_conf.getboolean("scheduler", "catchup_by_default")
 self.catchup = False # Schema default
 self.dag_id = self.dag_display_name = dag_id
 self.dagrun_timeout = None
 self.deadline = None
 self.default_args = {}
 self.description = None
 self.disable_bundle_versioning = airflow_conf.getboolean("dag_processor", "disable_bundle_versioning")
 self.disable_bundle_versioning = False
 self.doc_md = None
 self.edge_info = {}
 self.end_date = None
 self.fail_fast = False
 self.has_on_failure_callback = False
 self.has_on_success_callback = False
 self.is_paused_upon_creation = None
 self.max_active_runs = airflow_conf.getint("core", "max_active_runs_per_dag")
Comment thread
kaxil marked this conversation as resolved.

 self.max_active_tasks = airflow_conf.getint("core", "max_active_tasks_per_dag")
 self.max_consecutive_failed_dag_runs = airflow_conf.getint(
 "core", "max_consecutive_failed_dag_runs_per_dag"
 )
 self.max_active_runs = 16 # Schema default
 self.max_active_tasks = 16 # Schema default
 self.max_consecutive_failed_dag_runs = 0 # Schema default
 self.owner_links = {}
 self.params = ParamsDict()
 self.partial = False
Expand DownExpand Up@@ -2624,8 +2623,38 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 return False
 if attrname == "dag_display_name" and var == op.dag_id:
 return True

 # DAG schema defaults exclusion (same pattern as SerializedBaseOperator)
 dag_schema_defaults = cls.get_schema_defaults("dag")
 if attrname in dag_schema_defaults:
 if dag_schema_defaults[attrname] == var:
 return True

 optional_fields = cls.get_dag_optional_fields_from_schema()
 if var is None:
 return True
 if attrname in optional_fields:
 if var in [[], (), set(), {}]:
 return True

 return super()._is_excluded(var, attrname, op)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "dag"
 def get_dag_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("dag", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def to_dict(cls, var: Any) -> dict:
 """Stringifies DAGs and operators contained by var and returns a dict of var."""
Expand DownExpand Up@@ -3798,6 +3827,7 @@ class LazyDeserializedDAG(pydantic.BaseModel):
 "dag_display_name",
 "has_on_success_callback",
 "has_on_failure_callback",
 "tags",
 # Attr properties that are nullable, or have a default that loads from config
 "description",
 "start_date",
Expand Down

110 changes: 94 additions & 16 deletions
airflow-core/tests/unit/serialization/test_dag_serialization.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,13 +152,8 @@
 "downstream_task_ids": [],
 },
 "is_paused_upon_creation": False,
 "max_active_runs": 16,
 "max_active_tasks": 16,
 "max_consecutive_failed_dag_runs": 0,
 "dag_id": "simple_dag",
 "deadline": None,
 "catchup": False,
 "disable_bundle_versioning": False,
 "doc_md": "### DAG Tutorial Documentation",
 "fileloc": None,
 "_processor_dags_folder": (
Expand DownExpand Up@@ -269,7 +264,6 @@
 },
 ],
 "params": [],
 "tags": [],
 },
}

Expand DownExpand Up@@ -2177,7 +2171,8 @@ def test_dag_disable_bundle_versioning_roundtrip(self, dag_arg, conf_arg, expect
 """
 with conf_vars({("dag_processor", "disable_bundle_versioning"): conf_arg}):
 kwargs = {}
 kwargs["disable_bundle_versioning"] = dag_arg
 if dag_arg is not None:
 kwargs["disable_bundle_versioning"] = dag_arg
 dag = DAG(
 dag_id="test_dag_disable_bundle_versioning_roundtrip",
 schedule=None,
Expand DownExpand Up@@ -3299,17 +3294,34 @@ def test_handle_v1_serdag():
 SerializedDAG.conversion_v1_to_v2(v1)
 SerializedDAG.conversion_v2_to_v3(v1)

 # Update a few subtle differences
 v1["dag"]["tags"] = []
 v1["dag"]["catchup"] = False
 v1["dag"]["disable_bundle_versioning"] = False
 dag = SerializedDAG.from_dict(v1)

 expected = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected["dag"]["dag_dependencies"] = expected_dag_dependencies
 del expected["dag"]["tasks"][1]["__var"]["_operator_extra_links"]
 expected_sdag = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected = SerializedDAG.from_dict(expected_sdag)

 fields_to_verify = set(vars(expected).keys()) - {
 "task_group", # Tested separately
 "dag_dependencies", # Tested separately
 "last_loaded", # Dynamically set to utcnow
 }

 for f in fields_to_verify:
 dag_value = getattr(dag, f)
 expected_value = getattr(expected, f)

 assert dag_value == expected_value, (
 f"V2 DAG field '{f}' differs from V3: V2={dag_value!r} != V3={expected_value!r}"
 )

 for f in set(vars(expected.task_group).keys()) - {"dag"}:
 dag_tg_value = getattr(dag.task_group, f)
 expected_tg_value = getattr(expected.task_group, f)

 assert dag_tg_value == expected_tg_value, (
 f"V2 task_group field '{f}' differs: V2={dag_tg_value!r} != V3={expected_tg_value!r}"
 )

 del expected["client_defaults"]
 assert v1 == expected
 assert getattr(dag, "dag_dependencies") == expected_dag_dependencies


def test_handle_v2_serdag():
Expand DownExpand Up@@ -3514,6 +3526,72 @@ def test_handle_v2_serdag():
 )


def test_dag_schema_defaults_optimization():
 """Test that DAG fields matching schema defaults are excluded from serialization."""

 # Create DAG with all schema default values
 dag_with_defaults = DAG(
 dag_id="test_defaults_dag",
 start_date=datetime(2023, 1, 1),
 # These should match schema defaults and be excluded
 catchup=False,
 fail_fast=False,
 max_active_runs=16,
 max_active_tasks=16,
 max_consecutive_failed_dag_runs=0,
 render_template_as_native_obj=False,
 disable_bundle_versioning=False,
 # These should be excluded as None
 description=None,
 doc_md=None,
 )

 # Serialize and check exclusions
 serialized = SerializedDAG.to_dict(dag_with_defaults)
 dag_data = serialized["dag"]

 # Schema default fields should be excluded
 for field in SerializedDAG.get_schema_defaults("dag").keys():
 assert field not in dag_data, f"Schema default field '{field}' should be excluded"

 # None fields should also be excluded
 none_fields = ["description", "doc_md"]
 for field in none_fields:
 assert field not in dag_data, f"None field '{field}' should be excluded"

 # Test deserialization restores defaults correctly
 deserialized_dag = SerializedDAG.from_dict(serialized)

 # Verify schema defaults are restored
 assert deserialized_dag.catchup is False
 assert deserialized_dag.fail_fast is False
 assert deserialized_dag.max_active_runs == 16
 assert deserialized_dag.max_active_tasks == 16
 assert deserialized_dag.max_consecutive_failed_dag_runs == 0
 assert deserialized_dag.render_template_as_native_obj is False
 assert deserialized_dag.disable_bundle_versioning is False

 # Test with non-default values (should be included)
 dag_non_defaults = DAG(
 dag_id="test_non_defaults_dag",
 start_date=datetime(2023, 1, 1),
 catchup=True, # Non-default
 max_active_runs=32, # Non-default
 description="Test description", # Non-None
 )

 serialized_non_defaults = SerializedDAG.to_dict(dag_non_defaults)
 dag_non_defaults_data = serialized_non_defaults["dag"]

 # Non-default values should be included
 assert "catchup" in dag_non_defaults_data
 assert dag_non_defaults_data["catchup"] is True
 assert "max_active_runs" in dag_non_defaults_data
 assert dag_non_defaults_data["max_active_runs"] == 32
 assert "description" in dag_non_defaults_data
 assert dag_non_defaults_data["description"] == "Test description"


def test_email_optimization_removes_email_attrs_when_email_empty():
 """Test that email_on_failure and email_on_retry are removed when email is empty."""
 with DAG(dag_id="test_email_optimization") as dag:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Optimize DAG serialization by excluding schema default values#55849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

kaxil
merged 2 commits into

from

Sep 18, 2025
 Merged


Optimize DAG serialization by excluding schema default values
#55849
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
589a4e6

Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025
a8fd7ea

fixup! Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 9 additions & 9 deletions
airflow-core/src/airflow/serialization/schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,8 +175,8 @@
 "value": { "$ref": "#/definitions/dict" }
 }
 },
 "catchup": { "type": "boolean" },
 "fail_fast": { "type": "boolean" },
 "catchup": { "type": "boolean", "default": false },
 "fail_fast": { "type": "boolean", "default": false },
 "fileloc": { "type" : "string"},
 "relative_fileloc": { "type" : "string"},
 "_processor_dags_folder": {
Expand All@@ -198,27 +198,27 @@
 ]
 },
 "_concurrency": { "type" : "number"},
 "max_active_tasks": { "type" : "number"},
 "max_active_runs": { "type" : "number"},
 "max_consecutive_failed_dag_runs": { "type" : "number"},
 "max_active_tasks": { "type" : "number", "default": 16},
 "max_active_runs": { "type" : "number", "default": 16},
 "max_consecutive_failed_dag_runs": { "type" : "number", "default": 0},
 "default_args": { "$ref": "#/definitions/dict" },
 "start_date": { "$ref": "#/definitions/datetime" },
 "end_date": { "$ref": "#/definitions/datetime" },
 "dagrun_timeout": { "$ref": "#/definitions/timedelta" },
 "doc_md": { "type" : "string"},
 "access_control": {"$ref": "#/definitions/dict" },
 "is_paused_upon_creation": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean" },
 "has_on_failure_callback": { "type": "boolean" },
 "render_template_as_native_obj": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean", "default": false },
 "has_on_failure_callback": { "type": "boolean", "default": false },
 "render_template_as_native_obj": { "type": "boolean", "default": false },
 "tags": { "type": "array" },
 "task_group": {"anyOf": [
 { "type": "null" },
 { "$ref": "#/definitions/task_group" }
 ]},
 "edge_info": { "$ref": "#/definitions/edge_info" },
 "dag_dependencies": { "$ref": "#/definitions/dag_dependencies" },
 "disable_bundle_versioning": {"type": "boolean"}
 "disable_bundle_versioning": {"type": "boolean", "default": false }
 },
 "required": [
 "dag_id",
Expand Down

80 changes: 55 additions & 25 deletions
airflow-core/src/airflow/serialization/serialized_objects.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1094,21 +1094,7 @@ def _deserialize_params_dict(cls, encoded_params: list[tuple[str, dict]]) -> Par
 return ParamsDict(op_params)

 @classmethod
Comment thread
kaxil marked this conversation as resolved.

 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 @lru_cache(maxsize=4) # Cache for "operator", "dag", and a few others
 def get_schema_defaults(cls, object_type: str) -> dict[str, Any]:
 """
 Extract default values from JSON schema for any object type.
Expand DownExpand Up@@ -1713,6 +1699,22 @@ def set_task_dag_references(task: SerializedOperator | MappedOperator, dag: Seri
 # Bypass set_upstream etc here - it does more than we want
 dag.task_dict[task_id].upstream_task_ids.add(task.task_id)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "operator"
 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def deserialize_operator(
 cls,
Expand DownExpand Up@@ -1814,7 +1816,7 @@ def detect_dependencies(cls, op: SdkOperator) -> set[DagDependency]:
 return deps

 @classmethod
 def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:
 def _matches_client_defaults(cls, var: Any, attrname: str) -> bool:
 """
 Check if a field value matches client_defaults and should be excluded.

Expand All@@ -1823,7 +1825,6 @@ def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:

 :param var: The value to check
 :param attrname: The attribute name
 :param op: The operator instance
 :return: True if value matches client_defaults and should be excluded
 """
 try:
Expand DownExpand Up@@ -1851,7 +1852,7 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 :return: True if a variable is excluded, False otherwise.
 """
 # Check if value matches client_defaults (hierarchical defaults optimization)
 if cls._matches_client_defaults(var, attrname, op):
 if cls._matches_client_defaults(var, attrname):
 return True
 schema_defaults = cls.get_schema_defaults("operator")

Expand DownExpand Up@@ -2384,25 +2385,23 @@ class SerializedDAG(BaseSerialization):
 _processor_dags_folder: str

 def __init__(self, *, dag_id: str) -> None:
 self.catchup = airflow_conf.getboolean("scheduler", "catchup_by_default")
 self.catchup = False # Schema default
 self.dag_id = self.dag_display_name = dag_id
 self.dagrun_timeout = None
 self.deadline = None
 self.default_args = {}
 self.description = None
 self.disable_bundle_versioning = airflow_conf.getboolean("dag_processor", "disable_bundle_versioning")
 self.disable_bundle_versioning = False
 self.doc_md = None
 self.edge_info = {}
 self.end_date = None
 self.fail_fast = False
 self.has_on_failure_callback = False
 self.has_on_success_callback = False
 self.is_paused_upon_creation = None
 self.max_active_runs = airflow_conf.getint("core", "max_active_runs_per_dag")
Comment thread
kaxil marked this conversation as resolved.

 self.max_active_tasks = airflow_conf.getint("core", "max_active_tasks_per_dag")
 self.max_consecutive_failed_dag_runs = airflow_conf.getint(
 "core", "max_consecutive_failed_dag_runs_per_dag"
 )
 self.max_active_runs = 16 # Schema default
 self.max_active_tasks = 16 # Schema default
 self.max_consecutive_failed_dag_runs = 0 # Schema default
 self.owner_links = {}
 self.params = ParamsDict()
 self.partial = False
Expand DownExpand Up@@ -2624,8 +2623,38 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 return False
 if attrname == "dag_display_name" and var == op.dag_id:
 return True

 # DAG schema defaults exclusion (same pattern as SerializedBaseOperator)
 dag_schema_defaults = cls.get_schema_defaults("dag")
 if attrname in dag_schema_defaults:
 if dag_schema_defaults[attrname] == var:
 return True

 optional_fields = cls.get_dag_optional_fields_from_schema()
 if var is None:
 return True
 if attrname in optional_fields:
 if var in [[], (), set(), {}]:
 return True

 return super()._is_excluded(var, attrname, op)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "dag"
 def get_dag_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("dag", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def to_dict(cls, var: Any) -> dict:
 """Stringifies DAGs and operators contained by var and returns a dict of var."""
Expand DownExpand Up@@ -3798,6 +3827,7 @@ class LazyDeserializedDAG(pydantic.BaseModel):
 "dag_display_name",
 "has_on_success_callback",
 "has_on_failure_callback",
 "tags",
 # Attr properties that are nullable, or have a default that loads from config
 "description",
 "start_date",
Expand Down

110 changes: 94 additions & 16 deletions
airflow-core/tests/unit/serialization/test_dag_serialization.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,13 +152,8 @@
 "downstream_task_ids": [],
 },
 "is_paused_upon_creation": False,
 "max_active_runs": 16,
 "max_active_tasks": 16,
 "max_consecutive_failed_dag_runs": 0,
 "dag_id": "simple_dag",
 "deadline": None,
 "catchup": False,
 "disable_bundle_versioning": False,
 "doc_md": "### DAG Tutorial Documentation",
 "fileloc": None,
 "_processor_dags_folder": (
Expand DownExpand Up@@ -269,7 +264,6 @@
 },
 ],
 "params": [],
 "tags": [],
 },
}

Expand DownExpand Up@@ -2177,7 +2171,8 @@ def test_dag_disable_bundle_versioning_roundtrip(self, dag_arg, conf_arg, expect
 """
 with conf_vars({("dag_processor", "disable_bundle_versioning"): conf_arg}):
 kwargs = {}
 kwargs["disable_bundle_versioning"] = dag_arg
 if dag_arg is not None:
 kwargs["disable_bundle_versioning"] = dag_arg
 dag = DAG(
 dag_id="test_dag_disable_bundle_versioning_roundtrip",
 schedule=None,
Expand DownExpand Up@@ -3299,17 +3294,34 @@ def test_handle_v1_serdag():
 SerializedDAG.conversion_v1_to_v2(v1)
 SerializedDAG.conversion_v2_to_v3(v1)

 # Update a few subtle differences
 v1["dag"]["tags"] = []
 v1["dag"]["catchup"] = False
 v1["dag"]["disable_bundle_versioning"] = False
 dag = SerializedDAG.from_dict(v1)

 expected = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected["dag"]["dag_dependencies"] = expected_dag_dependencies
 del expected["dag"]["tasks"][1]["__var"]["_operator_extra_links"]
 expected_sdag = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected = SerializedDAG.from_dict(expected_sdag)

 fields_to_verify = set(vars(expected).keys()) - {
 "task_group", # Tested separately
 "dag_dependencies", # Tested separately
 "last_loaded", # Dynamically set to utcnow
 }

 for f in fields_to_verify:
 dag_value = getattr(dag, f)
 expected_value = getattr(expected, f)

 assert dag_value == expected_value, (
 f"V2 DAG field '{f}' differs from V3: V2={dag_value!r} != V3={expected_value!r}"
 )

 for f in set(vars(expected.task_group).keys()) - {"dag"}:
 dag_tg_value = getattr(dag.task_group, f)
 expected_tg_value = getattr(expected.task_group, f)

 assert dag_tg_value == expected_tg_value, (
 f"V2 task_group field '{f}' differs: V2={dag_tg_value!r} != V3={expected_tg_value!r}"
 )

 del expected["client_defaults"]
 assert v1 == expected
 assert getattr(dag, "dag_dependencies") == expected_dag_dependencies


def test_handle_v2_serdag():
Expand DownExpand Up@@ -3514,6 +3526,72 @@ def test_handle_v2_serdag():
 )


def test_dag_schema_defaults_optimization():
 """Test that DAG fields matching schema defaults are excluded from serialization."""

 # Create DAG with all schema default values
 dag_with_defaults = DAG(
 dag_id="test_defaults_dag",
 start_date=datetime(2023, 1, 1),
 # These should match schema defaults and be excluded
 catchup=False,
 fail_fast=False,
 max_active_runs=16,
 max_active_tasks=16,
 max_consecutive_failed_dag_runs=0,
 render_template_as_native_obj=False,
 disable_bundle_versioning=False,
 # These should be excluded as None
 description=None,
 doc_md=None,
 )

 # Serialize and check exclusions
 serialized = SerializedDAG.to_dict(dag_with_defaults)
 dag_data = serialized["dag"]

 # Schema default fields should be excluded
 for field in SerializedDAG.get_schema_defaults("dag").keys():
 assert field not in dag_data, f"Schema default field '{field}' should be excluded"

 # None fields should also be excluded
 none_fields = ["description", "doc_md"]
 for field in none_fields:
 assert field not in dag_data, f"None field '{field}' should be excluded"

 # Test deserialization restores defaults correctly
 deserialized_dag = SerializedDAG.from_dict(serialized)

 # Verify schema defaults are restored
 assert deserialized_dag.catchup is False
 assert deserialized_dag.fail_fast is False
 assert deserialized_dag.max_active_runs == 16
 assert deserialized_dag.max_active_tasks == 16
 assert deserialized_dag.max_consecutive_failed_dag_runs == 0
 assert deserialized_dag.render_template_as_native_obj is False
 assert deserialized_dag.disable_bundle_versioning is False

 # Test with non-default values (should be included)
 dag_non_defaults = DAG(
 dag_id="test_non_defaults_dag",
 start_date=datetime(2023, 1, 1),
 catchup=True, # Non-default
 max_active_runs=32, # Non-default
 description="Test description", # Non-None
 )

 serialized_non_defaults = SerializedDAG.to_dict(dag_non_defaults)
 dag_non_defaults_data = serialized_non_defaults["dag"]

 # Non-default values should be included
 assert "catchup" in dag_non_defaults_data
 assert dag_non_defaults_data["catchup"] is True
 assert "max_active_runs" in dag_non_defaults_data
 assert dag_non_defaults_data["max_active_runs"] == 32
 assert "description" in dag_non_defaults_data
 assert dag_non_defaults_data["description"] == "Test description"


def test_email_optimization_removes_email_attrs_when_email_empty():
 """Test that email_on_failure and email_on_retry are removed when email is empty."""
 with DAG(dag_id="test_email_optimization") as dag:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Optimize DAG serialization by excluding schema default values#55849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

kaxil
merged 2 commits into

from

Sep 18, 2025
 Merged


Optimize DAG serialization by excluding schema default values
#55849
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
589a4e6

Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025
a8fd7ea

fixup! Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 9 additions & 9 deletions
airflow-core/src/airflow/serialization/schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,8 +175,8 @@
 "value": { "$ref": "#/definitions/dict" }
 }
 },
 "catchup": { "type": "boolean" },
 "fail_fast": { "type": "boolean" },
 "catchup": { "type": "boolean", "default": false },
 "fail_fast": { "type": "boolean", "default": false },
 "fileloc": { "type" : "string"},
 "relative_fileloc": { "type" : "string"},
 "_processor_dags_folder": {
Expand All@@ -198,27 +198,27 @@
 ]
 },
 "_concurrency": { "type" : "number"},
 "max_active_tasks": { "type" : "number"},
 "max_active_runs": { "type" : "number"},
 "max_consecutive_failed_dag_runs": { "type" : "number"},
 "max_active_tasks": { "type" : "number", "default": 16},
 "max_active_runs": { "type" : "number", "default": 16},
 "max_consecutive_failed_dag_runs": { "type" : "number", "default": 0},
 "default_args": { "$ref": "#/definitions/dict" },
 "start_date": { "$ref": "#/definitions/datetime" },
 "end_date": { "$ref": "#/definitions/datetime" },
 "dagrun_timeout": { "$ref": "#/definitions/timedelta" },
 "doc_md": { "type" : "string"},
 "access_control": {"$ref": "#/definitions/dict" },
 "is_paused_upon_creation": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean" },
 "has_on_failure_callback": { "type": "boolean" },
 "render_template_as_native_obj": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean", "default": false },
 "has_on_failure_callback": { "type": "boolean", "default": false },
 "render_template_as_native_obj": { "type": "boolean", "default": false },
 "tags": { "type": "array" },
 "task_group": {"anyOf": [
 { "type": "null" },
 { "$ref": "#/definitions/task_group" }
 ]},
 "edge_info": { "$ref": "#/definitions/edge_info" },
 "dag_dependencies": { "$ref": "#/definitions/dag_dependencies" },
 "disable_bundle_versioning": {"type": "boolean"}
 "disable_bundle_versioning": {"type": "boolean", "default": false }
 },
 "required": [
 "dag_id",
Expand Down

80 changes: 55 additions & 25 deletions
airflow-core/src/airflow/serialization/serialized_objects.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1094,21 +1094,7 @@ def _deserialize_params_dict(cls, encoded_params: list[tuple[str, dict]]) -> Par
 return ParamsDict(op_params)

 @classmethod
Comment thread
kaxil marked this conversation as resolved.

 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 @lru_cache(maxsize=4) # Cache for "operator", "dag", and a few others
 def get_schema_defaults(cls, object_type: str) -> dict[str, Any]:
 """
 Extract default values from JSON schema for any object type.
Expand DownExpand Up@@ -1713,6 +1699,22 @@ def set_task_dag_references(task: SerializedOperator | MappedOperator, dag: Seri
 # Bypass set_upstream etc here - it does more than we want
 dag.task_dict[task_id].upstream_task_ids.add(task.task_id)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "operator"
 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def deserialize_operator(
 cls,
Expand DownExpand Up@@ -1814,7 +1816,7 @@ def detect_dependencies(cls, op: SdkOperator) -> set[DagDependency]:
 return deps

 @classmethod
 def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:
 def _matches_client_defaults(cls, var: Any, attrname: str) -> bool:
 """
 Check if a field value matches client_defaults and should be excluded.

Expand All@@ -1823,7 +1825,6 @@ def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:

 :param var: The value to check
 :param attrname: The attribute name
 :param op: The operator instance
 :return: True if value matches client_defaults and should be excluded
 """
 try:
Expand DownExpand Up@@ -1851,7 +1852,7 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 :return: True if a variable is excluded, False otherwise.
 """
 # Check if value matches client_defaults (hierarchical defaults optimization)
 if cls._matches_client_defaults(var, attrname, op):
 if cls._matches_client_defaults(var, attrname):
 return True
 schema_defaults = cls.get_schema_defaults("operator")

Expand DownExpand Up@@ -2384,25 +2385,23 @@ class SerializedDAG(BaseSerialization):
 _processor_dags_folder: str

 def __init__(self, *, dag_id: str) -> None:
 self.catchup = airflow_conf.getboolean("scheduler", "catchup_by_default")
 self.catchup = False # Schema default
 self.dag_id = self.dag_display_name = dag_id
 self.dagrun_timeout = None
 self.deadline = None
 self.default_args = {}
 self.description = None
 self.disable_bundle_versioning = airflow_conf.getboolean("dag_processor", "disable_bundle_versioning")
 self.disable_bundle_versioning = False
 self.doc_md = None
 self.edge_info = {}
 self.end_date = None
 self.fail_fast = False
 self.has_on_failure_callback = False
 self.has_on_success_callback = False
 self.is_paused_upon_creation = None
 self.max_active_runs = airflow_conf.getint("core", "max_active_runs_per_dag")
Comment thread
kaxil marked this conversation as resolved.

 self.max_active_tasks = airflow_conf.getint("core", "max_active_tasks_per_dag")
 self.max_consecutive_failed_dag_runs = airflow_conf.getint(
 "core", "max_consecutive_failed_dag_runs_per_dag"
 )
 self.max_active_runs = 16 # Schema default
 self.max_active_tasks = 16 # Schema default
 self.max_consecutive_failed_dag_runs = 0 # Schema default
 self.owner_links = {}
 self.params = ParamsDict()
 self.partial = False
Expand DownExpand Up@@ -2624,8 +2623,38 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 return False
 if attrname == "dag_display_name" and var == op.dag_id:
 return True

 # DAG schema defaults exclusion (same pattern as SerializedBaseOperator)
 dag_schema_defaults = cls.get_schema_defaults("dag")
 if attrname in dag_schema_defaults:
 if dag_schema_defaults[attrname] == var:
 return True

 optional_fields = cls.get_dag_optional_fields_from_schema()
 if var is None:
 return True
 if attrname in optional_fields:
 if var in [[], (), set(), {}]:
 return True

 return super()._is_excluded(var, attrname, op)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "dag"
 def get_dag_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("dag", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def to_dict(cls, var: Any) -> dict:
 """Stringifies DAGs and operators contained by var and returns a dict of var."""
Expand DownExpand Up@@ -3798,6 +3827,7 @@ class LazyDeserializedDAG(pydantic.BaseModel):
 "dag_display_name",
 "has_on_success_callback",
 "has_on_failure_callback",
 "tags",
 # Attr properties that are nullable, or have a default that loads from config
 "description",
 "start_date",
Expand Down

110 changes: 94 additions & 16 deletions
airflow-core/tests/unit/serialization/test_dag_serialization.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,13 +152,8 @@
 "downstream_task_ids": [],
 },
 "is_paused_upon_creation": False,
 "max_active_runs": 16,
 "max_active_tasks": 16,
 "max_consecutive_failed_dag_runs": 0,
 "dag_id": "simple_dag",
 "deadline": None,
 "catchup": False,
 "disable_bundle_versioning": False,
 "doc_md": "### DAG Tutorial Documentation",
 "fileloc": None,
 "_processor_dags_folder": (
Expand DownExpand Up@@ -269,7 +264,6 @@
 },
 ],
 "params": [],
 "tags": [],
 },
}

Expand DownExpand Up@@ -2177,7 +2171,8 @@ def test_dag_disable_bundle_versioning_roundtrip(self, dag_arg, conf_arg, expect
 """
 with conf_vars({("dag_processor", "disable_bundle_versioning"): conf_arg}):
 kwargs = {}
 kwargs["disable_bundle_versioning"] = dag_arg
 if dag_arg is not None:
 kwargs["disable_bundle_versioning"] = dag_arg
 dag = DAG(
 dag_id="test_dag_disable_bundle_versioning_roundtrip",
 schedule=None,
Expand DownExpand Up@@ -3299,17 +3294,34 @@ def test_handle_v1_serdag():
 SerializedDAG.conversion_v1_to_v2(v1)
 SerializedDAG.conversion_v2_to_v3(v1)

 # Update a few subtle differences
 v1["dag"]["tags"] = []
 v1["dag"]["catchup"] = False
 v1["dag"]["disable_bundle_versioning"] = False
 dag = SerializedDAG.from_dict(v1)

 expected = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected["dag"]["dag_dependencies"] = expected_dag_dependencies
 del expected["dag"]["tasks"][1]["__var"]["_operator_extra_links"]
 expected_sdag = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected = SerializedDAG.from_dict(expected_sdag)

 fields_to_verify = set(vars(expected).keys()) - {
 "task_group", # Tested separately
 "dag_dependencies", # Tested separately
 "last_loaded", # Dynamically set to utcnow
 }

 for f in fields_to_verify:
 dag_value = getattr(dag, f)
 expected_value = getattr(expected, f)

 assert dag_value == expected_value, (
 f"V2 DAG field '{f}' differs from V3: V2={dag_value!r} != V3={expected_value!r}"
 )

 for f in set(vars(expected.task_group).keys()) - {"dag"}:
 dag_tg_value = getattr(dag.task_group, f)
 expected_tg_value = getattr(expected.task_group, f)

 assert dag_tg_value == expected_tg_value, (
 f"V2 task_group field '{f}' differs: V2={dag_tg_value!r} != V3={expected_tg_value!r}"
 )

 del expected["client_defaults"]
 assert v1 == expected
 assert getattr(dag, "dag_dependencies") == expected_dag_dependencies


def test_handle_v2_serdag():
Expand DownExpand Up@@ -3514,6 +3526,72 @@ def test_handle_v2_serdag():
 )


def test_dag_schema_defaults_optimization():
 """Test that DAG fields matching schema defaults are excluded from serialization."""

 # Create DAG with all schema default values
 dag_with_defaults = DAG(
 dag_id="test_defaults_dag",
 start_date=datetime(2023, 1, 1),
 # These should match schema defaults and be excluded
 catchup=False,
 fail_fast=False,
 max_active_runs=16,
 max_active_tasks=16,
 max_consecutive_failed_dag_runs=0,
 render_template_as_native_obj=False,
 disable_bundle_versioning=False,
 # These should be excluded as None
 description=None,
 doc_md=None,
 )

 # Serialize and check exclusions
 serialized = SerializedDAG.to_dict(dag_with_defaults)
 dag_data = serialized["dag"]

 # Schema default fields should be excluded
 for field in SerializedDAG.get_schema_defaults("dag").keys():
 assert field not in dag_data, f"Schema default field '{field}' should be excluded"

 # None fields should also be excluded
 none_fields = ["description", "doc_md"]
 for field in none_fields:
 assert field not in dag_data, f"None field '{field}' should be excluded"

 # Test deserialization restores defaults correctly
 deserialized_dag = SerializedDAG.from_dict(serialized)

 # Verify schema defaults are restored
 assert deserialized_dag.catchup is False
 assert deserialized_dag.fail_fast is False
 assert deserialized_dag.max_active_runs == 16
 assert deserialized_dag.max_active_tasks == 16
 assert deserialized_dag.max_consecutive_failed_dag_runs == 0
 assert deserialized_dag.render_template_as_native_obj is False
 assert deserialized_dag.disable_bundle_versioning is False

 # Test with non-default values (should be included)
 dag_non_defaults = DAG(
 dag_id="test_non_defaults_dag",
 start_date=datetime(2023, 1, 1),
 catchup=True, # Non-default
 max_active_runs=32, # Non-default
 description="Test description", # Non-None
 )

 serialized_non_defaults = SerializedDAG.to_dict(dag_non_defaults)
 dag_non_defaults_data = serialized_non_defaults["dag"]

 # Non-default values should be included
 assert "catchup" in dag_non_defaults_data
 assert dag_non_defaults_data["catchup"] is True
 assert "max_active_runs" in dag_non_defaults_data
 assert dag_non_defaults_data["max_active_runs"] == 32
 assert "description" in dag_non_defaults_data
 assert dag_non_defaults_data["description"] == "Test description"


def test_email_optimization_removes_email_attrs_when_email_empty():
 """Test that email_on_failure and email_on_retry are removed when email is empty."""
 with DAG(dag_id="test_email_optimization") as dag:
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Optimize DAG serialization by excluding schema default values#55849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

kaxil
merged 2 commits into

from

Sep 18, 2025
 Merged


Optimize DAG serialization by excluding schema default values
#55849
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
589a4e6

Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025
a8fd7ea

fixup! Optimize DAG serialization by excluding schema default values


kaxil Sep 18, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


18 changes: 9 additions & 9 deletions
airflow-core/src/airflow/serialization/schema.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -175,8 +175,8 @@
 "value": { "$ref": "#/definitions/dict" }
 }
 },
 "catchup": { "type": "boolean" },
 "fail_fast": { "type": "boolean" },
 "catchup": { "type": "boolean", "default": false },
 "fail_fast": { "type": "boolean", "default": false },
 "fileloc": { "type" : "string"},
 "relative_fileloc": { "type" : "string"},
 "_processor_dags_folder": {
Expand All@@ -198,27 +198,27 @@
 ]
 },
 "_concurrency": { "type" : "number"},
 "max_active_tasks": { "type" : "number"},
 "max_active_runs": { "type" : "number"},
 "max_consecutive_failed_dag_runs": { "type" : "number"},
 "max_active_tasks": { "type" : "number", "default": 16},
 "max_active_runs": { "type" : "number", "default": 16},
 "max_consecutive_failed_dag_runs": { "type" : "number", "default": 0},
 "default_args": { "$ref": "#/definitions/dict" },
 "start_date": { "$ref": "#/definitions/datetime" },
 "end_date": { "$ref": "#/definitions/datetime" },
 "dagrun_timeout": { "$ref": "#/definitions/timedelta" },
 "doc_md": { "type" : "string"},
 "access_control": {"$ref": "#/definitions/dict" },
 "is_paused_upon_creation": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean" },
 "has_on_failure_callback": { "type": "boolean" },
 "render_template_as_native_obj": { "type": "boolean" },
 "has_on_success_callback": { "type": "boolean", "default": false },
 "has_on_failure_callback": { "type": "boolean", "default": false },
 "render_template_as_native_obj": { "type": "boolean", "default": false },
 "tags": { "type": "array" },
 "task_group": {"anyOf": [
 { "type": "null" },
 { "$ref": "#/definitions/task_group" }
 ]},
 "edge_info": { "$ref": "#/definitions/edge_info" },
 "dag_dependencies": { "$ref": "#/definitions/dag_dependencies" },
 "disable_bundle_versioning": {"type": "boolean"}
 "disable_bundle_versioning": {"type": "boolean", "default": false }
 },
 "required": [
 "dag_id",
Expand Down

80 changes: 55 additions & 25 deletions
airflow-core/src/airflow/serialization/serialized_objects.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1094,21 +1094,7 @@ def _deserialize_params_dict(cls, encoded_params: list[tuple[str, dict]]) -> Par
 return ParamsDict(op_params)

 @classmethod
Comment thread
kaxil marked this conversation as resolved.

 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 @lru_cache(maxsize=4) # Cache for "operator", "dag", and a few others
 def get_schema_defaults(cls, object_type: str) -> dict[str, Any]:
 """
 Extract default values from JSON schema for any object type.
Expand DownExpand Up@@ -1713,6 +1699,22 @@ def set_task_dag_references(task: SerializedOperator | MappedOperator, dag: Seri
 # Bypass set_upstream etc here - it does more than we want
 dag.task_dict[task_id].upstream_task_ids.add(task.task_id)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "operator"
 def get_operator_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("operator", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def deserialize_operator(
 cls,
Expand DownExpand Up@@ -1814,7 +1816,7 @@ def detect_dependencies(cls, op: SdkOperator) -> set[DagDependency]:
 return deps

 @classmethod
 def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:
 def _matches_client_defaults(cls, var: Any, attrname: str) -> bool:
 """
 Check if a field value matches client_defaults and should be excluded.

Expand All@@ -1823,7 +1825,6 @@ def _matches_client_defaults(cls, var: Any, attrname: str, op: DAGNode) -> bool:

 :param var: The value to check
 :param attrname: The attribute name
 :param op: The operator instance
 :return: True if value matches client_defaults and should be excluded
 """
 try:
Expand DownExpand Up@@ -1851,7 +1852,7 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 :return: True if a variable is excluded, False otherwise.
 """
 # Check if value matches client_defaults (hierarchical defaults optimization)
 if cls._matches_client_defaults(var, attrname, op):
 if cls._matches_client_defaults(var, attrname):
 return True
 schema_defaults = cls.get_schema_defaults("operator")

Expand DownExpand Up@@ -2384,25 +2385,23 @@ class SerializedDAG(BaseSerialization):
 _processor_dags_folder: str

 def __init__(self, *, dag_id: str) -> None:
 self.catchup = airflow_conf.getboolean("scheduler", "catchup_by_default")
 self.catchup = False # Schema default
 self.dag_id = self.dag_display_name = dag_id
 self.dagrun_timeout = None
 self.deadline = None
 self.default_args = {}
 self.description = None
 self.disable_bundle_versioning = airflow_conf.getboolean("dag_processor", "disable_bundle_versioning")
 self.disable_bundle_versioning = False
 self.doc_md = None
 self.edge_info = {}
 self.end_date = None
 self.fail_fast = False
 self.has_on_failure_callback = False
 self.has_on_success_callback = False
 self.is_paused_upon_creation = None
 self.max_active_runs = airflow_conf.getint("core", "max_active_runs_per_dag")
Comment thread
kaxil marked this conversation as resolved.

 self.max_active_tasks = airflow_conf.getint("core", "max_active_tasks_per_dag")
 self.max_consecutive_failed_dag_runs = airflow_conf.getint(
 "core", "max_consecutive_failed_dag_runs_per_dag"
 )
 self.max_active_runs = 16 # Schema default
 self.max_active_tasks = 16 # Schema default
 self.max_consecutive_failed_dag_runs = 0 # Schema default
 self.owner_links = {}
 self.params = ParamsDict()
 self.partial = False
Expand DownExpand Up@@ -2624,8 +2623,38 @@ def _is_excluded(cls, var: Any, attrname: str, op: DAGNode):
 return False
 if attrname == "dag_display_name" and var == op.dag_id:
 return True

 # DAG schema defaults exclusion (same pattern as SerializedBaseOperator)
 dag_schema_defaults = cls.get_schema_defaults("dag")
 if attrname in dag_schema_defaults:
 if dag_schema_defaults[attrname] == var:
 return True

 optional_fields = cls.get_dag_optional_fields_from_schema()
 if var is None:
 return True
 if attrname in optional_fields:
 if var in [[], (), set(), {}]:
 return True

 return super()._is_excluded(var, attrname, op)

 @classmethod
 @lru_cache(maxsize=1) # Only one type: "dag"
 def get_dag_optional_fields_from_schema(cls) -> set[str]:
 schema_loader = cls._json_schema

 if schema_loader is None:
 return set()

 schema_data = schema_loader.schema
 operator_def = schema_data.get("definitions", {}).get("dag", {})
 operator_fields = set(operator_def.get("properties", {}).keys())
 required_fields = set(operator_def.get("required", []))

 optional_fields = operator_fields - required_fields
 return optional_fields

 @classmethod
 def to_dict(cls, var: Any) -> dict:
 """Stringifies DAGs and operators contained by var and returns a dict of var."""
Expand DownExpand Up@@ -3798,6 +3827,7 @@ class LazyDeserializedDAG(pydantic.BaseModel):
 "dag_display_name",
 "has_on_success_callback",
 "has_on_failure_callback",
 "tags",
 # Attr properties that are nullable, or have a default that loads from config
 "description",
 "start_date",
Expand Down

110 changes: 94 additions & 16 deletions
airflow-core/tests/unit/serialization/test_dag_serialization.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -152,13 +152,8 @@
 "downstream_task_ids": [],
 },
 "is_paused_upon_creation": False,
 "max_active_runs": 16,
 "max_active_tasks": 16,
 "max_consecutive_failed_dag_runs": 0,
 "dag_id": "simple_dag",
 "deadline": None,
 "catchup": False,
 "disable_bundle_versioning": False,
 "doc_md": "### DAG Tutorial Documentation",
 "fileloc": None,
 "_processor_dags_folder": (
Expand DownExpand Up@@ -269,7 +264,6 @@
 },
 ],
 "params": [],
 "tags": [],
 },
}

Expand DownExpand Up@@ -2177,7 +2171,8 @@ def test_dag_disable_bundle_versioning_roundtrip(self, dag_arg, conf_arg, expect
 """
 with conf_vars({("dag_processor", "disable_bundle_versioning"): conf_arg}):
 kwargs = {}
 kwargs["disable_bundle_versioning"] = dag_arg
 if dag_arg is not None:
 kwargs["disable_bundle_versioning"] = dag_arg
 dag = DAG(
 dag_id="test_dag_disable_bundle_versioning_roundtrip",
 schedule=None,
Expand DownExpand Up@@ -3299,17 +3294,34 @@ def test_handle_v1_serdag():
 SerializedDAG.conversion_v1_to_v2(v1)
 SerializedDAG.conversion_v2_to_v3(v1)

 # Update a few subtle differences
 v1["dag"]["tags"] = []
 v1["dag"]["catchup"] = False
 v1["dag"]["disable_bundle_versioning"] = False
 dag = SerializedDAG.from_dict(v1)

 expected = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected["dag"]["dag_dependencies"] = expected_dag_dependencies
 del expected["dag"]["tasks"][1]["__var"]["_operator_extra_links"]
 expected_sdag = copy.deepcopy(serialized_simple_dag_ground_truth)
 expected = SerializedDAG.from_dict(expected_sdag)

 fields_to_verify = set(vars(expected).keys()) - {
 "task_group", # Tested separately
 "dag_dependencies", # Tested separately
 "last_loaded", # Dynamically set to utcnow
 }

 for f in fields_to_verify:
 dag_value = getattr(dag, f)
 expected_value = getattr(expected, f)

 assert dag_value == expected_value, (
 f"V2 DAG field '{f}' differs from V3: V2={dag_value!r} != V3={expected_value!r}"
 )

 for f in set(vars(expected.task_group).keys()) - {"dag"}:
 dag_tg_value = getattr(dag.task_group, f)
 expected_tg_value = getattr(expected.task_group, f)

 assert dag_tg_value == expected_tg_value, (
 f"V2 task_group field '{f}' differs: V2={dag_tg_value!r} != V3={expected_tg_value!r}"
 )

 del expected["client_defaults"]
 assert v1 == expected
 assert getattr(dag, "dag_dependencies") == expected_dag_dependencies


def test_handle_v2_serdag():
Expand DownExpand Up@@ -3514,6 +3526,72 @@ def test_handle_v2_serdag():
 )


def test_dag_schema_defaults_optimization():
 """Test that DAG fields matching schema defaults are excluded from serialization."""

 # Create DAG with all schema default values
 dag_with_defaults = DAG(
 dag_id="test_defaults_dag",
 start_date=datetime(2023, 1, 1),
 # These should match schema defaults and be excluded
 catchup=False,
 fail_fast=False,
 max_active_runs=16,
 max_active_tasks=16,
 max_consecutive_failed_dag_runs=0,
 render_template_as_native_obj=False,
 disable_bundle_versioning=False,
 # These should be excluded as None
 description=None,
 doc_md=None,
 )

 # Serialize and check exclusions
 serialized = SerializedDAG.to_dict(dag_with_defaults)
 dag_data = serialized["dag"]

 # Schema default fields should be excluded
 for field in SerializedDAG.get_schema_defaults("dag").keys():
 assert field not in dag_data, f"Schema default field '{field}' should be excluded"

 # None fields should also be excluded
 none_fields = ["description", "doc_md"]
 for field in none_fields:
 assert field not in dag_data, f"None field '{field}' should be excluded"

 # Test deserialization restores defaults correctly
 deserialized_dag = SerializedDAG.from_dict(serialized)

 # Verify schema defaults are restored
 assert deserialized_dag.catchup is False
 assert deserialized_dag.fail_fast is False
 assert deserialized_dag.max_active_runs == 16
 assert deserialized_dag.max_active_tasks == 16
 assert deserialized_dag.max_consecutive_failed_dag_runs == 0
 assert deserialized_dag.render_template_as_native_obj is False
 assert deserialized_dag.disable_bundle_versioning is False

 # Test with non-default values (should be included)
 dag_non_defaults = DAG(
 dag_id="test_non_defaults_dag",
 start_date=datetime(2023, 1, 1),
 catchup=True, # Non-default
 max_active_runs=32, # Non-default
 description="Test description", # Non-None
 )

 serialized_non_defaults = SerializedDAG.to_dict(dag_non_defaults)
 dag_non_defaults_data = serialized_non_defaults["dag"]

 # Non-default values should be included
 assert "catchup" in dag_non_defaults_data
 assert dag_non_defaults_data["catchup"] is True
 assert "max_active_runs" in dag_non_defaults_data
 assert dag_non_defaults_data["max_active_runs"] == 32
 assert "description" in dag_non_defaults_data
 assert dag_non_defaults_data["description"] == "Test description"


def test_email_optimization_removes_email_attrs_when_email_empty():
 """Test that email_on_failure and email_on_retry are removed when email is empty."""
 with DAG(dag_id="test_email_optimization") as dag:
Expand Down
Loading