Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

from airflow._shared.timezones import timezone
from airflow.api_fastapi.core_api.base import BaseModel
from airflow.api_fastapi.core_api.datamodels.dag_versions import DagVersionResponse
from airflow.utils.state import DagRunState
from airflow.utils.types import DagRunType

Expand DownExpand Up@@ -79,6 +80,7 @@ class GridRunsResponse(BaseModel):
run_after: datetime
state: DagRunState | None
run_type: DagRunType
dag_versions: list[DagVersionResponse] = []
has_missed_deadline: bool

@computed_field
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,6 +32,7 @@ class LightGridTaskInstanceSummary(BaseModel):
child_states: dict[TaskInstanceState | None, int] | None
min_start_date: datetime | None
max_end_date: datetime | None
dag_version_number: int | None = None


class GridTISummaries(BaseModel):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -2183,6 +2183,12 @@ components:
- type: 'null'
run_type:
$ref: '#/components/schemas/DagRunType'
dag_versions:
items:
$ref: '#/components/schemas/DagVersionResponse'
type: array
title: Dag Versions
default: []
has_missed_deadline:
type: boolean
title: Has Missed Deadline
Expand DownExpand Up@@ -2453,6 +2459,11 @@ components:
format: date-time
- type: 'null'
title: Max End Date
dag_version_number:
anyOf:
- type: integer
- type: 'null'
title: Dag Version Number
type: object
required:
- task_id
Expand Down
53 changes: 40 additions & 13 deletions airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@
import structlog
from fastapi import Depends, HTTPException, status
from sqlalchemy import exists, select
from sqlalchemy.orm import joinedload
from sqlalchemy.orm import joinedload, load_only, selectinload

from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
from airflow.api_fastapi.common.db.common import SessionDep, paginated_select
Expand DownExpand Up@@ -58,11 +58,13 @@
get_task_group_children_getter,
task_group_to_dict_grid,
)
from airflow.models.dag import DagModel
from airflow.models.dag_version import DagVersion
from airflow.models.dagrun import DagRun
from airflow.models.deadline import Deadline
from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.taskinstance import TaskInstance
from airflow.models.taskinstancehistory import TaskInstanceHistory

log = structlog.get_logger(logger_name=__name__)
grid_router = AirflowRouter(prefix="/grid", tags=["Grid"])
Expand DownExpand Up@@ -282,17 +284,33 @@ def get_grid_runs(
.correlate(DagRun)
.label("has_missed_deadline")
)
base_query = select(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
has_missed_deadline,
).where(DagRun.dag_id == dag_id)
base_query = (
select(DagRun, has_missed_deadline)
.where(DagRun.dag_id == dag_id)
.options(
load_only(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
DagRun.bundle_version,
),
joinedload(DagRun.dag_model).load_only(DagModel._dag_display_property_value),
joinedload(DagRun.created_dag_version).joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances)
.load_only(TaskInstance.dag_version_id)
.joinedload(TaskInstance.dag_version)
.joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances_histories)
.load_only(TaskInstanceHistory.dag_version_id)
.joinedload(TaskInstanceHistory.dag_version)
.joinedload(DagVersion.bundle),
)
)

# This comparison is to fall back to DAG timetable when no order_by is provided
if order_by.value == [order_by.get_primary_key_string()]:
Expand All@@ -309,8 +327,14 @@ def get_grid_runs(
offset=offset,
filters=[run_after, run_type, state, triggering_user],
limit=limit,
return_total_entries=False,
)
return [GridRunsResponse(**row._mapping) for row in session.execute(dag_runs_select_filter)]
results = session.execute(dag_runs_select_filter).unique().all()
grid_runs = []
for run, has_missed in results:
run.has_missed_deadline = has_missed
grid_runs.append(GridRunsResponse.model_validate(run, from_attributes=True))
return grid_runs


@grid_router.get(
Expand DownExpand Up@@ -363,7 +387,9 @@ def get_grid_ti_summaries(
TaskInstance.dag_version_id,
TaskInstance.start_date,
TaskInstance.end_date,
DagVersion.version_number,
)
.outerjoin(DagVersion, TaskInstance.dag_version_id == DagVersion.id)
.where(TaskInstance.dag_id == dag_id)
.where(
TaskInstance.run_id == run_id,
Expand All@@ -386,6 +412,7 @@ def get_grid_ti_summaries(
"state": ti.state,
"start_date": ti.start_date,
"end_date": ti.end_date,
"dag_version_number": ti.version_number,
}
)
serdag = _get_serdag(
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -72,11 +72,18 @@ def _get_aggs_for_node(detail):
max_end_date = max(x["end_date"] for x in detail if x["end_date"])
except ValueError:
max_end_date = None

dag_version_numbers = [
x.get("dag_version_number") for x in detail if x.get("dag_version_number") is not None
]
dag_version_number = max(dag_version_numbers) if dag_version_numbers else None

return {
"state": agg_state(states),
"min_start_date": min_start_date,
"max_end_date": max_end_date,
"child_states": dict(Counter(states)),
"dag_version_number": dag_version_number,
}


Expand Down
19 changes: 19 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -8143,6 +8143,14 @@ export const $GridRunsResponse = {
run_type: {
'$ref': '#/components/schemas/DagRunType'
},
dag_versions: {
items: {
'$ref': '#/components/schemas/DagVersionResponse'
},
type: 'array',
title: 'Dag Versions',
default: []
},
has_missed_deadline: {
type: 'boolean',
title: 'Has Missed Deadline'
Expand DownExpand Up@@ -8258,6 +8266,17 @@ export const $LightGridTaskInstanceSummary = {
}
],
title: 'Max End Date'
},
dag_version_number: {
anyOf: [
{
type: 'integer'
},
{
type: 'null'
}
],
title: 'Dag Version Number'
}
},
type: 'object',
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -1999,6 +1999,7 @@ export type GridRunsResponse = {
run_after: string;
state: DagRunState | null;
run_type: DagRunType;
dag_versions?: Array<DagVersionResponse>;
has_missed_deadline: boolean;
readonly duration: number;
};
Expand DownExpand Up@@ -2033,6 +2034,7 @@ export type LightGridTaskInstanceSummary = {
} | null;
min_start_date: string | null;
max_end_date: string | null;
dag_version_number?: number | null;
};

/**
Expand Down
9 changes: 9 additions & 0 deletions airflow-core/src/airflow/ui/public/i18n/locales/en/dag.json
Original file line numberDiff line numberDiff line change
Expand Up@@ -121,6 +121,15 @@
"graphDirection": {
"label": "Graph Direction"
},
"showVersionIndicator": {
"label": "Show Version Indicator",
"options": {
"hideAll": "Hide All",
"showAll": "Show All",
"showBundleVersion": "Show Bundle Version",
"showDagVersion": "Show Dag Version"
}
},
"taskStreamFilter": {
"activeFilter": "Active filter",
"clearFilter": "Clear Filter",
Expand Down
1 change: 1 addition & 0 deletions airflow-core/src/airflow/ui/src/constants/localStorage.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ export const CALENDAR_VIEW_MODE_KEY = "calendar-view-mode";
export const LOG_WRAP_KEY = "log_wrap";
export const LOG_SHOW_TIMESTAMP_KEY = "log_show_timestamp";
export const LOG_SHOW_SOURCE_KEY = "log_show_source";
export const VERSION_INDICATOR_DISPLAY_MODE_KEY = "version_indicator_display_mode";

// Dag-scoped keys
export const dagViewKey = (dagId: string) => `dag_view-${dagId}`;
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { createListCollection } from "@chakra-ui/react";

export enum VersionIndicatorOptions {
ALL = "all",
BUNDLE_VERSION = "bundle",
DAG_VERSION = "dag",
NONE = "none",
}

const validOptions = new Set<string>(Object.values(VersionIndicatorOptions));

export const isVersionIndicatorOption = (value: unknown): value is VersionIndicatorOptions =>
typeof value === "string" && validOptions.has(value);

export const showVersionIndicatorOptions = createListCollection({
items: [
{ label: "dag:panel.showVersionIndicator.options.showAll", value: VersionIndicatorOptions.ALL },
{
label: "dag:panel.showVersionIndicator.options.showBundleVersion",
value: VersionIndicatorOptions.BUNDLE_VERSION,
},
{
label: "dag:panel.showVersionIndicator.options.showDagVersion",
value: VersionIndicatorOptions.DAG_VERSION,
},
{ label: "dag:panel.showVersionIndicator.options.hideAll", value: VersionIndicatorOptions.NONE },
],
});
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
/* eslint-disable max-lines */

/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
Expand DownExpand Up@@ -50,6 +52,7 @@ import {
showGanttKey,
triggeringUserFilterKey,
} from "src/constants/localStorage";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { HoverProvider } from "src/context/hover";
import { OpenGroupsProvider } from "src/context/openGroups";

Expand DownExpand Up@@ -88,6 +91,11 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
);

const [showGantt, setShowGantt] = useLocalStorage<boolean>(showGanttKey(dagId), false);
// Global setting: applies to all Dags (intentionally not scoped to dagId)
const [showVersionIndicatorMode, setShowVersionIndicatorMode] = useLocalStorage<VersionIndicatorOptions>(
`version_indicator_display_mode`,
VersionIndicatorOptions.ALL,
);
const { fitView, getZoom } = useReactFlow();
const { data: warningData } = useDagWarningServiceListDagWarnings({ dagId });
const { onClose, onOpen, open } = useDisclosure();
Expand DownExpand Up@@ -161,8 +169,10 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
setLimit={setLimit}
setRunTypeFilter={setRunTypeFilter}
setShowGantt={setShowGantt}
setShowVersionIndicatorMode={setShowVersionIndicatorMode}
setTriggeringUserFilter={setTriggeringUserFilter}
showGantt={showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUserFilter={triggeringUserFilter}
/>
{dagView === "graph" ? (
Expand All@@ -174,6 +184,7 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
limit={limit}
runType={runTypeFilter}
showGantt={Boolean(runId) && showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUser={triggeringUserFilter}
/>
{showGantt ? (
Expand Down
27 changes: 22 additions & 5 deletions airflow-core/src/airflow/ui/src/layouts/Details/Grid/Bar.tsx
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,21 +19,27 @@
import { Flex, Box } from "@chakra-ui/react";
import { useParams, useSearchParams } from "react-router-dom";

import type { GridRunsResponse } from "openapi/requests";
import { RunTypeIcon } from "src/components/RunTypeIcon";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { useHover } from "src/context/hover";

import { GridButton } from "./GridButton";

const BAR_HEIGHT = 100;
import { BundleVersionIndicator, DagVersionIndicator } from "./VersionIndicator";
import { BAR_HEIGHT } from "./constants";
import {
getBundleVersion,
getMaxVersionNumber,
type GridRunWithVersionFlags,
} from "./useGridRunsWithVersionFlags";

type Props = {
readonly max: number;
readonly onClick?: () => void;
readonly run: GridRunsResponse;
readonly run: GridRunWithVersionFlags;
readonly showVersionIndicatorMode?: VersionIndicatorOptions;
};

export const Bar = ({ max, onClick, run }: Props) => {
export const Bar = ({ max, onClick, run, showVersionIndicatorMode }: Props) => {
const { dagId = "", runId } = useParams();
const [searchParams] = useSearchParams();
const { hoveredRunId, setHoveredRunId } = useHover();
Expand All@@ -53,6 +59,17 @@ export const Bar = ({ max, onClick, run }: Props) => {
position="relative"
transition="background-color 0.2s"
>
{run.isBundleVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.BUNDLE_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<BundleVersionIndicator bundleVersion={getBundleVersion(run)} />
) : undefined}
{run.isDagVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.DAG_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<DagVersionIndicator dagVersionNumber={getMaxVersionNumber(run)} orientation="vertical" />
) : undefined}

<Flex
alignItems="flex-end"
height={BAR_HEIGHT}
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Feat: Add version change indicators for Dag and bundle versions in Grid view by choo121600 · Pull Request #53216 · apache/airflow · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

from airflow._shared.timezones import timezone
from airflow.api_fastapi.core_api.base import BaseModel
from airflow.api_fastapi.core_api.datamodels.dag_versions import DagVersionResponse
from airflow.utils.state import DagRunState
from airflow.utils.types import DagRunType

Expand DownExpand Up@@ -79,6 +80,7 @@ class GridRunsResponse(BaseModel):
run_after: datetime
state: DagRunState | None
run_type: DagRunType
dag_versions: list[DagVersionResponse] = []
has_missed_deadline: bool

@computed_field
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,6 +32,7 @@ class LightGridTaskInstanceSummary(BaseModel):
child_states: dict[TaskInstanceState | None, int] | None
min_start_date: datetime | None
max_end_date: datetime | None
dag_version_number: int | None = None


class GridTISummaries(BaseModel):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -2183,6 +2183,12 @@ components:
- type: 'null'
run_type:
$ref: '#/components/schemas/DagRunType'
dag_versions:
items:
$ref: '#/components/schemas/DagVersionResponse'
type: array
title: Dag Versions
default: []
has_missed_deadline:
type: boolean
title: Has Missed Deadline
Expand DownExpand Up@@ -2453,6 +2459,11 @@ components:
format: date-time
- type: 'null'
title: Max End Date
dag_version_number:
anyOf:
- type: integer
- type: 'null'
title: Dag Version Number
type: object
required:
- task_id
Expand Down
53 changes: 40 additions & 13 deletions airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@
import structlog
from fastapi import Depends, HTTPException, status
from sqlalchemy import exists, select
from sqlalchemy.orm import joinedload
from sqlalchemy.orm import joinedload, load_only, selectinload

from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
from airflow.api_fastapi.common.db.common import SessionDep, paginated_select
Expand DownExpand Up@@ -58,11 +58,13 @@
get_task_group_children_getter,
task_group_to_dict_grid,
)
from airflow.models.dag import DagModel
from airflow.models.dag_version import DagVersion
from airflow.models.dagrun import DagRun
from airflow.models.deadline import Deadline
from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.taskinstance import TaskInstance
from airflow.models.taskinstancehistory import TaskInstanceHistory

log = structlog.get_logger(logger_name=__name__)
grid_router = AirflowRouter(prefix="/grid", tags=["Grid"])
Expand DownExpand Up@@ -282,17 +284,33 @@ def get_grid_runs(
.correlate(DagRun)
.label("has_missed_deadline")
)
base_query = select(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
has_missed_deadline,
).where(DagRun.dag_id == dag_id)
base_query = (
select(DagRun, has_missed_deadline)
.where(DagRun.dag_id == dag_id)
.options(
load_only(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
DagRun.bundle_version,
),
joinedload(DagRun.dag_model).load_only(DagModel._dag_display_property_value),
joinedload(DagRun.created_dag_version).joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances)
.load_only(TaskInstance.dag_version_id)
.joinedload(TaskInstance.dag_version)
.joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances_histories)
.load_only(TaskInstanceHistory.dag_version_id)
.joinedload(TaskInstanceHistory.dag_version)
.joinedload(DagVersion.bundle),
)
)

# This comparison is to fall back to DAG timetable when no order_by is provided
if order_by.value == [order_by.get_primary_key_string()]:
Expand All@@ -309,8 +327,14 @@ def get_grid_runs(
offset=offset,
filters=[run_after, run_type, state, triggering_user],
limit=limit,
return_total_entries=False,
)
return [GridRunsResponse(**row._mapping) for row in session.execute(dag_runs_select_filter)]
results = session.execute(dag_runs_select_filter).unique().all()
grid_runs = []
for run, has_missed in results:
run.has_missed_deadline = has_missed
grid_runs.append(GridRunsResponse.model_validate(run, from_attributes=True))
return grid_runs


@grid_router.get(
Expand DownExpand Up@@ -363,7 +387,9 @@ def get_grid_ti_summaries(
TaskInstance.dag_version_id,
TaskInstance.start_date,
TaskInstance.end_date,
DagVersion.version_number,
)
.outerjoin(DagVersion, TaskInstance.dag_version_id == DagVersion.id)
.where(TaskInstance.dag_id == dag_id)
.where(
TaskInstance.run_id == run_id,
Expand All@@ -386,6 +412,7 @@ def get_grid_ti_summaries(
"state": ti.state,
"start_date": ti.start_date,
"end_date": ti.end_date,
"dag_version_number": ti.version_number,
}
)
serdag = _get_serdag(
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -72,11 +72,18 @@ def _get_aggs_for_node(detail):
max_end_date = max(x["end_date"] for x in detail if x["end_date"])
except ValueError:
max_end_date = None

dag_version_numbers = [
x.get("dag_version_number") for x in detail if x.get("dag_version_number") is not None
]
dag_version_number = max(dag_version_numbers) if dag_version_numbers else None

return {
"state": agg_state(states),
"min_start_date": min_start_date,
"max_end_date": max_end_date,
"child_states": dict(Counter(states)),
"dag_version_number": dag_version_number,
}


Expand Down
19 changes: 19 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -8143,6 +8143,14 @@ export const $GridRunsResponse = {
run_type: {
'$ref': '#/components/schemas/DagRunType'
},
dag_versions: {
items: {
'$ref': '#/components/schemas/DagVersionResponse'
},
type: 'array',
title: 'Dag Versions',
default: []
},
has_missed_deadline: {
type: 'boolean',
title: 'Has Missed Deadline'
Expand DownExpand Up@@ -8258,6 +8266,17 @@ export const $LightGridTaskInstanceSummary = {
}
],
title: 'Max End Date'
},
dag_version_number: {
anyOf: [
{
type: 'integer'
},
{
type: 'null'
}
],
title: 'Dag Version Number'
}
},
type: 'object',
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -1999,6 +1999,7 @@ export type GridRunsResponse = {
run_after: string;
state: DagRunState | null;
run_type: DagRunType;
dag_versions?: Array<DagVersionResponse>;
has_missed_deadline: boolean;
readonly duration: number;
};
Expand DownExpand Up@@ -2033,6 +2034,7 @@ export type LightGridTaskInstanceSummary = {
} | null;
min_start_date: string | null;
max_end_date: string | null;
dag_version_number?: number | null;
};

/**
Expand Down
9 changes: 9 additions & 0 deletions airflow-core/src/airflow/ui/public/i18n/locales/en/dag.json
Original file line numberDiff line numberDiff line change
Expand Up@@ -121,6 +121,15 @@
"graphDirection": {
"label": "Graph Direction"
},
"showVersionIndicator": {
"label": "Show Version Indicator",
"options": {
"hideAll": "Hide All",
"showAll": "Show All",
"showBundleVersion": "Show Bundle Version",
"showDagVersion": "Show Dag Version"
}
},
"taskStreamFilter": {
"activeFilter": "Active filter",
"clearFilter": "Clear Filter",
Expand Down
1 change: 1 addition & 0 deletions airflow-core/src/airflow/ui/src/constants/localStorage.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ export const CALENDAR_VIEW_MODE_KEY = "calendar-view-mode";
export const LOG_WRAP_KEY = "log_wrap";
export const LOG_SHOW_TIMESTAMP_KEY = "log_show_timestamp";
export const LOG_SHOW_SOURCE_KEY = "log_show_source";
export const VERSION_INDICATOR_DISPLAY_MODE_KEY = "version_indicator_display_mode";

// Dag-scoped keys
export const dagViewKey = (dagId: string) => `dag_view-${dagId}`;
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { createListCollection } from "@chakra-ui/react";

export enum VersionIndicatorOptions {
ALL = "all",
BUNDLE_VERSION = "bundle",
DAG_VERSION = "dag",
NONE = "none",
}

const validOptions = new Set<string>(Object.values(VersionIndicatorOptions));

export const isVersionIndicatorOption = (value: unknown): value is VersionIndicatorOptions =>
typeof value === "string" && validOptions.has(value);

export const showVersionIndicatorOptions = createListCollection({
items: [
{ label: "dag:panel.showVersionIndicator.options.showAll", value: VersionIndicatorOptions.ALL },
{
label: "dag:panel.showVersionIndicator.options.showBundleVersion",
value: VersionIndicatorOptions.BUNDLE_VERSION,
},
{
label: "dag:panel.showVersionIndicator.options.showDagVersion",
value: VersionIndicatorOptions.DAG_VERSION,
},
{ label: "dag:panel.showVersionIndicator.options.hideAll", value: VersionIndicatorOptions.NONE },
],
});
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
/* eslint-disable max-lines */

/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
Expand DownExpand Up@@ -50,6 +52,7 @@ import {
showGanttKey,
triggeringUserFilterKey,
} from "src/constants/localStorage";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { HoverProvider } from "src/context/hover";
import { OpenGroupsProvider } from "src/context/openGroups";

Expand DownExpand Up@@ -88,6 +91,11 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
);

const [showGantt, setShowGantt] = useLocalStorage<boolean>(showGanttKey(dagId), false);
// Global setting: applies to all Dags (intentionally not scoped to dagId)
const [showVersionIndicatorMode, setShowVersionIndicatorMode] = useLocalStorage<VersionIndicatorOptions>(
`version_indicator_display_mode`,
VersionIndicatorOptions.ALL,
);
const { fitView, getZoom } = useReactFlow();
const { data: warningData } = useDagWarningServiceListDagWarnings({ dagId });
const { onClose, onOpen, open } = useDisclosure();
Expand DownExpand Up@@ -161,8 +169,10 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
setLimit={setLimit}
setRunTypeFilter={setRunTypeFilter}
setShowGantt={setShowGantt}
setShowVersionIndicatorMode={setShowVersionIndicatorMode}
setTriggeringUserFilter={setTriggeringUserFilter}
showGantt={showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUserFilter={triggeringUserFilter}
/>
{dagView === "graph" ? (
Expand All@@ -174,6 +184,7 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
limit={limit}
runType={runTypeFilter}
showGantt={Boolean(runId) && showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUser={triggeringUserFilter}
/>
{showGantt ? (
Expand Down
27 changes: 22 additions & 5 deletions airflow-core/src/airflow/ui/src/layouts/Details/Grid/Bar.tsx
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,21 +19,27 @@
import { Flex, Box } from "@chakra-ui/react";
import { useParams, useSearchParams } from "react-router-dom";

import type { GridRunsResponse } from "openapi/requests";
import { RunTypeIcon } from "src/components/RunTypeIcon";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { useHover } from "src/context/hover";

import { GridButton } from "./GridButton";

const BAR_HEIGHT = 100;
import { BundleVersionIndicator, DagVersionIndicator } from "./VersionIndicator";
import { BAR_HEIGHT } from "./constants";
import {
getBundleVersion,
getMaxVersionNumber,
type GridRunWithVersionFlags,
} from "./useGridRunsWithVersionFlags";

type Props = {
readonly max: number;
readonly onClick?: () => void;
readonly run: GridRunsResponse;
readonly run: GridRunWithVersionFlags;
readonly showVersionIndicatorMode?: VersionIndicatorOptions;
};

export const Bar = ({ max, onClick, run }: Props) => {
export const Bar = ({ max, onClick, run, showVersionIndicatorMode }: Props) => {
const { dagId = "", runId } = useParams();
const [searchParams] = useSearchParams();
const { hoveredRunId, setHoveredRunId } = useHover();
Expand All@@ -53,6 +59,17 @@ export const Bar = ({ max, onClick, run }: Props) => {
position="relative"
transition="background-color 0.2s"
>
{run.isBundleVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.BUNDLE_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<BundleVersionIndicator bundleVersion={getBundleVersion(run)} />
) : undefined}
{run.isDagVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.DAG_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<DagVersionIndicator dagVersionNumber={getMaxVersionNumber(run)} orientation="vertical" />
) : undefined}

<Flex
alignItems="flex-end"
height={BAR_HEIGHT}
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Force GitHub README to respect dark mode (function() { var style = document.createElement('style'); style.textContent = ' .markdown-body { color-scheme: dark light; } .markdown-body pre { background: #161b22 !important; } .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; } .markdown-body table th, .markdown-body table td { border-color: #30363d !important; } .markdown-body img { background: #0d1117; } .markdown-body blockquote { border-left-color: #8b949e; } .markdown-body hr { border-color: #30363d; } '; document.head.appendChild(style); })(); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Feat: Add version change indicators for Dag and bundle versions in Grid view by choo121600 · Pull Request #53216 · apache/airflow · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

from airflow._shared.timezones import timezone
from airflow.api_fastapi.core_api.base import BaseModel
from airflow.api_fastapi.core_api.datamodels.dag_versions import DagVersionResponse
from airflow.utils.state import DagRunState
from airflow.utils.types import DagRunType

Expand DownExpand Up@@ -79,6 +80,7 @@ class GridRunsResponse(BaseModel):
run_after: datetime
state: DagRunState | None
run_type: DagRunType
dag_versions: list[DagVersionResponse] = []
has_missed_deadline: bool

@computed_field
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,6 +32,7 @@ class LightGridTaskInstanceSummary(BaseModel):
child_states: dict[TaskInstanceState | None, int] | None
min_start_date: datetime | None
max_end_date: datetime | None
dag_version_number: int | None = None


class GridTISummaries(BaseModel):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -2183,6 +2183,12 @@ components:
- type: 'null'
run_type:
$ref: '#/components/schemas/DagRunType'
dag_versions:
items:
$ref: '#/components/schemas/DagVersionResponse'
type: array
title: Dag Versions
default: []
has_missed_deadline:
type: boolean
title: Has Missed Deadline
Expand DownExpand Up@@ -2453,6 +2459,11 @@ components:
format: date-time
- type: 'null'
title: Max End Date
dag_version_number:
anyOf:
- type: integer
- type: 'null'
title: Dag Version Number
type: object
required:
- task_id
Expand Down
53 changes: 40 additions & 13 deletions airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@
import structlog
from fastapi import Depends, HTTPException, status
from sqlalchemy import exists, select
from sqlalchemy.orm import joinedload
from sqlalchemy.orm import joinedload, load_only, selectinload

from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
from airflow.api_fastapi.common.db.common import SessionDep, paginated_select
Expand DownExpand Up@@ -58,11 +58,13 @@
get_task_group_children_getter,
task_group_to_dict_grid,
)
from airflow.models.dag import DagModel
from airflow.models.dag_version import DagVersion
from airflow.models.dagrun import DagRun
from airflow.models.deadline import Deadline
from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.taskinstance import TaskInstance
from airflow.models.taskinstancehistory import TaskInstanceHistory

log = structlog.get_logger(logger_name=__name__)
grid_router = AirflowRouter(prefix="/grid", tags=["Grid"])
Expand DownExpand Up@@ -282,17 +284,33 @@ def get_grid_runs(
.correlate(DagRun)
.label("has_missed_deadline")
)
base_query = select(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
has_missed_deadline,
).where(DagRun.dag_id == dag_id)
base_query = (
select(DagRun, has_missed_deadline)
.where(DagRun.dag_id == dag_id)
.options(
load_only(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
DagRun.bundle_version,
),
joinedload(DagRun.dag_model).load_only(DagModel._dag_display_property_value),
joinedload(DagRun.created_dag_version).joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances)
.load_only(TaskInstance.dag_version_id)
.joinedload(TaskInstance.dag_version)
.joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances_histories)
.load_only(TaskInstanceHistory.dag_version_id)
.joinedload(TaskInstanceHistory.dag_version)
.joinedload(DagVersion.bundle),
)
)

# This comparison is to fall back to DAG timetable when no order_by is provided
if order_by.value == [order_by.get_primary_key_string()]:
Expand All@@ -309,8 +327,14 @@ def get_grid_runs(
offset=offset,
filters=[run_after, run_type, state, triggering_user],
limit=limit,
return_total_entries=False,
)
return [GridRunsResponse(**row._mapping) for row in session.execute(dag_runs_select_filter)]
results = session.execute(dag_runs_select_filter).unique().all()
grid_runs = []
for run, has_missed in results:
run.has_missed_deadline = has_missed
grid_runs.append(GridRunsResponse.model_validate(run, from_attributes=True))
return grid_runs


@grid_router.get(
Expand DownExpand Up@@ -363,7 +387,9 @@ def get_grid_ti_summaries(
TaskInstance.dag_version_id,
TaskInstance.start_date,
TaskInstance.end_date,
DagVersion.version_number,
)
.outerjoin(DagVersion, TaskInstance.dag_version_id == DagVersion.id)
.where(TaskInstance.dag_id == dag_id)
.where(
TaskInstance.run_id == run_id,
Expand All@@ -386,6 +412,7 @@ def get_grid_ti_summaries(
"state": ti.state,
"start_date": ti.start_date,
"end_date": ti.end_date,
"dag_version_number": ti.version_number,
}
)
serdag = _get_serdag(
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -72,11 +72,18 @@ def _get_aggs_for_node(detail):
max_end_date = max(x["end_date"] for x in detail if x["end_date"])
except ValueError:
max_end_date = None

dag_version_numbers = [
x.get("dag_version_number") for x in detail if x.get("dag_version_number") is not None
]
dag_version_number = max(dag_version_numbers) if dag_version_numbers else None

return {
"state": agg_state(states),
"min_start_date": min_start_date,
"max_end_date": max_end_date,
"child_states": dict(Counter(states)),
"dag_version_number": dag_version_number,
}


Expand Down
19 changes: 19 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -8143,6 +8143,14 @@ export const $GridRunsResponse = {
run_type: {
'$ref': '#/components/schemas/DagRunType'
},
dag_versions: {
items: {
'$ref': '#/components/schemas/DagVersionResponse'
},
type: 'array',
title: 'Dag Versions',
default: []
},
has_missed_deadline: {
type: 'boolean',
title: 'Has Missed Deadline'
Expand DownExpand Up@@ -8258,6 +8266,17 @@ export const $LightGridTaskInstanceSummary = {
}
],
title: 'Max End Date'
},
dag_version_number: {
anyOf: [
{
type: 'integer'
},
{
type: 'null'
}
],
title: 'Dag Version Number'
}
},
type: 'object',
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -1999,6 +1999,7 @@ export type GridRunsResponse = {
run_after: string;
state: DagRunState | null;
run_type: DagRunType;
dag_versions?: Array<DagVersionResponse>;
has_missed_deadline: boolean;
readonly duration: number;
};
Expand DownExpand Up@@ -2033,6 +2034,7 @@ export type LightGridTaskInstanceSummary = {
} | null;
min_start_date: string | null;
max_end_date: string | null;
dag_version_number?: number | null;
};

/**
Expand Down
9 changes: 9 additions & 0 deletions airflow-core/src/airflow/ui/public/i18n/locales/en/dag.json
Original file line numberDiff line numberDiff line change
Expand Up@@ -121,6 +121,15 @@
"graphDirection": {
"label": "Graph Direction"
},
"showVersionIndicator": {
"label": "Show Version Indicator",
"options": {
"hideAll": "Hide All",
"showAll": "Show All",
"showBundleVersion": "Show Bundle Version",
"showDagVersion": "Show Dag Version"
}
},
"taskStreamFilter": {
"activeFilter": "Active filter",
"clearFilter": "Clear Filter",
Expand Down
1 change: 1 addition & 0 deletions airflow-core/src/airflow/ui/src/constants/localStorage.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ export const CALENDAR_VIEW_MODE_KEY = "calendar-view-mode";
export const LOG_WRAP_KEY = "log_wrap";
export const LOG_SHOW_TIMESTAMP_KEY = "log_show_timestamp";
export const LOG_SHOW_SOURCE_KEY = "log_show_source";
export const VERSION_INDICATOR_DISPLAY_MODE_KEY = "version_indicator_display_mode";

// Dag-scoped keys
export const dagViewKey = (dagId: string) => `dag_view-${dagId}`;
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { createListCollection } from "@chakra-ui/react";

export enum VersionIndicatorOptions {
ALL = "all",
BUNDLE_VERSION = "bundle",
DAG_VERSION = "dag",
NONE = "none",
}

const validOptions = new Set<string>(Object.values(VersionIndicatorOptions));

export const isVersionIndicatorOption = (value: unknown): value is VersionIndicatorOptions =>
typeof value === "string" && validOptions.has(value);

export const showVersionIndicatorOptions = createListCollection({
items: [
{ label: "dag:panel.showVersionIndicator.options.showAll", value: VersionIndicatorOptions.ALL },
{
label: "dag:panel.showVersionIndicator.options.showBundleVersion",
value: VersionIndicatorOptions.BUNDLE_VERSION,
},
{
label: "dag:panel.showVersionIndicator.options.showDagVersion",
value: VersionIndicatorOptions.DAG_VERSION,
},
{ label: "dag:panel.showVersionIndicator.options.hideAll", value: VersionIndicatorOptions.NONE },
],
});
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
/* eslint-disable max-lines */

/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
Expand DownExpand Up@@ -50,6 +52,7 @@ import {
showGanttKey,
triggeringUserFilterKey,
} from "src/constants/localStorage";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { HoverProvider } from "src/context/hover";
import { OpenGroupsProvider } from "src/context/openGroups";

Expand DownExpand Up@@ -88,6 +91,11 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
);

const [showGantt, setShowGantt] = useLocalStorage<boolean>(showGanttKey(dagId), false);
// Global setting: applies to all Dags (intentionally not scoped to dagId)
const [showVersionIndicatorMode, setShowVersionIndicatorMode] = useLocalStorage<VersionIndicatorOptions>(
`version_indicator_display_mode`,
VersionIndicatorOptions.ALL,
);
const { fitView, getZoom } = useReactFlow();
const { data: warningData } = useDagWarningServiceListDagWarnings({ dagId });
const { onClose, onOpen, open } = useDisclosure();
Expand DownExpand Up@@ -161,8 +169,10 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
setLimit={setLimit}
setRunTypeFilter={setRunTypeFilter}
setShowGantt={setShowGantt}
setShowVersionIndicatorMode={setShowVersionIndicatorMode}
setTriggeringUserFilter={setTriggeringUserFilter}
showGantt={showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUserFilter={triggeringUserFilter}
/>
{dagView === "graph" ? (
Expand All@@ -174,6 +184,7 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
limit={limit}
runType={runTypeFilter}
showGantt={Boolean(runId) && showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUser={triggeringUserFilter}
/>
{showGantt ? (
Expand Down
27 changes: 22 additions & 5 deletions airflow-core/src/airflow/ui/src/layouts/Details/Grid/Bar.tsx
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,21 +19,27 @@
import { Flex, Box } from "@chakra-ui/react";
import { useParams, useSearchParams } from "react-router-dom";

import type { GridRunsResponse } from "openapi/requests";
import { RunTypeIcon } from "src/components/RunTypeIcon";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { useHover } from "src/context/hover";

import { GridButton } from "./GridButton";

const BAR_HEIGHT = 100;
import { BundleVersionIndicator, DagVersionIndicator } from "./VersionIndicator";
import { BAR_HEIGHT } from "./constants";
import {
getBundleVersion,
getMaxVersionNumber,
type GridRunWithVersionFlags,
} from "./useGridRunsWithVersionFlags";

type Props = {
readonly max: number;
readonly onClick?: () => void;
readonly run: GridRunsResponse;
readonly run: GridRunWithVersionFlags;
readonly showVersionIndicatorMode?: VersionIndicatorOptions;
};

export const Bar = ({ max, onClick, run }: Props) => {
export const Bar = ({ max, onClick, run, showVersionIndicatorMode }: Props) => {
const { dagId = "", runId } = useParams();
const [searchParams] = useSearchParams();
const { hoveredRunId, setHoveredRunId } = useHover();
Expand All@@ -53,6 +59,17 @@ export const Bar = ({ max, onClick, run }: Props) => {
position="relative"
transition="background-color 0.2s"
>
{run.isBundleVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.BUNDLE_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<BundleVersionIndicator bundleVersion={getBundleVersion(run)} />
) : undefined}
{run.isDagVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.DAG_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<DagVersionIndicator dagVersionNumber={getMaxVersionNumber(run)} orientation="vertical" />
) : undefined}

<Flex
alignItems="flex-end"
height={BAR_HEIGHT}
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Highlight search terms from Google/DuckDuckGo/Bing referrer (function() { var ref = document.referrer; var terms = []; if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) { var url = new URL(ref); var q = url.searchParams.get('q') || url.searchParams.get('p'); if (q) { terms = q.split(/\s+/).filter(function(t) { return t.length > 2; }); } } if (terms.length === 0) return; var style = document.createElement('style'); style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }'; document.head.appendChild(style); function highlight(node) { if (node.nodeType === 3) { // text node var text = node.textContent; var found = false; terms.forEach(function(term) { var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Feat: Add version change indicators for Dag and bundle versions in Grid view by choo121600 · Pull Request #53216 · apache/airflow · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

from airflow._shared.timezones import timezone
from airflow.api_fastapi.core_api.base import BaseModel
from airflow.api_fastapi.core_api.datamodels.dag_versions import DagVersionResponse
from airflow.utils.state import DagRunState
from airflow.utils.types import DagRunType

Expand DownExpand Up@@ -79,6 +80,7 @@ class GridRunsResponse(BaseModel):
run_after: datetime
state: DagRunState | None
run_type: DagRunType
dag_versions: list[DagVersionResponse] = []
has_missed_deadline: bool

@computed_field
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,6 +32,7 @@ class LightGridTaskInstanceSummary(BaseModel):
child_states: dict[TaskInstanceState | None, int] | None
min_start_date: datetime | None
max_end_date: datetime | None
dag_version_number: int | None = None


class GridTISummaries(BaseModel):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -2183,6 +2183,12 @@ components:
- type: 'null'
run_type:
$ref: '#/components/schemas/DagRunType'
dag_versions:
items:
$ref: '#/components/schemas/DagVersionResponse'
type: array
title: Dag Versions
default: []
has_missed_deadline:
type: boolean
title: Has Missed Deadline
Expand DownExpand Up@@ -2453,6 +2459,11 @@ components:
format: date-time
- type: 'null'
title: Max End Date
dag_version_number:
anyOf:
- type: integer
- type: 'null'
title: Dag Version Number
type: object
required:
- task_id
Expand Down
53 changes: 40 additions & 13 deletions airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@
import structlog
from fastapi import Depends, HTTPException, status
from sqlalchemy import exists, select
from sqlalchemy.orm import joinedload
from sqlalchemy.orm import joinedload, load_only, selectinload

from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
from airflow.api_fastapi.common.db.common import SessionDep, paginated_select
Expand DownExpand Up@@ -58,11 +58,13 @@
get_task_group_children_getter,
task_group_to_dict_grid,
)
from airflow.models.dag import DagModel
from airflow.models.dag_version import DagVersion
from airflow.models.dagrun import DagRun
from airflow.models.deadline import Deadline
from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.taskinstance import TaskInstance
from airflow.models.taskinstancehistory import TaskInstanceHistory

log = structlog.get_logger(logger_name=__name__)
grid_router = AirflowRouter(prefix="/grid", tags=["Grid"])
Expand DownExpand Up@@ -282,17 +284,33 @@ def get_grid_runs(
.correlate(DagRun)
.label("has_missed_deadline")
)
base_query = select(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
has_missed_deadline,
).where(DagRun.dag_id == dag_id)
base_query = (
select(DagRun, has_missed_deadline)
.where(DagRun.dag_id == dag_id)
.options(
load_only(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
DagRun.bundle_version,
),
joinedload(DagRun.dag_model).load_only(DagModel._dag_display_property_value),
joinedload(DagRun.created_dag_version).joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances)
.load_only(TaskInstance.dag_version_id)
.joinedload(TaskInstance.dag_version)
.joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances_histories)
.load_only(TaskInstanceHistory.dag_version_id)
.joinedload(TaskInstanceHistory.dag_version)
.joinedload(DagVersion.bundle),
)
)

# This comparison is to fall back to DAG timetable when no order_by is provided
if order_by.value == [order_by.get_primary_key_string()]:
Expand All@@ -309,8 +327,14 @@ def get_grid_runs(
offset=offset,
filters=[run_after, run_type, state, triggering_user],
limit=limit,
return_total_entries=False,
)
return [GridRunsResponse(**row._mapping) for row in session.execute(dag_runs_select_filter)]
results = session.execute(dag_runs_select_filter).unique().all()
grid_runs = []
for run, has_missed in results:
run.has_missed_deadline = has_missed
grid_runs.append(GridRunsResponse.model_validate(run, from_attributes=True))
return grid_runs


@grid_router.get(
Expand DownExpand Up@@ -363,7 +387,9 @@ def get_grid_ti_summaries(
TaskInstance.dag_version_id,
TaskInstance.start_date,
TaskInstance.end_date,
DagVersion.version_number,
)
.outerjoin(DagVersion, TaskInstance.dag_version_id == DagVersion.id)
.where(TaskInstance.dag_id == dag_id)
.where(
TaskInstance.run_id == run_id,
Expand All@@ -386,6 +412,7 @@ def get_grid_ti_summaries(
"state": ti.state,
"start_date": ti.start_date,
"end_date": ti.end_date,
"dag_version_number": ti.version_number,
}
)
serdag = _get_serdag(
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -72,11 +72,18 @@ def _get_aggs_for_node(detail):
max_end_date = max(x["end_date"] for x in detail if x["end_date"])
except ValueError:
max_end_date = None

dag_version_numbers = [
x.get("dag_version_number") for x in detail if x.get("dag_version_number") is not None
]
dag_version_number = max(dag_version_numbers) if dag_version_numbers else None

return {
"state": agg_state(states),
"min_start_date": min_start_date,
"max_end_date": max_end_date,
"child_states": dict(Counter(states)),
"dag_version_number": dag_version_number,
}


Expand Down
19 changes: 19 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -8143,6 +8143,14 @@ export const $GridRunsResponse = {
run_type: {
'$ref': '#/components/schemas/DagRunType'
},
dag_versions: {
items: {
'$ref': '#/components/schemas/DagVersionResponse'
},
type: 'array',
title: 'Dag Versions',
default: []
},
has_missed_deadline: {
type: 'boolean',
title: 'Has Missed Deadline'
Expand DownExpand Up@@ -8258,6 +8266,17 @@ export const $LightGridTaskInstanceSummary = {
}
],
title: 'Max End Date'
},
dag_version_number: {
anyOf: [
{
type: 'integer'
},
{
type: 'null'
}
],
title: 'Dag Version Number'
}
},
type: 'object',
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -1999,6 +1999,7 @@ export type GridRunsResponse = {
run_after: string;
state: DagRunState | null;
run_type: DagRunType;
dag_versions?: Array<DagVersionResponse>;
has_missed_deadline: boolean;
readonly duration: number;
};
Expand DownExpand Up@@ -2033,6 +2034,7 @@ export type LightGridTaskInstanceSummary = {
} | null;
min_start_date: string | null;
max_end_date: string | null;
dag_version_number?: number | null;
};

/**
Expand Down
9 changes: 9 additions & 0 deletions airflow-core/src/airflow/ui/public/i18n/locales/en/dag.json
Original file line numberDiff line numberDiff line change
Expand Up@@ -121,6 +121,15 @@
"graphDirection": {
"label": "Graph Direction"
},
"showVersionIndicator": {
"label": "Show Version Indicator",
"options": {
"hideAll": "Hide All",
"showAll": "Show All",
"showBundleVersion": "Show Bundle Version",
"showDagVersion": "Show Dag Version"
}
},
"taskStreamFilter": {
"activeFilter": "Active filter",
"clearFilter": "Clear Filter",
Expand Down
1 change: 1 addition & 0 deletions airflow-core/src/airflow/ui/src/constants/localStorage.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ export const CALENDAR_VIEW_MODE_KEY = "calendar-view-mode";
export const LOG_WRAP_KEY = "log_wrap";
export const LOG_SHOW_TIMESTAMP_KEY = "log_show_timestamp";
export const LOG_SHOW_SOURCE_KEY = "log_show_source";
export const VERSION_INDICATOR_DISPLAY_MODE_KEY = "version_indicator_display_mode";

// Dag-scoped keys
export const dagViewKey = (dagId: string) => `dag_view-${dagId}`;
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { createListCollection } from "@chakra-ui/react";

export enum VersionIndicatorOptions {
ALL = "all",
BUNDLE_VERSION = "bundle",
DAG_VERSION = "dag",
NONE = "none",
}

const validOptions = new Set<string>(Object.values(VersionIndicatorOptions));

export const isVersionIndicatorOption = (value: unknown): value is VersionIndicatorOptions =>
typeof value === "string" && validOptions.has(value);

export const showVersionIndicatorOptions = createListCollection({
items: [
{ label: "dag:panel.showVersionIndicator.options.showAll", value: VersionIndicatorOptions.ALL },
{
label: "dag:panel.showVersionIndicator.options.showBundleVersion",
value: VersionIndicatorOptions.BUNDLE_VERSION,
},
{
label: "dag:panel.showVersionIndicator.options.showDagVersion",
value: VersionIndicatorOptions.DAG_VERSION,
},
{ label: "dag:panel.showVersionIndicator.options.hideAll", value: VersionIndicatorOptions.NONE },
],
});
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
/* eslint-disable max-lines */

/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
Expand DownExpand Up@@ -50,6 +52,7 @@ import {
showGanttKey,
triggeringUserFilterKey,
} from "src/constants/localStorage";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { HoverProvider } from "src/context/hover";
import { OpenGroupsProvider } from "src/context/openGroups";

Expand DownExpand Up@@ -88,6 +91,11 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
);

const [showGantt, setShowGantt] = useLocalStorage<boolean>(showGanttKey(dagId), false);
// Global setting: applies to all Dags (intentionally not scoped to dagId)
const [showVersionIndicatorMode, setShowVersionIndicatorMode] = useLocalStorage<VersionIndicatorOptions>(
`version_indicator_display_mode`,
VersionIndicatorOptions.ALL,
);
const { fitView, getZoom } = useReactFlow();
const { data: warningData } = useDagWarningServiceListDagWarnings({ dagId });
const { onClose, onOpen, open } = useDisclosure();
Expand DownExpand Up@@ -161,8 +169,10 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
setLimit={setLimit}
setRunTypeFilter={setRunTypeFilter}
setShowGantt={setShowGantt}
setShowVersionIndicatorMode={setShowVersionIndicatorMode}
setTriggeringUserFilter={setTriggeringUserFilter}
showGantt={showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUserFilter={triggeringUserFilter}
/>
{dagView === "graph" ? (
Expand All@@ -174,6 +184,7 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
limit={limit}
runType={runTypeFilter}
showGantt={Boolean(runId) && showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUser={triggeringUserFilter}
/>
{showGantt ? (
Expand Down
27 changes: 22 additions & 5 deletions airflow-core/src/airflow/ui/src/layouts/Details/Grid/Bar.tsx
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,21 +19,27 @@
import { Flex, Box } from "@chakra-ui/react";
import { useParams, useSearchParams } from "react-router-dom";

import type { GridRunsResponse } from "openapi/requests";
import { RunTypeIcon } from "src/components/RunTypeIcon";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { useHover } from "src/context/hover";

import { GridButton } from "./GridButton";

const BAR_HEIGHT = 100;
import { BundleVersionIndicator, DagVersionIndicator } from "./VersionIndicator";
import { BAR_HEIGHT } from "./constants";
import {
getBundleVersion,
getMaxVersionNumber,
type GridRunWithVersionFlags,
} from "./useGridRunsWithVersionFlags";

type Props = {
readonly max: number;
readonly onClick?: () => void;
readonly run: GridRunsResponse;
readonly run: GridRunWithVersionFlags;
readonly showVersionIndicatorMode?: VersionIndicatorOptions;
};

export const Bar = ({ max, onClick, run }: Props) => {
export const Bar = ({ max, onClick, run, showVersionIndicatorMode }: Props) => {
const { dagId = "", runId } = useParams();
const [searchParams] = useSearchParams();
const { hoveredRunId, setHoveredRunId } = useHover();
Expand All@@ -53,6 +59,17 @@ export const Bar = ({ max, onClick, run }: Props) => {
position="relative"
transition="background-color 0.2s"
>
{run.isBundleVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.BUNDLE_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<BundleVersionIndicator bundleVersion={getBundleVersion(run)} />
) : undefined}
{run.isDagVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.DAG_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<DagVersionIndicator dagVersionNumber={getMaxVersionNumber(run)} orientation="vertical" />
) : undefined}

<Flex
alignItems="flex-end"
height={BAR_HEIGHT}
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ' Feat: Add version change indicators for Dag and bundle versions in Grid view by choo121600 · Pull Request #53216 · apache/airflow · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

from airflow._shared.timezones import timezone
from airflow.api_fastapi.core_api.base import BaseModel
from airflow.api_fastapi.core_api.datamodels.dag_versions import DagVersionResponse
from airflow.utils.state import DagRunState
from airflow.utils.types import DagRunType

Expand DownExpand Up@@ -79,6 +80,7 @@ class GridRunsResponse(BaseModel):
run_after: datetime
state: DagRunState | None
run_type: DagRunType
dag_versions: list[DagVersionResponse] = []
has_missed_deadline: bool

@computed_field
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,6 +32,7 @@ class LightGridTaskInstanceSummary(BaseModel):
child_states: dict[TaskInstanceState | None, int] | None
min_start_date: datetime | None
max_end_date: datetime | None
dag_version_number: int | None = None


class GridTISummaries(BaseModel):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -2183,6 +2183,12 @@ components:
- type: 'null'
run_type:
$ref: '#/components/schemas/DagRunType'
dag_versions:
items:
$ref: '#/components/schemas/DagVersionResponse'
type: array
title: Dag Versions
default: []
has_missed_deadline:
type: boolean
title: Has Missed Deadline
Expand DownExpand Up@@ -2453,6 +2459,11 @@ components:
format: date-time
- type: 'null'
title: Max End Date
dag_version_number:
anyOf:
- type: integer
- type: 'null'
title: Dag Version Number
type: object
required:
- task_id
Expand Down
53 changes: 40 additions & 13 deletions airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@
import structlog
from fastapi import Depends, HTTPException, status
from sqlalchemy import exists, select
from sqlalchemy.orm import joinedload
from sqlalchemy.orm import joinedload, load_only, selectinload

from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
from airflow.api_fastapi.common.db.common import SessionDep, paginated_select
Expand DownExpand Up@@ -58,11 +58,13 @@
get_task_group_children_getter,
task_group_to_dict_grid,
)
from airflow.models.dag import DagModel
from airflow.models.dag_version import DagVersion
from airflow.models.dagrun import DagRun
from airflow.models.deadline import Deadline
from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.taskinstance import TaskInstance
from airflow.models.taskinstancehistory import TaskInstanceHistory

log = structlog.get_logger(logger_name=__name__)
grid_router = AirflowRouter(prefix="/grid", tags=["Grid"])
Expand DownExpand Up@@ -282,17 +284,33 @@ def get_grid_runs(
.correlate(DagRun)
.label("has_missed_deadline")
)
base_query = select(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
has_missed_deadline,
).where(DagRun.dag_id == dag_id)
base_query = (
select(DagRun, has_missed_deadline)
.where(DagRun.dag_id == dag_id)
.options(
load_only(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
DagRun.bundle_version,
),
joinedload(DagRun.dag_model).load_only(DagModel._dag_display_property_value),
joinedload(DagRun.created_dag_version).joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances)
.load_only(TaskInstance.dag_version_id)
.joinedload(TaskInstance.dag_version)
.joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances_histories)
.load_only(TaskInstanceHistory.dag_version_id)
.joinedload(TaskInstanceHistory.dag_version)
.joinedload(DagVersion.bundle),
)
)

# This comparison is to fall back to DAG timetable when no order_by is provided
if order_by.value == [order_by.get_primary_key_string()]:
Expand All@@ -309,8 +327,14 @@ def get_grid_runs(
offset=offset,
filters=[run_after, run_type, state, triggering_user],
limit=limit,
return_total_entries=False,
)
return [GridRunsResponse(**row._mapping) for row in session.execute(dag_runs_select_filter)]
results = session.execute(dag_runs_select_filter).unique().all()
grid_runs = []
for run, has_missed in results:
run.has_missed_deadline = has_missed
grid_runs.append(GridRunsResponse.model_validate(run, from_attributes=True))
return grid_runs


@grid_router.get(
Expand DownExpand Up@@ -363,7 +387,9 @@ def get_grid_ti_summaries(
TaskInstance.dag_version_id,
TaskInstance.start_date,
TaskInstance.end_date,
DagVersion.version_number,
)
.outerjoin(DagVersion, TaskInstance.dag_version_id == DagVersion.id)
.where(TaskInstance.dag_id == dag_id)
.where(
TaskInstance.run_id == run_id,
Expand All@@ -386,6 +412,7 @@ def get_grid_ti_summaries(
"state": ti.state,
"start_date": ti.start_date,
"end_date": ti.end_date,
"dag_version_number": ti.version_number,
}
)
serdag = _get_serdag(
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -72,11 +72,18 @@ def _get_aggs_for_node(detail):
max_end_date = max(x["end_date"] for x in detail if x["end_date"])
except ValueError:
max_end_date = None

dag_version_numbers = [
x.get("dag_version_number") for x in detail if x.get("dag_version_number") is not None
]
dag_version_number = max(dag_version_numbers) if dag_version_numbers else None

return {
"state": agg_state(states),
"min_start_date": min_start_date,
"max_end_date": max_end_date,
"child_states": dict(Counter(states)),
"dag_version_number": dag_version_number,
}


Expand Down
19 changes: 19 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -8143,6 +8143,14 @@ export const $GridRunsResponse = {
run_type: {
'$ref': '#/components/schemas/DagRunType'
},
dag_versions: {
items: {
'$ref': '#/components/schemas/DagVersionResponse'
},
type: 'array',
title: 'Dag Versions',
default: []
},
has_missed_deadline: {
type: 'boolean',
title: 'Has Missed Deadline'
Expand DownExpand Up@@ -8258,6 +8266,17 @@ export const $LightGridTaskInstanceSummary = {
}
],
title: 'Max End Date'
},
dag_version_number: {
anyOf: [
{
type: 'integer'
},
{
type: 'null'
}
],
title: 'Dag Version Number'
}
},
type: 'object',
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -1999,6 +1999,7 @@ export type GridRunsResponse = {
run_after: string;
state: DagRunState | null;
run_type: DagRunType;
dag_versions?: Array<DagVersionResponse>;
has_missed_deadline: boolean;
readonly duration: number;
};
Expand DownExpand Up@@ -2033,6 +2034,7 @@ export type LightGridTaskInstanceSummary = {
} | null;
min_start_date: string | null;
max_end_date: string | null;
dag_version_number?: number | null;
};

/**
Expand Down
9 changes: 9 additions & 0 deletions airflow-core/src/airflow/ui/public/i18n/locales/en/dag.json
Original file line numberDiff line numberDiff line change
Expand Up@@ -121,6 +121,15 @@
"graphDirection": {
"label": "Graph Direction"
},
"showVersionIndicator": {
"label": "Show Version Indicator",
"options": {
"hideAll": "Hide All",
"showAll": "Show All",
"showBundleVersion": "Show Bundle Version",
"showDagVersion": "Show Dag Version"
}
},
"taskStreamFilter": {
"activeFilter": "Active filter",
"clearFilter": "Clear Filter",
Expand Down
1 change: 1 addition & 0 deletions airflow-core/src/airflow/ui/src/constants/localStorage.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ export const CALENDAR_VIEW_MODE_KEY = "calendar-view-mode";
export const LOG_WRAP_KEY = "log_wrap";
export const LOG_SHOW_TIMESTAMP_KEY = "log_show_timestamp";
export const LOG_SHOW_SOURCE_KEY = "log_show_source";
export const VERSION_INDICATOR_DISPLAY_MODE_KEY = "version_indicator_display_mode";

// Dag-scoped keys
export const dagViewKey = (dagId: string) => `dag_view-${dagId}`;
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { createListCollection } from "@chakra-ui/react";

export enum VersionIndicatorOptions {
ALL = "all",
BUNDLE_VERSION = "bundle",
DAG_VERSION = "dag",
NONE = "none",
}

const validOptions = new Set<string>(Object.values(VersionIndicatorOptions));

export const isVersionIndicatorOption = (value: unknown): value is VersionIndicatorOptions =>
typeof value === "string" && validOptions.has(value);

export const showVersionIndicatorOptions = createListCollection({
items: [
{ label: "dag:panel.showVersionIndicator.options.showAll", value: VersionIndicatorOptions.ALL },
{
label: "dag:panel.showVersionIndicator.options.showBundleVersion",
value: VersionIndicatorOptions.BUNDLE_VERSION,
},
{
label: "dag:panel.showVersionIndicator.options.showDagVersion",
value: VersionIndicatorOptions.DAG_VERSION,
},
{ label: "dag:panel.showVersionIndicator.options.hideAll", value: VersionIndicatorOptions.NONE },
],
});
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
/* eslint-disable max-lines */

/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
Expand DownExpand Up@@ -50,6 +52,7 @@ import {
showGanttKey,
triggeringUserFilterKey,
} from "src/constants/localStorage";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { HoverProvider } from "src/context/hover";
import { OpenGroupsProvider } from "src/context/openGroups";

Expand DownExpand Up@@ -88,6 +91,11 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
);

const [showGantt, setShowGantt] = useLocalStorage<boolean>(showGanttKey(dagId), false);
// Global setting: applies to all Dags (intentionally not scoped to dagId)
const [showVersionIndicatorMode, setShowVersionIndicatorMode] = useLocalStorage<VersionIndicatorOptions>(
`version_indicator_display_mode`,
VersionIndicatorOptions.ALL,
);
const { fitView, getZoom } = useReactFlow();
const { data: warningData } = useDagWarningServiceListDagWarnings({ dagId });
const { onClose, onOpen, open } = useDisclosure();
Expand DownExpand Up@@ -161,8 +169,10 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
setLimit={setLimit}
setRunTypeFilter={setRunTypeFilter}
setShowGantt={setShowGantt}
setShowVersionIndicatorMode={setShowVersionIndicatorMode}
setTriggeringUserFilter={setTriggeringUserFilter}
showGantt={showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUserFilter={triggeringUserFilter}
/>
{dagView === "graph" ? (
Expand All@@ -174,6 +184,7 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
limit={limit}
runType={runTypeFilter}
showGantt={Boolean(runId) && showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUser={triggeringUserFilter}
/>
{showGantt ? (
Expand Down
27 changes: 22 additions & 5 deletions airflow-core/src/airflow/ui/src/layouts/Details/Grid/Bar.tsx
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,21 +19,27 @@
import { Flex, Box } from "@chakra-ui/react";
import { useParams, useSearchParams } from "react-router-dom";

import type { GridRunsResponse } from "openapi/requests";
import { RunTypeIcon } from "src/components/RunTypeIcon";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { useHover } from "src/context/hover";

import { GridButton } from "./GridButton";

const BAR_HEIGHT = 100;
import { BundleVersionIndicator, DagVersionIndicator } from "./VersionIndicator";
import { BAR_HEIGHT } from "./constants";
import {
getBundleVersion,
getMaxVersionNumber,
type GridRunWithVersionFlags,
} from "./useGridRunsWithVersionFlags";

type Props = {
readonly max: number;
readonly onClick?: () => void;
readonly run: GridRunsResponse;
readonly run: GridRunWithVersionFlags;
readonly showVersionIndicatorMode?: VersionIndicatorOptions;
};

export const Bar = ({ max, onClick, run }: Props) => {
export const Bar = ({ max, onClick, run, showVersionIndicatorMode }: Props) => {
const { dagId = "", runId } = useParams();
const [searchParams] = useSearchParams();
const { hoveredRunId, setHoveredRunId } = useHover();
Expand All@@ -53,6 +59,17 @@ export const Bar = ({ max, onClick, run }: Props) => {
position="relative"
transition="background-color 0.2s"
>
{run.isBundleVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.BUNDLE_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<BundleVersionIndicator bundleVersion={getBundleVersion(run)} />
) : undefined}
{run.isDagVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.DAG_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<DagVersionIndicator dagVersionNumber={getMaxVersionNumber(run)} orientation="vertical" />
) : undefined}

<Flex
alignItems="flex-end"
height={BAR_HEIGHT}
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Feat: Add version change indicators for Dag and bundle versions in Grid view by choo121600 · Pull Request #53216 · apache/airflow · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

from airflow._shared.timezones import timezone
from airflow.api_fastapi.core_api.base import BaseModel
from airflow.api_fastapi.core_api.datamodels.dag_versions import DagVersionResponse
from airflow.utils.state import DagRunState
from airflow.utils.types import DagRunType

Expand DownExpand Up@@ -79,6 +80,7 @@ class GridRunsResponse(BaseModel):
run_after: datetime
state: DagRunState | None
run_type: DagRunType
dag_versions: list[DagVersionResponse] = []
has_missed_deadline: bool

@computed_field
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,6 +32,7 @@ class LightGridTaskInstanceSummary(BaseModel):
child_states: dict[TaskInstanceState | None, int] | None
min_start_date: datetime | None
max_end_date: datetime | None
dag_version_number: int | None = None


class GridTISummaries(BaseModel):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -2183,6 +2183,12 @@ components:
- type: 'null'
run_type:
$ref: '#/components/schemas/DagRunType'
dag_versions:
items:
$ref: '#/components/schemas/DagVersionResponse'
type: array
title: Dag Versions
default: []
has_missed_deadline:
type: boolean
title: Has Missed Deadline
Expand DownExpand Up@@ -2453,6 +2459,11 @@ components:
format: date-time
- type: 'null'
title: Max End Date
dag_version_number:
anyOf:
- type: integer
- type: 'null'
title: Dag Version Number
type: object
required:
- task_id
Expand Down
53 changes: 40 additions & 13 deletions airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@
import structlog
from fastapi import Depends, HTTPException, status
from sqlalchemy import exists, select
from sqlalchemy.orm import joinedload
from sqlalchemy.orm import joinedload, load_only, selectinload

from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
from airflow.api_fastapi.common.db.common import SessionDep, paginated_select
Expand DownExpand Up@@ -58,11 +58,13 @@
get_task_group_children_getter,
task_group_to_dict_grid,
)
from airflow.models.dag import DagModel
from airflow.models.dag_version import DagVersion
from airflow.models.dagrun import DagRun
from airflow.models.deadline import Deadline
from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.taskinstance import TaskInstance
from airflow.models.taskinstancehistory import TaskInstanceHistory

log = structlog.get_logger(logger_name=__name__)
grid_router = AirflowRouter(prefix="/grid", tags=["Grid"])
Expand DownExpand Up@@ -282,17 +284,33 @@ def get_grid_runs(
.correlate(DagRun)
.label("has_missed_deadline")
)
base_query = select(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
has_missed_deadline,
).where(DagRun.dag_id == dag_id)
base_query = (
select(DagRun, has_missed_deadline)
.where(DagRun.dag_id == dag_id)
.options(
load_only(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
DagRun.bundle_version,
),
joinedload(DagRun.dag_model).load_only(DagModel._dag_display_property_value),
joinedload(DagRun.created_dag_version).joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances)
.load_only(TaskInstance.dag_version_id)
.joinedload(TaskInstance.dag_version)
.joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances_histories)
.load_only(TaskInstanceHistory.dag_version_id)
.joinedload(TaskInstanceHistory.dag_version)
.joinedload(DagVersion.bundle),
)
)

# This comparison is to fall back to DAG timetable when no order_by is provided
if order_by.value == [order_by.get_primary_key_string()]:
Expand All@@ -309,8 +327,14 @@ def get_grid_runs(
offset=offset,
filters=[run_after, run_type, state, triggering_user],
limit=limit,
return_total_entries=False,
)
return [GridRunsResponse(**row._mapping) for row in session.execute(dag_runs_select_filter)]
results = session.execute(dag_runs_select_filter).unique().all()
grid_runs = []
for run, has_missed in results:
run.has_missed_deadline = has_missed
grid_runs.append(GridRunsResponse.model_validate(run, from_attributes=True))
return grid_runs


@grid_router.get(
Expand DownExpand Up@@ -363,7 +387,9 @@ def get_grid_ti_summaries(
TaskInstance.dag_version_id,
TaskInstance.start_date,
TaskInstance.end_date,
DagVersion.version_number,
)
.outerjoin(DagVersion, TaskInstance.dag_version_id == DagVersion.id)
.where(TaskInstance.dag_id == dag_id)
.where(
TaskInstance.run_id == run_id,
Expand All@@ -386,6 +412,7 @@ def get_grid_ti_summaries(
"state": ti.state,
"start_date": ti.start_date,
"end_date": ti.end_date,
"dag_version_number": ti.version_number,
}
)
serdag = _get_serdag(
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -72,11 +72,18 @@ def _get_aggs_for_node(detail):
max_end_date = max(x["end_date"] for x in detail if x["end_date"])
except ValueError:
max_end_date = None

dag_version_numbers = [
x.get("dag_version_number") for x in detail if x.get("dag_version_number") is not None
]
dag_version_number = max(dag_version_numbers) if dag_version_numbers else None

return {
"state": agg_state(states),
"min_start_date": min_start_date,
"max_end_date": max_end_date,
"child_states": dict(Counter(states)),
"dag_version_number": dag_version_number,
}


Expand Down
19 changes: 19 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -8143,6 +8143,14 @@ export const $GridRunsResponse = {
run_type: {
'$ref': '#/components/schemas/DagRunType'
},
dag_versions: {
items: {
'$ref': '#/components/schemas/DagVersionResponse'
},
type: 'array',
title: 'Dag Versions',
default: []
},
has_missed_deadline: {
type: 'boolean',
title: 'Has Missed Deadline'
Expand DownExpand Up@@ -8258,6 +8266,17 @@ export const $LightGridTaskInstanceSummary = {
}
],
title: 'Max End Date'
},
dag_version_number: {
anyOf: [
{
type: 'integer'
},
{
type: 'null'
}
],
title: 'Dag Version Number'
}
},
type: 'object',
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -1999,6 +1999,7 @@ export type GridRunsResponse = {
run_after: string;
state: DagRunState | null;
run_type: DagRunType;
dag_versions?: Array<DagVersionResponse>;
has_missed_deadline: boolean;
readonly duration: number;
};
Expand DownExpand Up@@ -2033,6 +2034,7 @@ export type LightGridTaskInstanceSummary = {
} | null;
min_start_date: string | null;
max_end_date: string | null;
dag_version_number?: number | null;
};

/**
Expand Down
9 changes: 9 additions & 0 deletions airflow-core/src/airflow/ui/public/i18n/locales/en/dag.json
Original file line numberDiff line numberDiff line change
Expand Up@@ -121,6 +121,15 @@
"graphDirection": {
"label": "Graph Direction"
},
"showVersionIndicator": {
"label": "Show Version Indicator",
"options": {
"hideAll": "Hide All",
"showAll": "Show All",
"showBundleVersion": "Show Bundle Version",
"showDagVersion": "Show Dag Version"
}
},
"taskStreamFilter": {
"activeFilter": "Active filter",
"clearFilter": "Clear Filter",
Expand Down
1 change: 1 addition & 0 deletions airflow-core/src/airflow/ui/src/constants/localStorage.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ export const CALENDAR_VIEW_MODE_KEY = "calendar-view-mode";
export const LOG_WRAP_KEY = "log_wrap";
export const LOG_SHOW_TIMESTAMP_KEY = "log_show_timestamp";
export const LOG_SHOW_SOURCE_KEY = "log_show_source";
export const VERSION_INDICATOR_DISPLAY_MODE_KEY = "version_indicator_display_mode";

// Dag-scoped keys
export const dagViewKey = (dagId: string) => `dag_view-${dagId}`;
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { createListCollection } from "@chakra-ui/react";

export enum VersionIndicatorOptions {
ALL = "all",
BUNDLE_VERSION = "bundle",
DAG_VERSION = "dag",
NONE = "none",
}

const validOptions = new Set<string>(Object.values(VersionIndicatorOptions));

export const isVersionIndicatorOption = (value: unknown): value is VersionIndicatorOptions =>
typeof value === "string" && validOptions.has(value);

export const showVersionIndicatorOptions = createListCollection({
items: [
{ label: "dag:panel.showVersionIndicator.options.showAll", value: VersionIndicatorOptions.ALL },
{
label: "dag:panel.showVersionIndicator.options.showBundleVersion",
value: VersionIndicatorOptions.BUNDLE_VERSION,
},
{
label: "dag:panel.showVersionIndicator.options.showDagVersion",
value: VersionIndicatorOptions.DAG_VERSION,
},
{ label: "dag:panel.showVersionIndicator.options.hideAll", value: VersionIndicatorOptions.NONE },
],
});
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
/* eslint-disable max-lines */

/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
Expand DownExpand Up@@ -50,6 +52,7 @@ import {
showGanttKey,
triggeringUserFilterKey,
} from "src/constants/localStorage";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { HoverProvider } from "src/context/hover";
import { OpenGroupsProvider } from "src/context/openGroups";

Expand DownExpand Up@@ -88,6 +91,11 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
);

const [showGantt, setShowGantt] = useLocalStorage<boolean>(showGanttKey(dagId), false);
// Global setting: applies to all Dags (intentionally not scoped to dagId)
const [showVersionIndicatorMode, setShowVersionIndicatorMode] = useLocalStorage<VersionIndicatorOptions>(
`version_indicator_display_mode`,
VersionIndicatorOptions.ALL,
);
const { fitView, getZoom } = useReactFlow();
const { data: warningData } = useDagWarningServiceListDagWarnings({ dagId });
const { onClose, onOpen, open } = useDisclosure();
Expand DownExpand Up@@ -161,8 +169,10 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
setLimit={setLimit}
setRunTypeFilter={setRunTypeFilter}
setShowGantt={setShowGantt}
setShowVersionIndicatorMode={setShowVersionIndicatorMode}
setTriggeringUserFilter={setTriggeringUserFilter}
showGantt={showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUserFilter={triggeringUserFilter}
/>
{dagView === "graph" ? (
Expand All@@ -174,6 +184,7 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
limit={limit}
runType={runTypeFilter}
showGantt={Boolean(runId) && showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUser={triggeringUserFilter}
/>
{showGantt ? (
Expand Down
27 changes: 22 additions & 5 deletions airflow-core/src/airflow/ui/src/layouts/Details/Grid/Bar.tsx
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,21 +19,27 @@
import { Flex, Box } from "@chakra-ui/react";
import { useParams, useSearchParams } from "react-router-dom";

import type { GridRunsResponse } from "openapi/requests";
import { RunTypeIcon } from "src/components/RunTypeIcon";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { useHover } from "src/context/hover";

import { GridButton } from "./GridButton";

const BAR_HEIGHT = 100;
import { BundleVersionIndicator, DagVersionIndicator } from "./VersionIndicator";
import { BAR_HEIGHT } from "./constants";
import {
getBundleVersion,
getMaxVersionNumber,
type GridRunWithVersionFlags,
} from "./useGridRunsWithVersionFlags";

type Props = {
readonly max: number;
readonly onClick?: () => void;
readonly run: GridRunsResponse;
readonly run: GridRunWithVersionFlags;
readonly showVersionIndicatorMode?: VersionIndicatorOptions;
};

export const Bar = ({ max, onClick, run }: Props) => {
export const Bar = ({ max, onClick, run, showVersionIndicatorMode }: Props) => {
const { dagId = "", runId } = useParams();
const [searchParams] = useSearchParams();
const { hoveredRunId, setHoveredRunId } = useHover();
Expand All@@ -53,6 +59,17 @@ export const Bar = ({ max, onClick, run }: Props) => {
position="relative"
transition="background-color 0.2s"
>
{run.isBundleVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.BUNDLE_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<BundleVersionIndicator bundleVersion={getBundleVersion(run)} />
) : undefined}
{run.isDagVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.DAG_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<DagVersionIndicator dagVersionNumber={getMaxVersionNumber(run)} orientation="vertical" />
) : undefined}

<Flex
alignItems="flex-end"
height={BAR_HEIGHT}
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' Feat: Add version change indicators for Dag and bundle versions in Grid view by choo121600 · Pull Request #53216 · apache/airflow · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

from airflow._shared.timezones import timezone
from airflow.api_fastapi.core_api.base import BaseModel
from airflow.api_fastapi.core_api.datamodels.dag_versions import DagVersionResponse
from airflow.utils.state import DagRunState
from airflow.utils.types import DagRunType

Expand DownExpand Up@@ -79,6 +80,7 @@ class GridRunsResponse(BaseModel):
run_after: datetime
state: DagRunState | None
run_type: DagRunType
dag_versions: list[DagVersionResponse] = []
has_missed_deadline: bool

@computed_field
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,6 +32,7 @@ class LightGridTaskInstanceSummary(BaseModel):
child_states: dict[TaskInstanceState | None, int] | None
min_start_date: datetime | None
max_end_date: datetime | None
dag_version_number: int | None = None


class GridTISummaries(BaseModel):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -2183,6 +2183,12 @@ components:
- type: 'null'
run_type:
$ref: '#/components/schemas/DagRunType'
dag_versions:
items:
$ref: '#/components/schemas/DagVersionResponse'
type: array
title: Dag Versions
default: []
has_missed_deadline:
type: boolean
title: Has Missed Deadline
Expand DownExpand Up@@ -2453,6 +2459,11 @@ components:
format: date-time
- type: 'null'
title: Max End Date
dag_version_number:
anyOf:
- type: integer
- type: 'null'
title: Dag Version Number
type: object
required:
- task_id
Expand Down
53 changes: 40 additions & 13 deletions airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@
import structlog
from fastapi import Depends, HTTPException, status
from sqlalchemy import exists, select
from sqlalchemy.orm import joinedload
from sqlalchemy.orm import joinedload, load_only, selectinload

from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
from airflow.api_fastapi.common.db.common import SessionDep, paginated_select
Expand DownExpand Up@@ -58,11 +58,13 @@
get_task_group_children_getter,
task_group_to_dict_grid,
)
from airflow.models.dag import DagModel
from airflow.models.dag_version import DagVersion
from airflow.models.dagrun import DagRun
from airflow.models.deadline import Deadline
from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.taskinstance import TaskInstance
from airflow.models.taskinstancehistory import TaskInstanceHistory

log = structlog.get_logger(logger_name=__name__)
grid_router = AirflowRouter(prefix="/grid", tags=["Grid"])
Expand DownExpand Up@@ -282,17 +284,33 @@ def get_grid_runs(
.correlate(DagRun)
.label("has_missed_deadline")
)
base_query = select(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
has_missed_deadline,
).where(DagRun.dag_id == dag_id)
base_query = (
select(DagRun, has_missed_deadline)
.where(DagRun.dag_id == dag_id)
.options(
load_only(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
DagRun.bundle_version,
),
joinedload(DagRun.dag_model).load_only(DagModel._dag_display_property_value),
joinedload(DagRun.created_dag_version).joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances)
.load_only(TaskInstance.dag_version_id)
.joinedload(TaskInstance.dag_version)
.joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances_histories)
.load_only(TaskInstanceHistory.dag_version_id)
.joinedload(TaskInstanceHistory.dag_version)
.joinedload(DagVersion.bundle),
)
)

# This comparison is to fall back to DAG timetable when no order_by is provided
if order_by.value == [order_by.get_primary_key_string()]:
Expand All@@ -309,8 +327,14 @@ def get_grid_runs(
offset=offset,
filters=[run_after, run_type, state, triggering_user],
limit=limit,
return_total_entries=False,
)
return [GridRunsResponse(**row._mapping) for row in session.execute(dag_runs_select_filter)]
results = session.execute(dag_runs_select_filter).unique().all()
grid_runs = []
for run, has_missed in results:
run.has_missed_deadline = has_missed
grid_runs.append(GridRunsResponse.model_validate(run, from_attributes=True))
return grid_runs


@grid_router.get(
Expand DownExpand Up@@ -363,7 +387,9 @@ def get_grid_ti_summaries(
TaskInstance.dag_version_id,
TaskInstance.start_date,
TaskInstance.end_date,
DagVersion.version_number,
)
.outerjoin(DagVersion, TaskInstance.dag_version_id == DagVersion.id)
.where(TaskInstance.dag_id == dag_id)
.where(
TaskInstance.run_id == run_id,
Expand All@@ -386,6 +412,7 @@ def get_grid_ti_summaries(
"state": ti.state,
"start_date": ti.start_date,
"end_date": ti.end_date,
"dag_version_number": ti.version_number,
}
)
serdag = _get_serdag(
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -72,11 +72,18 @@ def _get_aggs_for_node(detail):
max_end_date = max(x["end_date"] for x in detail if x["end_date"])
except ValueError:
max_end_date = None

dag_version_numbers = [
x.get("dag_version_number") for x in detail if x.get("dag_version_number") is not None
]
dag_version_number = max(dag_version_numbers) if dag_version_numbers else None

return {
"state": agg_state(states),
"min_start_date": min_start_date,
"max_end_date": max_end_date,
"child_states": dict(Counter(states)),
"dag_version_number": dag_version_number,
}


Expand Down
19 changes: 19 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -8143,6 +8143,14 @@ export const $GridRunsResponse = {
run_type: {
'$ref': '#/components/schemas/DagRunType'
},
dag_versions: {
items: {
'$ref': '#/components/schemas/DagVersionResponse'
},
type: 'array',
title: 'Dag Versions',
default: []
},
has_missed_deadline: {
type: 'boolean',
title: 'Has Missed Deadline'
Expand DownExpand Up@@ -8258,6 +8266,17 @@ export const $LightGridTaskInstanceSummary = {
}
],
title: 'Max End Date'
},
dag_version_number: {
anyOf: [
{
type: 'integer'
},
{
type: 'null'
}
],
title: 'Dag Version Number'
}
},
type: 'object',
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -1999,6 +1999,7 @@ export type GridRunsResponse = {
run_after: string;
state: DagRunState | null;
run_type: DagRunType;
dag_versions?: Array<DagVersionResponse>;
has_missed_deadline: boolean;
readonly duration: number;
};
Expand DownExpand Up@@ -2033,6 +2034,7 @@ export type LightGridTaskInstanceSummary = {
} | null;
min_start_date: string | null;
max_end_date: string | null;
dag_version_number?: number | null;
};

/**
Expand Down
9 changes: 9 additions & 0 deletions airflow-core/src/airflow/ui/public/i18n/locales/en/dag.json
Original file line numberDiff line numberDiff line change
Expand Up@@ -121,6 +121,15 @@
"graphDirection": {
"label": "Graph Direction"
},
"showVersionIndicator": {
"label": "Show Version Indicator",
"options": {
"hideAll": "Hide All",
"showAll": "Show All",
"showBundleVersion": "Show Bundle Version",
"showDagVersion": "Show Dag Version"
}
},
"taskStreamFilter": {
"activeFilter": "Active filter",
"clearFilter": "Clear Filter",
Expand Down
1 change: 1 addition & 0 deletions airflow-core/src/airflow/ui/src/constants/localStorage.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ export const CALENDAR_VIEW_MODE_KEY = "calendar-view-mode";
export const LOG_WRAP_KEY = "log_wrap";
export const LOG_SHOW_TIMESTAMP_KEY = "log_show_timestamp";
export const LOG_SHOW_SOURCE_KEY = "log_show_source";
export const VERSION_INDICATOR_DISPLAY_MODE_KEY = "version_indicator_display_mode";

// Dag-scoped keys
export const dagViewKey = (dagId: string) => `dag_view-${dagId}`;
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { createListCollection } from "@chakra-ui/react";

export enum VersionIndicatorOptions {
ALL = "all",
BUNDLE_VERSION = "bundle",
DAG_VERSION = "dag",
NONE = "none",
}

const validOptions = new Set<string>(Object.values(VersionIndicatorOptions));

export const isVersionIndicatorOption = (value: unknown): value is VersionIndicatorOptions =>
typeof value === "string" && validOptions.has(value);

export const showVersionIndicatorOptions = createListCollection({
items: [
{ label: "dag:panel.showVersionIndicator.options.showAll", value: VersionIndicatorOptions.ALL },
{
label: "dag:panel.showVersionIndicator.options.showBundleVersion",
value: VersionIndicatorOptions.BUNDLE_VERSION,
},
{
label: "dag:panel.showVersionIndicator.options.showDagVersion",
value: VersionIndicatorOptions.DAG_VERSION,
},
{ label: "dag:panel.showVersionIndicator.options.hideAll", value: VersionIndicatorOptions.NONE },
],
});
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
/* eslint-disable max-lines */

/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
Expand DownExpand Up@@ -50,6 +52,7 @@ import {
showGanttKey,
triggeringUserFilterKey,
} from "src/constants/localStorage";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { HoverProvider } from "src/context/hover";
import { OpenGroupsProvider } from "src/context/openGroups";

Expand DownExpand Up@@ -88,6 +91,11 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
);

const [showGantt, setShowGantt] = useLocalStorage<boolean>(showGanttKey(dagId), false);
// Global setting: applies to all Dags (intentionally not scoped to dagId)
const [showVersionIndicatorMode, setShowVersionIndicatorMode] = useLocalStorage<VersionIndicatorOptions>(
`version_indicator_display_mode`,
VersionIndicatorOptions.ALL,
);
const { fitView, getZoom } = useReactFlow();
const { data: warningData } = useDagWarningServiceListDagWarnings({ dagId });
const { onClose, onOpen, open } = useDisclosure();
Expand DownExpand Up@@ -161,8 +169,10 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
setLimit={setLimit}
setRunTypeFilter={setRunTypeFilter}
setShowGantt={setShowGantt}
setShowVersionIndicatorMode={setShowVersionIndicatorMode}
setTriggeringUserFilter={setTriggeringUserFilter}
showGantt={showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUserFilter={triggeringUserFilter}
/>
{dagView === "graph" ? (
Expand All@@ -174,6 +184,7 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
limit={limit}
runType={runTypeFilter}
showGantt={Boolean(runId) && showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUser={triggeringUserFilter}
/>
{showGantt ? (
Expand Down
27 changes: 22 additions & 5 deletions airflow-core/src/airflow/ui/src/layouts/Details/Grid/Bar.tsx
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,21 +19,27 @@
import { Flex, Box } from "@chakra-ui/react";
import { useParams, useSearchParams } from "react-router-dom";

import type { GridRunsResponse } from "openapi/requests";
import { RunTypeIcon } from "src/components/RunTypeIcon";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { useHover } from "src/context/hover";

import { GridButton } from "./GridButton";

const BAR_HEIGHT = 100;
import { BundleVersionIndicator, DagVersionIndicator } from "./VersionIndicator";
import { BAR_HEIGHT } from "./constants";
import {
getBundleVersion,
getMaxVersionNumber,
type GridRunWithVersionFlags,
} from "./useGridRunsWithVersionFlags";

type Props = {
readonly max: number;
readonly onClick?: () => void;
readonly run: GridRunsResponse;
readonly run: GridRunWithVersionFlags;
readonly showVersionIndicatorMode?: VersionIndicatorOptions;
};

export const Bar = ({ max, onClick, run }: Props) => {
export const Bar = ({ max, onClick, run, showVersionIndicatorMode }: Props) => {
const { dagId = "", runId } = useParams();
const [searchParams] = useSearchParams();
const { hoveredRunId, setHoveredRunId } = useHover();
Expand All@@ -53,6 +59,17 @@ export const Bar = ({ max, onClick, run }: Props) => {
position="relative"
transition="background-color 0.2s"
>
{run.isBundleVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.BUNDLE_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<BundleVersionIndicator bundleVersion={getBundleVersion(run)} />
) : undefined}
{run.isDagVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.DAG_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<DagVersionIndicator dagVersionNumber={getMaxVersionNumber(run)} orientation="vertical" />
) : undefined}

<Flex
alignItems="flex-end"
height={BAR_HEIGHT}
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Universal Dark Mode - works on any site (function() { var enabled = true; function applyDarkMode() { if (!enabled) return; // Create style element if it doesn't exist var style = document.getElementById('universal-dark-mode-style'); if (!style) { style = document.createElement('style'); style.id = 'universal-dark-mode-style'; document.head.appendChild(style); } // Dark mode CSS - inverts colors but preserves images/video style.textContent = ' /* Invert everything except media */ html { filter: invert(1) hue-rotate(180deg) !important; background: #1a1a2e !important; } /* Restore images, videos, iframes, canvas */ img, video, iframe, canvas, svg, picture, [style*="background-image"] { filter: invert(1) hue-rotate(180deg) !important; } /* Preserve specific elements that should not be inverted */ .no-dark-mode, .no-dark-mode *, [data-theme="light"], [data-theme="light"], .ace_editor, .ace_editor *, .CodeMirror, .CodeMirror *, .monaco-editor, .monaco-editor *, .markdown-body pre, .markdown-body pre *, .highlight, .highlight *, pre code, pre code * { filter: none !important; } /* Fix common UI elements */ .modal, .popup, .dropdown-menu, .tooltip, .popover { filter: invert(1) hue-rotate(180deg) !important; background: #2d2d44 !important; border-color: #444 !important; } /* Scrollbars */ ::-webkit-scrollbar { background: #1a1a2e !important; } ::-webkit-scrollbar-thumb { background: #444 !important; } ::-webkit-scrollbar-thumb:hover { background: #555 !important; } /* Selection */ ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; } ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; } '; } function removeDarkMode() { var style = document.getElementById('universal-dark-mode-style'); if (style) style.remove(); } // Toggle with Alt+Shift+D document.addEventListener('keydown', function(e) { if (e.altKey && e.shiftKey && e.key === 'D') { e.preventDefault(); enabled = !enabled; if (enabled) { applyDarkMode(); console.log('[Universal Dark Mode] Enabled'); } else { removeDarkMode(); console.log('[Universal Dark Mode] Disabled'); } } }); // Apply on load applyDarkMode(); // Re-apply on dynamic content var observer = new MutationObserver(function(mutations) { if (enabled && !document.getElementById('universal-dark-mode-style')) { applyDarkMode(); } }); observer.observe(document.head, { childList: true }); console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle'); })(); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })(); Feat: Add version change indicators for Dag and bundle versions in Grid view by choo121600 · Pull Request #53216 · apache/airflow · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

from airflow._shared.timezones import timezone
from airflow.api_fastapi.core_api.base import BaseModel
from airflow.api_fastapi.core_api.datamodels.dag_versions import DagVersionResponse
from airflow.utils.state import DagRunState
from airflow.utils.types import DagRunType

Expand DownExpand Up@@ -79,6 +80,7 @@ class GridRunsResponse(BaseModel):
run_after: datetime
state: DagRunState | None
run_type: DagRunType
dag_versions: list[DagVersionResponse] = []
has_missed_deadline: bool

@computed_field
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,6 +32,7 @@ class LightGridTaskInstanceSummary(BaseModel):
child_states: dict[TaskInstanceState | None, int] | None
min_start_date: datetime | None
max_end_date: datetime | None
dag_version_number: int | None = None


class GridTISummaries(BaseModel):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -2183,6 +2183,12 @@ components:
- type: 'null'
run_type:
$ref: '#/components/schemas/DagRunType'
dag_versions:
items:
$ref: '#/components/schemas/DagVersionResponse'
type: array
title: Dag Versions
default: []
has_missed_deadline:
type: boolean
title: Has Missed Deadline
Expand DownExpand Up@@ -2453,6 +2459,11 @@ components:
format: date-time
- type: 'null'
title: Max End Date
dag_version_number:
anyOf:
- type: integer
- type: 'null'
title: Dag Version Number
type: object
required:
- task_id
Expand Down
53 changes: 40 additions & 13 deletions airflow-core/src/airflow/api_fastapi/core_api/routes/ui/grid.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@
import structlog
from fastapi import Depends, HTTPException, status
from sqlalchemy import exists, select
from sqlalchemy.orm import joinedload
from sqlalchemy.orm import joinedload, load_only, selectinload

from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
from airflow.api_fastapi.common.db.common import SessionDep, paginated_select
Expand DownExpand Up@@ -58,11 +58,13 @@
get_task_group_children_getter,
task_group_to_dict_grid,
)
from airflow.models.dag import DagModel
from airflow.models.dag_version import DagVersion
from airflow.models.dagrun import DagRun
from airflow.models.deadline import Deadline
from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.taskinstance import TaskInstance
from airflow.models.taskinstancehistory import TaskInstanceHistory

log = structlog.get_logger(logger_name=__name__)
grid_router = AirflowRouter(prefix="/grid", tags=["Grid"])
Expand DownExpand Up@@ -282,17 +284,33 @@ def get_grid_runs(
.correlate(DagRun)
.label("has_missed_deadline")
)
base_query = select(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
has_missed_deadline,
).where(DagRun.dag_id == dag_id)
base_query = (
select(DagRun, has_missed_deadline)
.where(DagRun.dag_id == dag_id)
.options(
load_only(
DagRun.dag_id,
DagRun.run_id,
DagRun.queued_at,
DagRun.start_date,
DagRun.end_date,
DagRun.run_after,
DagRun.state,
DagRun.run_type,
DagRun.bundle_version,
),
joinedload(DagRun.dag_model).load_only(DagModel._dag_display_property_value),
joinedload(DagRun.created_dag_version).joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances)
.load_only(TaskInstance.dag_version_id)
.joinedload(TaskInstance.dag_version)
.joinedload(DagVersion.bundle),
selectinload(DagRun.task_instances_histories)
.load_only(TaskInstanceHistory.dag_version_id)
.joinedload(TaskInstanceHistory.dag_version)
.joinedload(DagVersion.bundle),
)
)

# This comparison is to fall back to DAG timetable when no order_by is provided
if order_by.value == [order_by.get_primary_key_string()]:
Expand All@@ -309,8 +327,14 @@ def get_grid_runs(
offset=offset,
filters=[run_after, run_type, state, triggering_user],
limit=limit,
return_total_entries=False,
)
return [GridRunsResponse(**row._mapping) for row in session.execute(dag_runs_select_filter)]
results = session.execute(dag_runs_select_filter).unique().all()
grid_runs = []
for run, has_missed in results:
run.has_missed_deadline = has_missed
grid_runs.append(GridRunsResponse.model_validate(run, from_attributes=True))
return grid_runs


@grid_router.get(
Expand DownExpand Up@@ -363,7 +387,9 @@ def get_grid_ti_summaries(
TaskInstance.dag_version_id,
TaskInstance.start_date,
TaskInstance.end_date,
DagVersion.version_number,
)
.outerjoin(DagVersion, TaskInstance.dag_version_id == DagVersion.id)
.where(TaskInstance.dag_id == dag_id)
.where(
TaskInstance.run_id == run_id,
Expand All@@ -386,6 +412,7 @@ def get_grid_ti_summaries(
"state": ti.state,
"start_date": ti.start_date,
"end_date": ti.end_date,
"dag_version_number": ti.version_number,
}
)
serdag = _get_serdag(
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -72,11 +72,18 @@ def _get_aggs_for_node(detail):
max_end_date = max(x["end_date"] for x in detail if x["end_date"])
except ValueError:
max_end_date = None

dag_version_numbers = [
x.get("dag_version_number") for x in detail if x.get("dag_version_number") is not None
]
dag_version_number = max(dag_version_numbers) if dag_version_numbers else None

return {
"state": agg_state(states),
"min_start_date": min_start_date,
"max_end_date": max_end_date,
"child_states": dict(Counter(states)),
"dag_version_number": dag_version_number,
}


Expand Down
19 changes: 19 additions & 0 deletions airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -8143,6 +8143,14 @@ export const $GridRunsResponse = {
run_type: {
'$ref': '#/components/schemas/DagRunType'
},
dag_versions: {
items: {
'$ref': '#/components/schemas/DagVersionResponse'
},
type: 'array',
title: 'Dag Versions',
default: []
},
has_missed_deadline: {
type: 'boolean',
title: 'Has Missed Deadline'
Expand DownExpand Up@@ -8258,6 +8266,17 @@ export const $LightGridTaskInstanceSummary = {
}
],
title: 'Max End Date'
},
dag_version_number: {
anyOf: [
{
type: 'integer'
},
{
type: 'null'
}
],
title: 'Dag Version Number'
}
},
type: 'object',
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -1999,6 +1999,7 @@ export type GridRunsResponse = {
run_after: string;
state: DagRunState | null;
run_type: DagRunType;
dag_versions?: Array<DagVersionResponse>;
has_missed_deadline: boolean;
readonly duration: number;
};
Expand DownExpand Up@@ -2033,6 +2034,7 @@ export type LightGridTaskInstanceSummary = {
} | null;
min_start_date: string | null;
max_end_date: string | null;
dag_version_number?: number | null;
};

/**
Expand Down
9 changes: 9 additions & 0 deletions airflow-core/src/airflow/ui/public/i18n/locales/en/dag.json
Original file line numberDiff line numberDiff line change
Expand Up@@ -121,6 +121,15 @@
"graphDirection": {
"label": "Graph Direction"
},
"showVersionIndicator": {
"label": "Show Version Indicator",
"options": {
"hideAll": "Hide All",
"showAll": "Show All",
"showBundleVersion": "Show Bundle Version",
"showDagVersion": "Show Dag Version"
}
},
"taskStreamFilter": {
"activeFilter": "Active filter",
"clearFilter": "Clear Filter",
Expand Down
1 change: 1 addition & 0 deletions airflow-core/src/airflow/ui/src/constants/localStorage.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@ export const CALENDAR_VIEW_MODE_KEY = "calendar-view-mode";
export const LOG_WRAP_KEY = "log_wrap";
export const LOG_SHOW_TIMESTAMP_KEY = "log_show_timestamp";
export const LOG_SHOW_SOURCE_KEY = "log_show_source";
export const VERSION_INDICATOR_DISPLAY_MODE_KEY = "version_indicator_display_mode";

// Dag-scoped keys
export const dagViewKey = (dagId: string) => `dag_view-${dagId}`;
Expand Down
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { createListCollection } from "@chakra-ui/react";

export enum VersionIndicatorOptions {
ALL = "all",
BUNDLE_VERSION = "bundle",
DAG_VERSION = "dag",
NONE = "none",
}

const validOptions = new Set<string>(Object.values(VersionIndicatorOptions));

export const isVersionIndicatorOption = (value: unknown): value is VersionIndicatorOptions =>
typeof value === "string" && validOptions.has(value);

export const showVersionIndicatorOptions = createListCollection({
items: [
{ label: "dag:panel.showVersionIndicator.options.showAll", value: VersionIndicatorOptions.ALL },
{
label: "dag:panel.showVersionIndicator.options.showBundleVersion",
value: VersionIndicatorOptions.BUNDLE_VERSION,
},
{
label: "dag:panel.showVersionIndicator.options.showDagVersion",
value: VersionIndicatorOptions.DAG_VERSION,
},
{ label: "dag:panel.showVersionIndicator.options.hideAll", value: VersionIndicatorOptions.NONE },
],
});
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
/* eslint-disable max-lines */

/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
Expand DownExpand Up@@ -50,6 +52,7 @@ import {
showGanttKey,
triggeringUserFilterKey,
} from "src/constants/localStorage";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { HoverProvider } from "src/context/hover";
import { OpenGroupsProvider } from "src/context/openGroups";

Expand DownExpand Up@@ -88,6 +91,11 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
);

const [showGantt, setShowGantt] = useLocalStorage<boolean>(showGanttKey(dagId), false);
// Global setting: applies to all Dags (intentionally not scoped to dagId)
const [showVersionIndicatorMode, setShowVersionIndicatorMode] = useLocalStorage<VersionIndicatorOptions>(
`version_indicator_display_mode`,
VersionIndicatorOptions.ALL,
);
const { fitView, getZoom } = useReactFlow();
const { data: warningData } = useDagWarningServiceListDagWarnings({ dagId });
const { onClose, onOpen, open } = useDisclosure();
Expand DownExpand Up@@ -161,8 +169,10 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
setLimit={setLimit}
setRunTypeFilter={setRunTypeFilter}
setShowGantt={setShowGantt}
setShowVersionIndicatorMode={setShowVersionIndicatorMode}
setTriggeringUserFilter={setTriggeringUserFilter}
showGantt={showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUserFilter={triggeringUserFilter}
/>
{dagView === "graph" ? (
Expand All@@ -174,6 +184,7 @@ export const DetailsLayout = ({ children, error, isLoading, tabs }: Props) => {
limit={limit}
runType={runTypeFilter}
showGantt={Boolean(runId) && showGantt}
showVersionIndicatorMode={showVersionIndicatorMode}
triggeringUser={triggeringUserFilter}
/>
{showGantt ? (
Expand Down
27 changes: 22 additions & 5 deletions airflow-core/src/airflow/ui/src/layouts/Details/Grid/Bar.tsx
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,21 +19,27 @@
import { Flex, Box } from "@chakra-ui/react";
import { useParams, useSearchParams } from "react-router-dom";

import type { GridRunsResponse } from "openapi/requests";
import { RunTypeIcon } from "src/components/RunTypeIcon";
import { VersionIndicatorOptions } from "src/constants/showVersionIndicatorOptions";
import { useHover } from "src/context/hover";

import { GridButton } from "./GridButton";

const BAR_HEIGHT = 100;
import { BundleVersionIndicator, DagVersionIndicator } from "./VersionIndicator";
import { BAR_HEIGHT } from "./constants";
import {
getBundleVersion,
getMaxVersionNumber,
type GridRunWithVersionFlags,
} from "./useGridRunsWithVersionFlags";

type Props = {
readonly max: number;
readonly onClick?: () => void;
readonly run: GridRunsResponse;
readonly run: GridRunWithVersionFlags;
readonly showVersionIndicatorMode?: VersionIndicatorOptions;
};

export const Bar = ({ max, onClick, run }: Props) => {
export const Bar = ({ max, onClick, run, showVersionIndicatorMode }: Props) => {
const { dagId = "", runId } = useParams();
const [searchParams] = useSearchParams();
const { hoveredRunId, setHoveredRunId } = useHover();
Expand All@@ -53,6 +59,17 @@ export const Bar = ({ max, onClick, run }: Props) => {
position="relative"
transition="background-color 0.2s"
>
{run.isBundleVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.BUNDLE_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<BundleVersionIndicator bundleVersion={getBundleVersion(run)} />
) : undefined}
{run.isDagVersionChange &&
(showVersionIndicatorMode === VersionIndicatorOptions.DAG_VERSION ||
showVersionIndicatorMode === VersionIndicatorOptions.ALL) ? (
<DagVersionIndicator dagVersionNumber={getMaxVersionNumber(run)} orientation="vertical" />
) : undefined}

<Flex
alignItems="flex-end"
height={BAR_HEIGHT}
Expand Down
Loading
Loading