From cc2915db948c569f6e5a35ee7ba0f0435c2f345c Mon Sep 17 00:00:00 2001 From: Babis Date: Thu, 20 Apr 2023 10:51:18 +0100 Subject: [PATCH] Added index on taskinstance which includes the start_date column --- ..._2_7_0_add_index_to_task_instance_table.py | 50 +++++++ airflow/models/taskinstance.py | 9 ++ docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/img/airflow_erd.svg | 136 +++++++++--------- docs/apache-airflow/migrations-ref.rst | 4 +- newsfragments/30762.improvement.rst | 1 + 6 files changed, 132 insertions(+), 70 deletions(-) create mode 100644 airflow/migrations/versions/0125_2_7_0_add_index_to_task_instance_table.py create mode 100644 newsfragments/30762.improvement.rst diff --git a/airflow/migrations/versions/0125_2_7_0_add_index_to_task_instance_table.py b/airflow/migrations/versions/0125_2_7_0_add_index_to_task_instance_table.py new file mode 100644 index 0000000000000..b9a1df82fd9ff --- /dev/null +++ b/airflow/migrations/versions/0125_2_7_0_add_index_to_task_instance_table.py @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Add index to task_instance table + +Revision ID: 937cbd173ca1 +Revises: 98ae134e6fff +Create Date: 2023-05-03 11:31:32.527362 + +""" +from __future__ import annotations + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "937cbd173ca1" +down_revision = "98ae134e6fff" +branch_labels = None +depends_on = None +airflow_version = "2.7.0" + + +def upgrade(): + """Apply Add index to task_instance table""" + op.create_index( + "ti_state_incl_start_date", + "task_instance", + ["dag_id", "task_id", "state"], + postgresql_include=["start_date"], + ) + + +def downgrade(): + """Unapply Add index to task_instance table""" + op.drop_index("ti_state_incl_start_date", table_name="task_instance") diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 5ee6b01f82327..d2ee3d09b2d0e 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -422,6 +422,15 @@ class TaskInstance(Base, LoggingMixin): Index("ti_dag_run", dag_id, run_id), Index("ti_state", state), Index("ti_state_lkp", dag_id, task_id, run_id, state), + # The below index has been added to improve performance on postgres setups with tens of millions of + # taskinstance rows. Aim is to improve the below query (it can be used to find the last successful + # execution date of a task instance): + # SELECT start_date FROM task_instance WHERE dag_id = 'xx' AND task_id = 'yy' AND state = 'success' + # ORDER BY start_date DESC NULLS LAST LIMIT 1; + # Existing "ti_state_lkp" is not enough for such query when this table has millions of rows, since + # rows have to be fetched in order to retrieve the start_date column. With this index, INDEX ONLY SCAN + # is performed and that query runs within milliseconds. + Index("ti_state_incl_start_date", dag_id, task_id, state, postgresql_include=["start_date"]), Index("ti_pool", pool, state, priority_weight), Index("ti_job_id", job_id), Index("ti_trigger_id", trigger_id), diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index ded035ba22f23..bc462539adbe2 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -b409a0826f10dc5ef934616fe2d5e72fff322bb1aebe321aad475e4d72c25a4f \ No newline at end of file +87f70d3f35c883e544ce7fe6337227c000a9cf995d148648ab19a931b490ecb8 \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index f9895d3c6e8a8..b531759ac4ec7 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -507,8 +507,8 @@ dag--dag_owner_attributes -0..N -{0,1} +1 +1 @@ -536,8 +536,8 @@ dag--dag_schedule_dataset_reference -0..N -{0,1} +1 +1 @@ -557,8 +557,8 @@ dag--dag_tag -0..N -{0,1} +1 +1 @@ -586,8 +586,8 @@ dag--dag_warning -0..N -{0,1} +1 +1 @@ -611,8 +611,8 @@ dag--dataset_dag_run_queue -0..N -{0,1} +1 +1 @@ -644,8 +644,8 @@ dag--task_outlet_dataset_reference -0..N -{0,1} +1 +1 @@ -757,8 +757,8 @@ dag_run--dag_run_note -0..N -{0,1} +1 +1 @@ -778,8 +778,8 @@ dag_run--dagrun_dataset_event -0..N -{0,1} +1 +1 @@ -881,15 +881,15 @@ dag_run--task_instance -0..N -{0,1} +1 +1 dag_run--task_instance -0..N -{0,1} +1 +1 @@ -942,70 +942,70 @@ dag_run--task_reschedule 0..N -{0,1} +1 dag_run--task_reschedule 0..N -{0,1} +1 task_instance--task_instance_note -0..N -{0,1} +1 +1 task_instance--task_instance_note -0..N -{0,1} +1 +1 task_instance--task_instance_note -0..N -{0,1} +1 +1 task_instance--task_instance_note -0..N -{0,1} +1 +1 task_instance--task_reschedule 0..N -{0,1} +1 task_instance--task_reschedule 0..N -{0,1} +1 task_instance--task_reschedule 0..N -{0,1} +1 task_instance--task_reschedule 0..N -{0,1} +1 @@ -1040,29 +1040,29 @@ task_instance--rendered_task_instance_fields -0..N -{0,1} +1 +1 task_instance--rendered_task_instance_fields -0..N -{0,1} +1 +1 task_instance--rendered_task_instance_fields -0..N -{0,1} +1 +1 task_instance--rendered_task_instance_fields -0..N -{0,1} +1 +1 @@ -1104,28 +1104,28 @@ task_instance--task_fail 0..N -{0,1} +1 task_instance--task_fail 0..N -{0,1} +1 task_instance--task_fail 0..N -{0,1} +1 task_instance--task_fail 0..N -{0,1} +1 @@ -1160,29 +1160,29 @@ task_instance--task_map -0..N -{0,1} +1 +1 task_instance--task_map -0..N -{0,1} +1 +1 task_instance--task_map -0..N -{0,1} +1 +1 task_instance--task_map -0..N -{0,1} +1 +1 @@ -1226,28 +1226,28 @@ task_instance--xcom 0..N -{0,1} +1 task_instance--xcom 0..N -{0,1} +1 task_instance--xcom -0..N -{0,1} +1 +1 task_instance--xcom -0..N -{0,1} +1 +1 @@ -1312,22 +1312,22 @@ dataset--dag_schedule_dataset_reference -0..N -{0,1} +1 +1 dataset--dataset_dag_run_queue -0..N -{0,1} +1 +1 dataset--task_outlet_dataset_reference -0..N -{0,1} +1 +1 @@ -1367,8 +1367,8 @@ dataset_event--dagrun_dataset_event -0..N -{0,1} +1 +1 diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 08365cd25035e..2fdc682025a07 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=================================+===================+===================+==============================================================+ -| ``98ae134e6fff`` (head) | ``6abdffdd4815`` | ``2.6.0`` | Increase length of user identifier columns in ``ab_user`` | +| ``937cbd173ca1`` (head) | ``98ae134e6fff`` | ``2.7.0`` | Add index to task_instance table | ++---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ +| ``98ae134e6fff`` | ``6abdffdd4815`` | ``2.6.0`` | Increase length of user identifier columns in ``ab_user`` | | | | | and ``ab_register_user`` tables | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``6abdffdd4815`` | ``290244fb8b83`` | ``2.6.0`` | add dttm index on log table | diff --git a/newsfragments/30762.improvement.rst b/newsfragments/30762.improvement.rst new file mode 100644 index 0000000000000..36aa3ca35b192 --- /dev/null +++ b/newsfragments/30762.improvement.rst @@ -0,0 +1 @@ +PostgreSQL only improvement: Added index on taskinstance table. This index seems to have great positive effect in a setup with tens of millions such rows.