From 5642dc1d3cb6537f81353f0c2af3fd99dd0770aa Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 5 Mar 2023 02:05:48 +0100 Subject: [PATCH] Separate CI Job to run Pytest collection check Before we attempt to run tests in parallel, we quickly check once if Pytest collection works. This is in order to avoid costly parallel test execution if that makes no sense to initialize all the parallel machines. This check used to be done in "Wait for CI Inages" step, but running it there has the undesireable side effect that it is not obvious that it's the collection that fails, also it prevents other jobs (for example static checks and docs building) from running. This means that the contributor does not get all the feedback that could be given immediately. This PR separates the collection into separate job and only makes "test" jobs depend on it - all the other jobs that need CI image depend on "wait for CI image" one and should continue running even if pytest collection fails. CI diagrams are also updated to reflect a bit better optionality and parallelism of the CI jobs. --- .github/workflows/ci.yml | 46 +++++++-- CI.rst | 2 + CI_DIAGRAMS.md | 207 ++++++++++++++++++++++----------------- 3 files changed, 158 insertions(+), 97 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 689dcb5318990..64a95a87058bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -540,8 +540,6 @@ jobs: env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} - - name: "Tests Pytest collection: ${{matrix.python-version}}" - run: breeze shell "python /opt/airflow/scripts/in_container/test_pytest_collection.py" - name: "Fix ownership" run: breeze ci fix-ownership if: always() @@ -834,13 +832,43 @@ jobs: - name: "Post Helm Tests" uses: ./.github/actions/post_tests + test-pytest-collection: + timeout-minutes: 5 + name: "Test Pytest collection" + runs-on: "${{needs.build-info.outputs.runs-on}}" + needs: [build-info, wait-for-ci-images] + if: needs.build-info.outputs.image-build == 'true' + env: + RUNS_ON: "${{needs.build-info.outputs.runs-on}}" + BACKEND: sqlite + PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" + steps: + - name: Cleanup repo + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} + id: wait-for-images + run: breeze ci-image pull --wait-for-image --tag-as-latest + env: + DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + - name: "Tests Pytest collection" + run: breeze shell "python /opt/airflow/scripts/in_container/test_pytest_collection.py" + - name: "Fix ownership" + run: breeze ci fix-ownership + if: always() + tests-postgres: timeout-minutes: 130 name: > Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}} runs-on: "${{needs.build-info.outputs.runs-on}}" - needs: [build-info, wait-for-ci-images] + needs: [build-info, test-pytest-collection] strategy: matrix: python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" @@ -884,7 +912,7 @@ jobs: name: > MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}} runs-on: "${{needs.build-info.outputs.runs-on}}" - needs: [build-info, wait-for-ci-images] + needs: [build-info, test-pytest-collection] strategy: matrix: python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" @@ -928,7 +956,7 @@ jobs: name: > MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}} runs-on: "${{needs.build-info.outputs.runs-on}}" - needs: [build-info, wait-for-ci-images] + needs: [build-info, test-pytest-collection] strategy: matrix: python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" @@ -970,7 +998,7 @@ jobs: name: > Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}} runs-on: "${{needs.build-info.outputs.runs-on}}" - needs: [build-info, wait-for-ci-images] + needs: [build-info, test-pytest-collection] strategy: matrix: python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} @@ -1011,7 +1039,7 @@ jobs: timeout-minutes: 130 name: Integration Tests Postgres runs-on: "${{needs.build-info.outputs.runs-on}}" - needs: [build-info, wait-for-ci-images] + needs: [build-info, test-pytest-collection] env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" TEST_TYPES: "${{needs.build-info.outputs.test-types}}" @@ -1071,7 +1099,7 @@ jobs: timeout-minutes: 130 name: Integration Tests MySQL runs-on: "${{needs.build-info.outputs.runs-on}}" - needs: [build-info, wait-for-ci-images] + needs: [build-info, test-pytest-collection] env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" TEST_TYPES: "${{needs.build-info.outputs.test-types}}" @@ -1112,7 +1140,7 @@ jobs: name: "Quarantined tests" runs-on: "${{needs.build-info.outputs.runs-on}}" continue-on-error: true - needs: [build-info, wait-for-ci-images] + needs: [build-info, test-pytest-collection] env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" TEST_TYPES: "Quarantined" diff --git a/CI.rst b/CI.rst index 8f9862f8354c1..b0b5240423243 100644 --- a/CI.rst +++ b/CI.rst @@ -382,6 +382,8 @@ This workflow is a regular workflow that performs all checks of Airflow code. +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+ | Build docs | Builds documentation | Yes | Yes | Yes | Yes | +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+ +| Test Pytest collection | Tests if pytest collection works | Yes | Yes | Yes | Yes | ++---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+ | Tests | Run the Pytest unit tests (Backend/Python matrix) | Yes | Yes | Yes | Yes (8) | +---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+ | Integration tests | Runs integration tests (Postgres/Mysql) | Yes | Yes | Yes | Yes (9) | diff --git a/CI_DIAGRAMS.md b/CI_DIAGRAMS.md index 4fc7724c15819..a47c62cc70b7e 100644 --- a/CI_DIAGRAMS.md +++ b/CI_DIAGRAMS.md @@ -47,6 +47,10 @@ sequenceDiagram Note over Tests: Test examples
PROD image building and Note over Tests: Test git clone on Windows + and + opt + Note over Tests: Run basic
static checks + end end Build Images ->> GitHub Registry: Push CI Images
[COMMIT_SHA] loop Wait for CI images @@ -58,39 +62,49 @@ sequenceDiagram Note over Build Images: Build PROD Images
[COMMIT_SHA] and opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] Note over Tests: Run static checks end and opt - Note over Tests: Run basic
static checks - end - and - opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] Note over Tests: Build docs end and opt - Note over Tests: Unit Tests - end - and - opt - Note over Tests: Integration Tests - end - and - opt - Note over Tests: Quarantined Tests + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test Pytest collection
[COMMIT_SHA] + par + opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Unit Tests
Python/DB matrix + end + and + opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Integration Tests + end + and + opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Quarantined Tests + end + end end and opt - Note over Tests: Test provider
packages build + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test provider
packages build end and opt - Note over Tests: Test airflow
packages build + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test airflow
packages build end and opt - Note over Tests: Helm tests + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Helm tests end end Note over Tests: Summarize Warnings @@ -102,14 +116,17 @@ sequenceDiagram Note over Tests: Verify PROD Image
[COMMIT_SHA] par opt + GitHub Registry ->> Tests: Pull PROD Images
[COMMIT_SHA] Note over Tests: Run Kubernetes
tests end and opt + GitHub Registry ->> Tests: Pull PROD Images
[COMMIT_SHA] Note over Tests: Run docker-compose
tests end end opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] Note over Tests: Generate constraints end Note over Tests: Build ARM CI images @@ -145,6 +162,10 @@ sequenceDiagram Note over Tests: Test examples
PROD image building and Note over Tests: Test git clone on Windows + and + opt + Note over Tests: Run basic
static checks + end end Tests ->> GitHub Registry: Push CI Images
[COMMIT_SHA] GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] @@ -153,39 +174,49 @@ sequenceDiagram Note over Tests: Build PROD Images
[COMMIT_SHA] and opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] Note over Tests: Run static checks end and opt - Note over Tests: Run basic
static checks - end - and - opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] Note over Tests: Build docs end and opt - Note over Tests: Unit Tests + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test Pytest collection
[COMMIT_SHA] + par + opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Unit Tests
Python/DB matrix + end + and + opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Integration Tests + end + and + opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Quarantined Tests + end + end end and opt - Note over Tests: Integration Tests - end - and - opt - Note over Tests: Quarantined Tests - end - and - opt - Note over Tests: Test provider
packages build + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test provider
packages build end and opt - Note over Tests: Test airflow
packages build + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test airflow
packages build end and opt - Note over Tests: Helm tests + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Helm tests end end Note over Tests: Summarize Warnings @@ -194,14 +225,17 @@ sequenceDiagram Note over Tests: Verify PROD Image
[COMMIT_SHA] par opt + GitHub Registry ->> Tests: Pull PROD Images
[COMMIT_SHA] Note over Tests: Run Kubernetes
tests end and opt + GitHub Registry ->> Tests: Pull PROD Images
[COMMIT_SHA] Note over Tests: Run docker-compose
tests end end opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] Note over Tests: Generate constraints end Note over Tests: Build ARM CI images @@ -241,36 +275,36 @@ sequenceDiagram par Note over Tests: Build PROD Images
[COMMIT_SHA] and - opt - Note over Tests: Run static checks - end - and - opt - Note over Tests: Run basic
static checks - end - and - opt - Note over Tests: Build docs - end + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Run static checks and - opt - Note over Tests: Unit Tests - end + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Build docs and - opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test Pytest collection
[COMMIT_SHA] + par + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Unit Tests
Python/DB matrix + and + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] Note over Tests: Integration Tests + and + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Quarantined Tests end and opt - Note over Tests: Test provider
packages build + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test provider
packages build end and - opt - Note over Tests: Test airflow
packages build - end + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test airflow
packages build and opt - Note over Tests: Helm tests + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Helm tests end end Note over Tests: Summarize Warnings @@ -278,20 +312,21 @@ sequenceDiagram GitHub Registry ->> Tests: Pull PROD Image
[COMMIT_SHA] Note over Tests: Verify PROD Image
[COMMIT_SHA] par - opt - Note over Tests: Run Kubernetes
tests - end + GitHub Registry ->> Tests: Pull PROD Image
[COMMIT_SHA] + Note over Tests: Run Kubernetes
tests and - opt - Note over Tests: Run docker-compose
tests - end + GitHub Registry ->> Tests: Pull PROD Image
[COMMIT_SHA] + Note over Tests: Run docker-compose
tests end + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] Note over Tests: Generate constraints Tests ->> Airflow Repo: Push constraints if changed Note over Tests: Build CI Images
[latest]
Use latest constraints Tests ->> GitHub Registry: Push CI Image
[latest] Note over Tests: Build PROD Images
[latest]
Use latest constraints Tests ->> GitHub Registry: Push PROD Image
[latest] + Note over Tests: Build ARM CI images + Tests ->> GitHub Registry: Push ARM CI Image cache Tests -->> Airflow Repo: Status update deactivate Airflow Repo deactivate Tests @@ -326,51 +361,46 @@ sequenceDiagram GitHub Registry ->> Tests: Pull PROD Images
[latest] Note over Tests: Build PROD Images
[COMMIT_SHA] and - opt - Note over Tests: Run static checks - end - and - opt - Note over Tests: Run basic
static checks - end - and - opt - Note over Tests: Build docs - end + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Run static checks and - opt - Note over Tests: Unit Tests - end + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Build docs and - opt + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test Pytest collection
[COMMIT_SHA] + par + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Unit Tests
Python/DB matrix + and + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] Note over Tests: Integration Tests + and + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Quarantined Tests end and - opt - Note over Tests: Test provider
packages build - end + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test provider
packages build and - opt - Note over Tests: Test airflow
packages build - end + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Test airflow
packages build and - opt - Note over Tests: Helm tests - end + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] + Note over Tests: Helm tests end Note over Tests: Summarize Warnings Tests ->> GitHub Registry: Push PROD Images
[COMMIT_SHA] GitHub Registry ->> Tests: Pull PROD Image
[COMMIT_SHA] Note over Tests: Verify PROD Image
[COMMIT_SHA] par - opt - Note over Tests: Run Kubernetes
tests - end + GitHub Registry ->> Tests: Pull PROD Image
[COMMIT_SHA] + Note over Tests: Run Kubernetes
tests and - opt - Note over Tests: Run docker-compose
tests - end + GitHub Registry ->> Tests: Pull PROD Image
[COMMIT_SHA] + Note over Tests: Run docker-compose
tests end + GitHub Registry ->> Tests: Pull CI Images
[COMMIT_SHA] Note over Tests: Generate constraints Tests ->> Airflow Repo: Push constraints if changed Note over Tests: Build CI Images
[latest]
Use latest constraints @@ -378,6 +408,7 @@ sequenceDiagram Note over Tests: Build PROD Images
[latest]
Use latest constraints Tests ->> GitHub Registry: Push PROD Image cache + latest Note over Tests: Build ARM CI images + Tests ->> GitHub Registry: Push ARM CI Image cache Tests -->> Airflow Repo: Status update deactivate Airflow Repo deactivate Tests