From 5ca4c10811d750b85a66e9a22ea3c576072f5128 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 16 Mar 2023 15:28:34 +0100 Subject: [PATCH] Run separate job for latest boto/botocore version when needed By using aiobotocore, we are pinning botocore and boto to a specific version (in aiobotocore extra of amazon provider). This means that the boto/botocore dependencies will be updated less frequent in main (every 3-4 months instead of every few days). This has the danger, that we will detect errors resulting from the upgrade much later, so we should run our amazon tests and depending tests with the latest boto/botocore (having aibotocore uninstalled). All deferrable tests should skip when aiobotocore is missing so only non-deferrable tests should run in this job. --- .github/workflows/ci.yml | 45 ++++++++ Dockerfile.ci | 7 ++ .../commands/testing_commands.py | 8 ++ .../commands/testing_commands_config.py | 1 + .../src/airflow_breeze/params/shell_params.py | 1 + .../utils/docker_command_utils.py | 2 + .../airflow_breeze/utils/selective_checks.py | 23 ++-- dev/breeze/tests/test_selective_checks.py | 67 ++++++++++++ images/breeze/output-commands-hash.txt | 4 +- images/breeze/output_testing.svg | 24 ++--- images/breeze/output_testing_tests.svg | 102 +++++++++--------- scripts/ci/docker-compose/_docker.env | 1 + scripts/ci/docker-compose/base.yml | 1 + scripts/ci/docker-compose/devcontainer.env | 1 + scripts/docker/entrypoint_ci.sh | 7 ++ .../aws/deferrable/hooks/test_base_aws.py | 2 + .../deferrable/hooks/test_redshift_cluster.py | 2 + .../triggers/test_redshift_cluster.py | 2 + 18 files changed, 228 insertions(+), 72 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46d7537fa5f0b..bbee4e92a2e12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,6 +151,7 @@ jobs: providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }} skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }} run-tests: ${{ steps.selective-checks.outputs.run-tests }} + run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} @@ -906,6 +907,50 @@ jobs: - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}" uses: ./.github/actions/post_tests + tests-postgres-boto: + timeout-minutes: 130 + name: > + LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}}, + Py${{needs.build-info.outputs.default-python-version}}: + ${{needs.build-info.outputs.test-types}} + runs-on: "${{needs.build-info.outputs.runs-on}}" + needs: [build-info, test-pytest-collection] + env: + RUNS_ON: "${{needs.build-info.outputs.runs-on}}" + TEST_TYPES: "${{needs.build-info.outputs.test-types}}" + PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" + FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" + DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" + BACKEND: "postgres" + PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" + PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" + POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" + BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" + UPGRADE_BOTO: "true" + JOB_ID: > + postgres-${{needs.build-info.outputs.default-python-version}}- + ${{needs.build-info.outputs.default-postgres-version}} + COVERAGE: "${{needs.build-info.outputs.run-coverage}}" + if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' + steps: + - name: Cleanup repo + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: > + Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} + uses: ./.github/actions/prepare_breeze_and_image + - name: > + Tests: ${{needs.build-info.outputs.default-python-version}}:${{needs.build-info.outputs.test-types}} + run: breeze testing tests --run-in-parallel + - name: > + Post Tests: ${{needs.build-info.outputs.default-python-version}}: + ${{needs.build-info.outputs.test-types}} + uses: ./.github/actions/post_tests + tests-mysql: timeout-minutes: 130 name: > diff --git a/Dockerfile.ci b/Dockerfile.ci index 22a5e95faa957..abde6eefb9868 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1016,6 +1016,13 @@ else exit 1 fi fi +if [[ ${UPGRADE_BOTO=} == "true" ]]; then + echo + echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" + echo + pip uninstall aiobotocore -y || true + pip install --upgrade boto3 botocore +fi readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ ALL_TESTS ALL_PRESELECTED_TESTS diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index def292fa443cb..a0033b9075e1c 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -357,6 +357,12 @@ def run_tests_in_parallel( is_flag=True, envvar="FULL_TESTS_NEEDED", ) +@click.option( + "--upgrade-boto", + help="Remove aiobotocore and upgrade botocore and boto to the latest version.", + is_flag=True, + envvar="UPGRADE_BOTO", +) @option_verbose @option_dry_run @click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED) @@ -380,6 +386,7 @@ def tests( full_tests_needed: bool, mount_sources: str, extra_pytest_args: tuple, + upgrade_boto: bool, ): docker_filesystem = get_filesystem_type("/var/lib/docker") get_console().print(f"Docker filesystem: {docker_filesystem}") @@ -394,6 +401,7 @@ def tests( mount_sources=mount_sources, forward_ports=False, test_type=test_type, + upgrade_boto=upgrade_boto, ) rebuild_or_pull_ci_image_if_needed(command_params=exec_shell_params) cleanup_python_generated_files() diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py index c9d7f86fab367..7cd81a33d9b7d 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py @@ -53,6 +53,7 @@ "options": [ "--image-tag", "--mount-sources", + "--upgrade-boto", ], }, ], diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index a28d62bcad553..10b820617f595 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -112,6 +112,7 @@ class ShellParams: version_suffix_for_pypi: str = "" dry_run: bool = False verbose: bool = False + upgrade_boto: bool = False def clone_with_test(self, test_type: str) -> ShellParams: new_params = deepcopy(self) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 9f5b7a2f1140a..98d341ce19c9e 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -618,6 +618,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false") set_value_to_default_if_not_set(env, "TEST_TYPE", "") set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60") + set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false") set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false") set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false") set_value_to_default_if_not_set(env, "VERBOSE", "false") @@ -660,6 +661,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: "SKIP_PROVIDER_TESTS": "skip_provider_tests", "SQLITE_URL": "sqlite_url", "START_AIRFLOW": "start_airflow", + "UPGRADE_BOTO": "upgrade_boto", "USE_AIRFLOW_VERSION": "use_airflow_version", "USE_PACKAGES_FROM_DIST": "use_packages_from_dist", "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi", diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 75fb5f0faa387..e8911b8eb1cf1 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -154,14 +154,14 @@ def __hash__(self): r"\.py$", ], FileGroupForCi.ALL_SOURCE_FILES: [ - "^.pre-commit-config.yaml$", - "^airflow", - "^chart", - "^tests", - "^kubernetes_tests", + r"^.pre-commit-config.yaml$", + r"^airflow", + r"^chart", + r"^tests", + r"^kubernetes_tests", ], FileGroupForCi.SYSTEM_TEST_FILES: [ - "^tests/system/", + r"^tests/system/", ], } ) @@ -180,10 +180,11 @@ def __hash__(self): r"^tests/cli", ], SelectiveUnitTestTypes.PROVIDERS: [ - "^airflow/providers/", - "^tests/providers/", + r"^airflow/providers/", + r"^tests/system/providers/", + r"^tests/providers/", ], - SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www"], + SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"], } ) @@ -479,6 +480,10 @@ def needs_api_codegen(self) -> bool: def run_www_tests(self) -> bool: return self._should_be_run(FileGroupForCi.WWW_FILES) + @cached_property + def run_amazon_tests(self) -> bool: + return "amazon" in self.test_types or "Providers" in self.test_types.split(" ") + @cached_property def run_kubernetes_tests(self) -> bool: return self._should_be_run(FileGroupForCi.KUBERNETES_FILES) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index c67f0c39c02b7..c468e5a78abd3 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -41,6 +41,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "false", "needs-helm-tests": "false", "run-tests": "false", + "run-amazon-tests": "false", "docs-build": "false", "upgrade-to-newer-dependencies": "false", "test-types": "", @@ -59,6 +60,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", + "run-amazon-tests": "false", "docs-build": "true", "upgrade-to-newer-dependencies": "false", "test-types": "API Always", @@ -80,6 +82,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", + "run-amazon-tests": "true", "docs-build": "true", "upgrade-to-newer-dependencies": "false", "test-types": "API Always Providers[amazon,common.sql,google,postgres]", @@ -98,6 +101,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", + "run-amazon-tests": "false", "docs-build": "false", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", @@ -117,6 +121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "false", "run-tests": "false", + "run-amazon-tests": "false", "docs-build": "true", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", @@ -139,6 +144,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", + "run-amazon-tests": "true", "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -163,6 +169,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", + "run-amazon-tests": "false", "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -187,6 +194,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", + "run-amazon-tests": "false", "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -212,6 +220,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "needs-helm-tests": "true", "run-tests": "true", "docs-build": "true", + "run-amazon-tests": "false", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "test-types": "Always", @@ -230,6 +239,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", + "run-amazon-tests": "true", "docs-build": "true", "upgrade-to-newer-dependencies": "true", "test-types": "API Always CLI Core Other Providers WWW", @@ -249,6 +259,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", + "run-amazon-tests": "true", "docs-build": "true", "upgrade-to-newer-dependencies": "true", "test-types": "API Always CLI Core Other Providers WWW", @@ -256,6 +267,62 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Everything should run and upgrading to newer requirements as dependencies change", ) ), + pytest.param( + ("airflow/providers/amazon/__init__.py",), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "python-versions": "['3.7']", + "python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "docs-build": "true", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "run-amazon-tests": "true", + "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes," + "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]", + }, + id="Providers tests run including amazon tests if amazon provider files changed", + ), + pytest.param( + ("tests/providers/airbyte/__init__.py",), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "python-versions": "['3.7']", + "python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "run-amazon-tests": "false", + "docs-build": "false", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "test-types": "Always Providers[airbyte,http]", + }, + id="Providers tests run including amazon tests if amazon tests provider files changed", + ), + pytest.param( + ("airflow/providers/amazon/file.py",), + { + "all-python-versions": "['3.7']", + "all-python-versions-list-as-string": "3.7", + "python-versions": "['3.7']", + "python-versions-list-as-string": "3.7", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "run-amazon-tests": "true", + "docs-build": "true", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes," + "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]", + }, + id="Providers tests run without amazon tests if no amazon file changed", + ), ], ) def test_expected_output_pull_request_main( diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index b56c3f38668a5..3f197d9eabcce 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -60,5 +60,5 @@ stop:e5aa686b4e53707ced4039d8414d5cd6 testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c testing:helm-tests:94a442e7f3f63b34c4831a84d165690a testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98 -testing:tests:ce6d80b3073bafd1f10c50812432885a -testing:0f292a29b2a97fbd95778a72249ee997 +testing:tests:ddda3dc31e0002d67fad26b682ff689a +testing:0b3792aed114983d6c9d12741561ec1c diff --git a/images/breeze/output_testing.svg b/images/breeze/output_testing.svg index db5690e0ad3e1..691a330582364 100644 --- a/images/breeze/output_testing.svg +++ b/images/breeze/output_testing.svg @@ -35,8 +35,8 @@ .breeze-testing-r1 { fill: #c5c8c6;font-weight: bold } .breeze-testing-r2 { fill: #c5c8c6 } .breeze-testing-r3 { fill: #d0b344;font-weight: bold } -.breeze-testing-r4 { fill: #68a0b3;font-weight: bold } -.breeze-testing-r5 { fill: #868887 } +.breeze-testing-r4 { fill: #868887 } +.breeze-testing-r5 { fill: #68a0b3;font-weight: bold } .breeze-testing-r6 { fill: #98a84b;font-weight: bold } @@ -96,19 +96,19 @@ -Usage: breeze testing [OPTIONS] COMMAND [ARGS]... +Usage: breeze testing [OPTIONS] COMMAND [ARGS]... Tools that developers can use to run tests -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│tests                                    Run the specified unit test targets.                                     │ -│integration-tests                        Run the specified integratio tests.                                      │ -│helm-tests                               Run Helm chart tests.                                                    │ -│docker-compose-tests                     Run docker-compose tests.                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│tests                                    Run the specified unit test targets.                                     │ +│integration-tests                        Run the specified integratio tests.                                      │ +│helm-tests                               Run Helm chart tests.                                                    │ +│docker-compose-tests                     Run docker-compose tests.                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg index 2dbcc1c42ead7..754c0811e21af 100644 --- a/images/breeze/output_testing_tests.svg +++ b/images/breeze/output_testing_tests.svg @@ -1,4 +1,4 @@ - + - + @@ -186,9 +186,12 @@ + + + - Command: testing tests + Command: testing tests @@ -199,53 +202,54 @@ -Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... +Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... Run the specified unit test targets. -╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ -│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ -│should be run - for example --test-type "Providers[airbyte,http]"                           │ -│(All | API | Always | CLI | Core | Other | Providers | WWW | Helm | Postgres | MySQL |      │ -│Quarantine)                                                                                 │ -│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ -│(INTEGER RANGE)                                                                  │ -│[default: 60; x>=0]                                                              │ -│--db-reset-dReset DB when entering the container.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                               │ -│(all | all-testable | cassandra | celery | kerberos | mongo | otel | pinot | statsd | statsd│ -│| trino)                                                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ -│[default: API Always CLI Core Other Providers WWW]              │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -│--full-tests-neededWhether full set of tests is run.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ -│[default: latest]                                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ -│(selected | all | skip | remove)                                                               │ -│[default: selected]                                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ +│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ +│should be run - for example --test-type "Providers[airbyte,http]"                           │ +│(All | API | Always | CLI | Core | Other | Providers | WWW | Helm | Postgres | MySQL |      │ +│Quarantine)                                                                                 │ +│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ +│(INTEGER RANGE)                                                                  │ +│[default: 60; x>=0]                                                              │ +│--db-reset-dReset DB when entering the container.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                               │ +│(all | all-testable | cassandra | celery | kerberos | mongo | otel | pinot | statsd | statsd│ +│| trino)                                                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ +│[default: API Always CLI Core Other Providers WWW]              │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +│--full-tests-neededWhether full set of tests is run.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ +│[default: latest]                                                              │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ +│(selected | all | skip | remove)                                                               │ +│[default: selected]                                                                            │ +│--upgrade-botoRemove aiobotocore and upgrade botocore and boto to the latest version.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env index dbea935eeae90..737b7e46ef4c1 100644 --- a/scripts/ci/docker-compose/_docker.env +++ b/scripts/ci/docker-compose/_docker.env @@ -63,6 +63,7 @@ SKIP_PROVIDER_TESTS SKIP_SSH_SETUP TEST_TIMEOUT TEST_TYPE +UPGRADE_BOTO UPGRADE_TO_NEWER_DEPENDENCIES VERBOSE VERBOSE_COMMANDS diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 7f11966b30702..a35df1963026f 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -76,6 +76,7 @@ services: - SKIP_SSH_SETUP=${SKIP_SSH_SETUP} - TEST_TYPE=${TEST_TYPE} - TEST_TIMEOUT=${TEST_TIMEOUT} + - UPGRADE_BOTO=${UPGRADE_BOTO} - UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} - VERBOSE=${VERBOSE} - VERBOSE_COMMANDS=${VERBOSE_COMMANDS} diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env index c3ca6e842029e..86dd3e562996d 100644 --- a/scripts/ci/docker-compose/devcontainer.env +++ b/scripts/ci/docker-compose/devcontainer.env @@ -61,6 +61,7 @@ SKIP_SSH_SETUP="true" SKIP_ENVIRONMENT_INITIALIZATION="false" SKIP_PROVIDER_TESTS="false" TEST_TYPE= +UPGRADE_BOTO="false" UPGRADE_TO_NEWER_DEPENDENCIES="false" VERBOSE="false" VERBOSE_COMMANDS="false" diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 1ea9307623a5c..98bacb01e4072 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -443,6 +443,13 @@ else exit 1 fi fi +if [[ ${UPGRADE_BOTO=} == "true" ]]; then + echo + echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" + echo + pip uninstall aiobotocore -y || true + pip install --upgrade boto3 botocore +fi readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ ALL_TESTS ALL_PRESELECTED_TESTS diff --git a/tests/providers/amazon/aws/deferrable/hooks/test_base_aws.py b/tests/providers/amazon/aws/deferrable/hooks/test_base_aws.py index eef118ce6794e..969729efa30cd 100644 --- a/tests/providers/amazon/aws/deferrable/hooks/test_base_aws.py +++ b/tests/providers/amazon/aws/deferrable/hooks/test_base_aws.py @@ -24,6 +24,8 @@ from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseAsyncHook from tests.providers.amazon.aws.utils.compat import async_mock +pytest.importorskip("aiobotocore") + try: from aiobotocore.credentials import AioCredentials except ImportError: diff --git a/tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py b/tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py index b9552837d6347..3cbc39cedabc9 100644 --- a/tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py +++ b/tests/providers/amazon/aws/deferrable/hooks/test_redshift_cluster.py @@ -24,6 +24,8 @@ from airflow.providers.amazon.aws.hooks.redshift_cluster import RedshiftAsyncHook from tests.providers.amazon.aws.utils.compat import async_mock +pytest.importorskip("aiobotocore") + class TestRedshiftAsyncHook: @pytest.mark.asyncio diff --git a/tests/providers/amazon/aws/deferrable/triggers/test_redshift_cluster.py b/tests/providers/amazon/aws/deferrable/triggers/test_redshift_cluster.py index 0bfd40065620d..b933cae91707b 100644 --- a/tests/providers/amazon/aws/deferrable/triggers/test_redshift_cluster.py +++ b/tests/providers/amazon/aws/deferrable/triggers/test_redshift_cluster.py @@ -24,6 +24,8 @@ from airflow.triggers.base import TriggerEvent from tests.providers.amazon.aws.utils.compat import async_mock +pytest.importorskip("aiobotocore") + TASK_ID = "redshift_trigger_check" POLLING_PERIOD_SECONDS = 1.0