Run separate CI job for latest boto/botocore version when needed #30144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 1 commit into from
Mar 16, 2023
Merged

Run separate CI job for latest boto/botocore version when needed #30144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,7 @@ jobs:
providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }}
skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
Expand DownExpand Up@@ -906,6 +907,50 @@ jobs:
- name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
uses: ./.github/actions/post_tests

tests-postgres-boto:
timeout-minutes: 130
name: >
LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}},
Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.test-types}}
runs-on: "${{needs.build-info.outputs.runs-on}}"
needs: [build-info, test-pytest-collection]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
UPGRADE_BOTO: "true"
JOB_ID: >
postgres-${{needs.build-info.outputs.default-python-version}}-
${{needs.build-info.outputs.default-postgres-version}}
COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: ${{needs.build-info.outputs.default-python-version}}:${{needs.build-info.outputs.test-types}}
run: breeze testing tests --run-in-parallel
- name: >
Post Tests: ${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.test-types}}
uses: ./.github/actions/post_tests

tests-mysql:
timeout-minutes: 130
name: >
Expand Down
7 changes: 7 additions & 0 deletions Dockerfile.ci
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1016,6 +1016,13 @@ else
exit 1
fi
fi
if [[ ${UPGRADE_BOTO=} == "true" ]]; then
echo
echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
echo
pip uninstall aiobotocore -y || true
pip install --upgrade boto3 botocore
fi
readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
ALL_TESTS ALL_PRESELECTED_TESTS

Expand Down
8 changes: 8 additions & 0 deletions dev/breeze/src/airflow_breeze/commands/testing_commands.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -357,6 +357,12 @@ def run_tests_in_parallel(
is_flag=True,
envvar="FULL_TESTS_NEEDED",
)
@click.option(
"--upgrade-boto",
help="Remove aiobotocore and upgrade botocore and boto to the latest version.",
is_flag=True,
envvar="UPGRADE_BOTO",
)
@option_verbose
@option_dry_run
@click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED)
Expand All@@ -380,6 +386,7 @@ def tests(
full_tests_needed: bool,
mount_sources: str,
extra_pytest_args: tuple,
upgrade_boto: bool,
):
docker_filesystem = get_filesystem_type("/var/lib/docker")
get_console().print(f"Docker filesystem: {docker_filesystem}")
Expand All@@ -394,6 +401,7 @@ def tests(
mount_sources=mount_sources,
forward_ports=False,
test_type=test_type,
upgrade_boto=upgrade_boto,
)
rebuild_or_pull_ci_image_if_needed(command_params=exec_shell_params)
cleanup_python_generated_files()
Expand Down
1 change: 1 addition & 0 deletions dev/breeze/src/airflow_breeze/commands/testing_commands_config.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@
"options": [
"--image-tag",
"--mount-sources",
"--upgrade-boto",
],
},
],
Expand Down
1 change: 1 addition & 0 deletions dev/breeze/src/airflow_breeze/params/shell_params.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -112,6 +112,7 @@ class ShellParams:
version_suffix_for_pypi: str = ""
dry_run: bool = False
verbose: bool = False
upgrade_boto: bool = False

def clone_with_test(self, test_type: str) -> ShellParams:
new_params = deepcopy(self)
Expand Down
2 changes: 2 additions & 0 deletions dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -618,6 +618,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false")
set_value_to_default_if_not_set(env, "TEST_TYPE", "")
set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60")
set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false")
set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false")
set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false")
set_value_to_default_if_not_set(env, "VERBOSE", "false")
Expand DownExpand Up@@ -660,6 +661,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
"SKIP_PROVIDER_TESTS": "skip_provider_tests",

@potiukpotiukMar 16, 2023

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is unrelated - but I found it along the way. Separate PR follows.

"SQLITE_URL": "sqlite_url",
"START_AIRFLOW": "start_airflow",
"UPGRADE_BOTO": "upgrade_boto",
"USE_AIRFLOW_VERSION": "use_airflow_version",
"USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
"VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
Expand Down
23 changes: 14 additions & 9 deletions dev/breeze/src/airflow_breeze/utils/selective_checks.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -154,14 +154,14 @@ def __hash__(self):
r"\.py$",
],
FileGroupForCi.ALL_SOURCE_FILES: [
"^.pre-commit-config.yaml$",
"^airflow",
"^chart",
"^tests",
"^kubernetes_tests",
r"^.pre-commit-config.yaml$",
r"^airflow",
r"^chart",
r"^tests",
r"^kubernetes_tests",
],
FileGroupForCi.SYSTEM_TEST_FILES: [
"^tests/system/",
r"^tests/system/",
],
}
)
Expand All@@ -180,10 +180,11 @@ def __hash__(self):
r"^tests/cli",
],
SelectiveUnitTestTypes.PROVIDERS: [
"^airflow/providers/",
"^tests/providers/",
r"^airflow/providers/",
r"^tests/system/providers/",
r"^tests/providers/",
],
SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www"],
SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"],
}
)

Expand DownExpand Up@@ -479,6 +480,10 @@ def needs_api_codegen(self) -> bool:
def run_www_tests(self) -> bool:
return self._should_be_run(FileGroupForCi.WWW_FILES)

@cached_property
def run_amazon_tests(self) -> bool:
return "amazon" in self.test_types or "Providers" in self.test_types.split(" ")

@cached_property
def run_kubernetes_tests(self) -> bool:
return self._should_be_run(FileGroupForCi.KUBERNETES_FILES)
Expand Down
67 changes: 67 additions & 0 deletions dev/breeze/tests/test_selective_checks.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,6 +41,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"image-build": "false",
"needs-helm-tests": "false",
"run-tests": "false",
"run-amazon-tests": "false",
"docs-build": "false",
"upgrade-to-newer-dependencies": "false",
"test-types": "",
Expand All@@ -59,6 +60,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"image-build": "true",
"needs-helm-tests": "false",
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "true",
"upgrade-to-newer-dependencies": "false",
"test-types": "API Always",
Expand All@@ -80,6 +82,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"image-build": "true",
"needs-helm-tests": "false",
"run-tests": "true",
"run-amazon-tests": "true",
"docs-build": "true",
"upgrade-to-newer-dependencies": "false",
"test-types": "API Always Providers[amazon,common.sql,google,postgres]",
Expand All@@ -98,6 +101,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"image-build": "true",
"needs-helm-tests": "false",
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "false",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
Expand All@@ -117,6 +121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"image-build": "true",
"needs-helm-tests": "false",
"run-tests": "false",
"run-amazon-tests": "false",
"docs-build": "true",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
Expand All@@ -139,6 +144,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
"run-amazon-tests": "true",
"docs-build": "true",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
Expand All@@ -163,6 +169,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "true",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
Expand All@@ -187,6 +194,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "true",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
Expand All@@ -212,6 +220,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"needs-helm-tests": "true",
"run-tests": "true",
"docs-build": "true",
"run-amazon-tests": "false",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
"test-types": "Always",
Expand All@@ -230,6 +239,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
"run-amazon-tests": "true",
"docs-build": "true",
"upgrade-to-newer-dependencies": "true",
"test-types": "API Always CLI Core Other Providers WWW",
Expand All@@ -249,13 +259,70 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
"run-amazon-tests": "true",
"docs-build": "true",
"upgrade-to-newer-dependencies": "true",
"test-types": "API Always CLI Core Other Providers WWW",
},
id="Everything should run and upgrading to newer requirements as dependencies change",
)
),
pytest.param(
("airflow/providers/amazon/__init__.py",),
{
"all-python-versions": "['3.7']",
"all-python-versions-list-as-string": "3.7",
"python-versions": "['3.7']",
"python-versions-list-as-string": "3.7",
"image-build": "true",
"needs-helm-tests": "false",
"run-tests": "true",
"docs-build": "true",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"run-amazon-tests": "true",
"test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
"common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
},
id="Providers tests run including amazon tests if amazon provider files changed",
),
pytest.param(
("tests/providers/airbyte/__init__.py",),
{
"all-python-versions": "['3.7']",
"all-python-versions-list-as-string": "3.7",
"python-versions": "['3.7']",
"python-versions-list-as-string": "3.7",
"image-build": "true",
"needs-helm-tests": "false",
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "false",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"test-types": "Always Providers[airbyte,http]",
},
id="Providers tests run including amazon tests if amazon tests provider files changed",
),
pytest.param(
("airflow/providers/amazon/file.py",),
{
"all-python-versions": "['3.7']",
"all-python-versions-list-as-string": "3.7",
"python-versions": "['3.7']",
"python-versions-list-as-string": "3.7",
"image-build": "true",
"needs-helm-tests": "false",
"run-tests": "true",
"run-amazon-tests": "true",
"docs-build": "true",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
"common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
},
id="Providers tests run without amazon tests if no amazon file changed",
),
],
)
def test_expected_output_pull_request_main(
Expand Down
4 changes: 2 additions & 2 deletions images/breeze/output-commands-hash.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -60,5 +60,5 @@ stop:e5aa686b4e53707ced4039d8414d5cd6
testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c
testing:helm-tests:94a442e7f3f63b34c4831a84d165690a
testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98
testing:tests:ce6d80b3073bafd1f10c50812432885a
testing:0f292a29b2a97fbd95778a72249ee997
testing:tests:ddda3dc31e0002d67fad26b682ff689a
testing:0b3792aed114983d6c9d12741561ec1c
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Run separate CI job for latest boto/botocore version when needed #30144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

potiuk
merged 1 commit into

from

Mar 16, 2023
 Merged


Run separate CI job for latest boto/botocore version when needed #30144
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


45 changes: 45 additions & 0 deletions
.github/workflows/ci.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,7 @@ jobs:
 providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }}
 skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }}
 run-tests: ${{ steps.selective-checks.outputs.run-tests }}
 run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
 run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
 run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
 basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
Expand DownExpand Up@@ -906,6 +907,50 @@ jobs:
 - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
 uses: ./.github/actions/post_tests

 tests-postgres-boto:
 timeout-minutes: 130
 name: >
 LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}},
 Py${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 runs-on: "${{needs.build-info.outputs.runs-on}}"
 needs: [build-info, test-pytest-collection]
 env:
 RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
 TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
 PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
 FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
 DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
 BACKEND: "postgres"
 PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
 PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
 POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 UPGRADE_BOTO: "true"
 JOB_ID: >
 postgres-${{needs.build-info.outputs.default-python-version}}-
 ${{needs.build-info.outputs.default-postgres-version}}
 COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
 if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
 steps:
 - name: Cleanup repo
 shell: bash
 run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
 - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
 uses: actions/checkout@v3
 with:
 persist-credentials: false
 - name: >
 Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
 uses: ./.github/actions/prepare_breeze_and_image
 - name: >
 Tests: ${{needs.build-info.outputs.default-python-version}}:${{needs.build-info.outputs.test-types}}
 run: breeze testing tests --run-in-parallel
 - name: >
 Post Tests: ${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 uses: ./.github/actions/post_tests

 tests-mysql:
 timeout-minutes: 130
 name: >
Expand Down

7 changes: 7 additions & 0 deletions
Dockerfile.ci

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1016,6 +1016,13 @@ else
 exit 1
 fi
fi
if [[ ${UPGRADE_BOTO=} == "true" ]]; then
 echo
 echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
 echo
 pip uninstall aiobotocore -y || true
 pip install --upgrade boto3 botocore
fi
readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
 ALL_TESTS ALL_PRESELECTED_TESTS

Expand Down

8 changes: 8 additions & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -357,6 +357,12 @@ def run_tests_in_parallel(
 is_flag=True,
 envvar="FULL_TESTS_NEEDED",
)
@click.option(
 "--upgrade-boto",
 help="Remove aiobotocore and upgrade botocore and boto to the latest version.",
 is_flag=True,
 envvar="UPGRADE_BOTO",
)
@option_verbose
@option_dry_run
@click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED)
Expand All@@ -380,6 +386,7 @@ def tests(
 full_tests_needed: bool,
 mount_sources: str,
 extra_pytest_args: tuple,
 upgrade_boto: bool,
):
 docker_filesystem = get_filesystem_type("/var/lib/docker")
 get_console().print(f"Docker filesystem: {docker_filesystem}")
Expand All@@ -394,6 +401,7 @@ def tests(
 mount_sources=mount_sources,
 forward_ports=False,
 test_type=test_type,
 upgrade_boto=upgrade_boto,
 )
 rebuild_or_pull_ci_image_if_needed(command_params=exec_shell_params)
 cleanup_python_generated_files()
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands_config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@
 "options": [
 "--image-tag",
 "--mount-sources",
 "--upgrade-boto",
 ],
 },
 ],
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/params/shell_params.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,6 +112,7 @@ class ShellParams:
 version_suffix_for_pypi: str = ""
 dry_run: bool = False
 verbose: bool = False
 upgrade_boto: bool = False

 def clone_with_test(self, test_type: str) -> ShellParams:
 new_params = deepcopy(self)
Expand Down

2 changes: 2 additions & 0 deletions
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -618,6 +618,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false")
 set_value_to_default_if_not_set(env, "TEST_TYPE", "")
 set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60")
 set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false")
 set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false")
 set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false")
 set_value_to_default_if_not_set(env, "VERBOSE", "false")
Expand DownExpand Up@@ -660,6 +661,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 "SKIP_PROVIDER_TESTS": "skip_provider_tests",
@potiukpotiukMar 16, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is unrelated - but I found it along the way. Separate PR follows.
 "SQLITE_URL": "sqlite_url",
 "START_AIRFLOW": "start_airflow",
 "UPGRADE_BOTO": "upgrade_boto",
 "USE_AIRFLOW_VERSION": "use_airflow_version",
 "USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
 "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
Expand Down

23 changes: 14 additions & 9 deletions
dev/breeze/src/airflow_breeze/utils/selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,14 +154,14 @@ def __hash__(self):
 r"\.py$",
 ],
 FileGroupForCi.ALL_SOURCE_FILES: [
 "^.pre-commit-config.yaml$",
 "^airflow",
 "^chart",
 "^tests",
 "^kubernetes_tests",
r"^.pre-commit-config.yaml$",
r"^airflow",
r"^chart",
r"^tests",
r"^kubernetes_tests",
 ],
 FileGroupForCi.SYSTEM_TEST_FILES: [
 "^tests/system/",
r"^tests/system/",
 ],
 }
)
Expand All@@ -180,10 +180,11 @@ def __hash__(self):
 r"^tests/cli",
 ],
 SelectiveUnitTestTypes.PROVIDERS: [
 "^airflow/providers/",
 "^tests/providers/",
 r"^airflow/providers/",
 r"^tests/system/providers/",
 r"^tests/providers/",
 ],
 SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www"],
 SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"],
 }
)

Expand DownExpand Up@@ -479,6 +480,10 @@ def needs_api_codegen(self) -> bool:
 def run_www_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.WWW_FILES)

 @cached_property
 def run_amazon_tests(self) -> bool:
 return "amazon" in self.test_types or "Providers" in self.test_types.split(" ")

 @cached_property
 def run_kubernetes_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.KUBERNETES_FILES)
Expand Down

67 changes: 67 additions & 0 deletions
dev/breeze/tests/test_selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,6 +41,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "false",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "",
Expand All@@ -59,6 +60,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always",
Expand All@@ -80,6 +82,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always Providers[amazon,common.sql,google,postgres]",
Expand All@@ -98,6 +101,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -117,6 +121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -139,6 +144,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -163,6 +169,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -187,6 +194,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -212,6 +220,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "needs-helm-tests": "true",
 "run-tests": "true",
 "docs-build": "true",
 "run-amazon-tests": "false",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always",
Expand All@@ -230,6 +239,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
Expand All@@ -249,13 +259,70 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
 },
 id="Everything should run and upgrading to newer requirements as dependencies change",
 )
 ),
 pytest.param(
 ("airflow/providers/amazon/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "run-amazon-tests": "true",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run including amazon tests if amazon provider files changed",
 ),
 pytest.param(
 ("tests/providers/airbyte/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[airbyte,http]",
 },
 id="Providers tests run including amazon tests if amazon tests provider files changed",
 ),
 pytest.param(
 ("airflow/providers/amazon/file.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run without amazon tests if no amazon file changed",
 ),
 ],
)
def test_expected_output_pull_request_main(
Expand Down

4 changes: 2 additions & 2 deletions
images/breeze/output-commands-hash.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,5 +60,5 @@ stop:e5aa686b4e53707ced4039d8414d5cd6
testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c
testing:helm-tests:94a442e7f3f63b34c4831a84d165690a
testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98
testing:tests:ce6d80b3073bafd1f10c50812432885a
testing:0f292a29b2a97fbd95778a72249ee997
testing:tests:ddda3dc31e0002d67fad26b682ff689a
testing:0b3792aed114983d6c9d12741561ec1c
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Run separate CI job for latest boto/botocore version when needed #30144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

potiuk
merged 1 commit into

from

Mar 16, 2023
 Merged


Run separate CI job for latest boto/botocore version when needed #30144
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


45 changes: 45 additions & 0 deletions
.github/workflows/ci.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,7 @@ jobs:
 providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }}
 skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }}
 run-tests: ${{ steps.selective-checks.outputs.run-tests }}
 run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
 run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
 run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
 basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
Expand DownExpand Up@@ -906,6 +907,50 @@ jobs:
 - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
 uses: ./.github/actions/post_tests

 tests-postgres-boto:
 timeout-minutes: 130
 name: >
 LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}},
 Py${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 runs-on: "${{needs.build-info.outputs.runs-on}}"
 needs: [build-info, test-pytest-collection]
 env:
 RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
 TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
 PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
 FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
 DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
 BACKEND: "postgres"
 PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
 PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
 POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 UPGRADE_BOTO: "true"
 JOB_ID: >
 postgres-${{needs.build-info.outputs.default-python-version}}-
 ${{needs.build-info.outputs.default-postgres-version}}
 COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
 if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
 steps:
 - name: Cleanup repo
 shell: bash
 run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
 - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
 uses: actions/checkout@v3
 with:
 persist-credentials: false
 - name: >
 Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
 uses: ./.github/actions/prepare_breeze_and_image
 - name: >
 Tests: ${{needs.build-info.outputs.default-python-version}}:${{needs.build-info.outputs.test-types}}
 run: breeze testing tests --run-in-parallel
 - name: >
 Post Tests: ${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 uses: ./.github/actions/post_tests

 tests-mysql:
 timeout-minutes: 130
 name: >
Expand Down

7 changes: 7 additions & 0 deletions
Dockerfile.ci

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1016,6 +1016,13 @@ else
 exit 1
 fi
fi
if [[ ${UPGRADE_BOTO=} == "true" ]]; then
 echo
 echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
 echo
 pip uninstall aiobotocore -y || true
 pip install --upgrade boto3 botocore
fi
readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
 ALL_TESTS ALL_PRESELECTED_TESTS

Expand Down

8 changes: 8 additions & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -357,6 +357,12 @@ def run_tests_in_parallel(
 is_flag=True,
 envvar="FULL_TESTS_NEEDED",
)
@click.option(
 "--upgrade-boto",
 help="Remove aiobotocore and upgrade botocore and boto to the latest version.",
 is_flag=True,
 envvar="UPGRADE_BOTO",
)
@option_verbose
@option_dry_run
@click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED)
Expand All@@ -380,6 +386,7 @@ def tests(
 full_tests_needed: bool,
 mount_sources: str,
 extra_pytest_args: tuple,
 upgrade_boto: bool,
):
 docker_filesystem = get_filesystem_type("/var/lib/docker")
 get_console().print(f"Docker filesystem: {docker_filesystem}")
Expand All@@ -394,6 +401,7 @@ def tests(
 mount_sources=mount_sources,
 forward_ports=False,
 test_type=test_type,
 upgrade_boto=upgrade_boto,
 )
 rebuild_or_pull_ci_image_if_needed(command_params=exec_shell_params)
 cleanup_python_generated_files()
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands_config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@
 "options": [
 "--image-tag",
 "--mount-sources",
 "--upgrade-boto",
 ],
 },
 ],
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/params/shell_params.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,6 +112,7 @@ class ShellParams:
 version_suffix_for_pypi: str = ""
 dry_run: bool = False
 verbose: bool = False
 upgrade_boto: bool = False

 def clone_with_test(self, test_type: str) -> ShellParams:
 new_params = deepcopy(self)
Expand Down

2 changes: 2 additions & 0 deletions
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -618,6 +618,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false")
 set_value_to_default_if_not_set(env, "TEST_TYPE", "")
 set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60")
 set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false")
 set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false")
 set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false")
 set_value_to_default_if_not_set(env, "VERBOSE", "false")
Expand DownExpand Up@@ -660,6 +661,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 "SKIP_PROVIDER_TESTS": "skip_provider_tests",
@potiukpotiukMar 16, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is unrelated - but I found it along the way. Separate PR follows.
 "SQLITE_URL": "sqlite_url",
 "START_AIRFLOW": "start_airflow",
 "UPGRADE_BOTO": "upgrade_boto",
 "USE_AIRFLOW_VERSION": "use_airflow_version",
 "USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
 "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
Expand Down

23 changes: 14 additions & 9 deletions
dev/breeze/src/airflow_breeze/utils/selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,14 +154,14 @@ def __hash__(self):
 r"\.py$",
 ],
 FileGroupForCi.ALL_SOURCE_FILES: [
 "^.pre-commit-config.yaml$",
 "^airflow",
 "^chart",
 "^tests",
 "^kubernetes_tests",
r"^.pre-commit-config.yaml$",
r"^airflow",
r"^chart",
r"^tests",
r"^kubernetes_tests",
 ],
 FileGroupForCi.SYSTEM_TEST_FILES: [
 "^tests/system/",
r"^tests/system/",
 ],
 }
)
Expand All@@ -180,10 +180,11 @@ def __hash__(self):
 r"^tests/cli",
 ],
 SelectiveUnitTestTypes.PROVIDERS: [
 "^airflow/providers/",
 "^tests/providers/",
 r"^airflow/providers/",
 r"^tests/system/providers/",
 r"^tests/providers/",
 ],
 SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www"],
 SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"],
 }
)

Expand DownExpand Up@@ -479,6 +480,10 @@ def needs_api_codegen(self) -> bool:
 def run_www_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.WWW_FILES)

 @cached_property
 def run_amazon_tests(self) -> bool:
 return "amazon" in self.test_types or "Providers" in self.test_types.split(" ")

 @cached_property
 def run_kubernetes_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.KUBERNETES_FILES)
Expand Down

67 changes: 67 additions & 0 deletions
dev/breeze/tests/test_selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,6 +41,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "false",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "",
Expand All@@ -59,6 +60,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always",
Expand All@@ -80,6 +82,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always Providers[amazon,common.sql,google,postgres]",
Expand All@@ -98,6 +101,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -117,6 +121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -139,6 +144,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -163,6 +169,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -187,6 +194,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -212,6 +220,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "needs-helm-tests": "true",
 "run-tests": "true",
 "docs-build": "true",
 "run-amazon-tests": "false",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always",
Expand All@@ -230,6 +239,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
Expand All@@ -249,13 +259,70 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
 },
 id="Everything should run and upgrading to newer requirements as dependencies change",
 )
 ),
 pytest.param(
 ("airflow/providers/amazon/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "run-amazon-tests": "true",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run including amazon tests if amazon provider files changed",
 ),
 pytest.param(
 ("tests/providers/airbyte/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[airbyte,http]",
 },
 id="Providers tests run including amazon tests if amazon tests provider files changed",
 ),
 pytest.param(
 ("airflow/providers/amazon/file.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run without amazon tests if no amazon file changed",
 ),
 ],
)
def test_expected_output_pull_request_main(
Expand Down

4 changes: 2 additions & 2 deletions
images/breeze/output-commands-hash.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,5 +60,5 @@ stop:e5aa686b4e53707ced4039d8414d5cd6
testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c
testing:helm-tests:94a442e7f3f63b34c4831a84d165690a
testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98
testing:tests:ce6d80b3073bafd1f10c50812432885a
testing:0f292a29b2a97fbd95778a72249ee997
testing:tests:ddda3dc31e0002d67fad26b682ff689a
testing:0b3792aed114983d6c9d12741561ec1c
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Run separate CI job for latest boto/botocore version when needed #30144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

potiuk
merged 1 commit into

from

Mar 16, 2023
 Merged


Run separate CI job for latest boto/botocore version when needed #30144
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


45 changes: 45 additions & 0 deletions
.github/workflows/ci.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,7 @@ jobs:
 providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }}
 skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }}
 run-tests: ${{ steps.selective-checks.outputs.run-tests }}
 run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
 run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
 run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
 basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
Expand DownExpand Up@@ -906,6 +907,50 @@ jobs:
 - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
 uses: ./.github/actions/post_tests

 tests-postgres-boto:
 timeout-minutes: 130
 name: >
 LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}},
 Py${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 runs-on: "${{needs.build-info.outputs.runs-on}}"
 needs: [build-info, test-pytest-collection]
 env:
 RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
 TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
 PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
 FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
 DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
 BACKEND: "postgres"
 PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
 PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
 POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 UPGRADE_BOTO: "true"
 JOB_ID: >
 postgres-${{needs.build-info.outputs.default-python-version}}-
 ${{needs.build-info.outputs.default-postgres-version}}
 COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
 if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
 steps:
 - name: Cleanup repo
 shell: bash
 run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
 - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
 uses: actions/checkout@v3
 with:
 persist-credentials: false
 - name: >
 Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
 uses: ./.github/actions/prepare_breeze_and_image
 - name: >
 Tests: ${{needs.build-info.outputs.default-python-version}}:${{needs.build-info.outputs.test-types}}
 run: breeze testing tests --run-in-parallel
 - name: >
 Post Tests: ${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 uses: ./.github/actions/post_tests

 tests-mysql:
 timeout-minutes: 130
 name: >
Expand Down

7 changes: 7 additions & 0 deletions
Dockerfile.ci

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1016,6 +1016,13 @@ else
 exit 1
 fi
fi
if [[ ${UPGRADE_BOTO=} == "true" ]]; then
 echo
 echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
 echo
 pip uninstall aiobotocore -y || true
 pip install --upgrade boto3 botocore
fi
readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
 ALL_TESTS ALL_PRESELECTED_TESTS

Expand Down

8 changes: 8 additions & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -357,6 +357,12 @@ def run_tests_in_parallel(
 is_flag=True,
 envvar="FULL_TESTS_NEEDED",
)
@click.option(
 "--upgrade-boto",
 help="Remove aiobotocore and upgrade botocore and boto to the latest version.",
 is_flag=True,
 envvar="UPGRADE_BOTO",
)
@option_verbose
@option_dry_run
@click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED)
Expand All@@ -380,6 +386,7 @@ def tests(
 full_tests_needed: bool,
 mount_sources: str,
 extra_pytest_args: tuple,
 upgrade_boto: bool,
):
 docker_filesystem = get_filesystem_type("/var/lib/docker")
 get_console().print(f"Docker filesystem: {docker_filesystem}")
Expand All@@ -394,6 +401,7 @@ def tests(
 mount_sources=mount_sources,
 forward_ports=False,
 test_type=test_type,
 upgrade_boto=upgrade_boto,
 )
 rebuild_or_pull_ci_image_if_needed(command_params=exec_shell_params)
 cleanup_python_generated_files()
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands_config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@
 "options": [
 "--image-tag",
 "--mount-sources",
 "--upgrade-boto",
 ],
 },
 ],
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/params/shell_params.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,6 +112,7 @@ class ShellParams:
 version_suffix_for_pypi: str = ""
 dry_run: bool = False
 verbose: bool = False
 upgrade_boto: bool = False

 def clone_with_test(self, test_type: str) -> ShellParams:
 new_params = deepcopy(self)
Expand Down

2 changes: 2 additions & 0 deletions
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -618,6 +618,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false")
 set_value_to_default_if_not_set(env, "TEST_TYPE", "")
 set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60")
 set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false")
 set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false")
 set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false")
 set_value_to_default_if_not_set(env, "VERBOSE", "false")
Expand DownExpand Up@@ -660,6 +661,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 "SKIP_PROVIDER_TESTS": "skip_provider_tests",
@potiukpotiukMar 16, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is unrelated - but I found it along the way. Separate PR follows.
 "SQLITE_URL": "sqlite_url",
 "START_AIRFLOW": "start_airflow",
 "UPGRADE_BOTO": "upgrade_boto",
 "USE_AIRFLOW_VERSION": "use_airflow_version",
 "USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
 "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
Expand Down

23 changes: 14 additions & 9 deletions
dev/breeze/src/airflow_breeze/utils/selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,14 +154,14 @@ def __hash__(self):
 r"\.py$",
 ],
 FileGroupForCi.ALL_SOURCE_FILES: [
 "^.pre-commit-config.yaml$",
 "^airflow",
 "^chart",
 "^tests",
 "^kubernetes_tests",
r"^.pre-commit-config.yaml$",
r"^airflow",
r"^chart",
r"^tests",
r"^kubernetes_tests",
 ],
 FileGroupForCi.SYSTEM_TEST_FILES: [
 "^tests/system/",
r"^tests/system/",
 ],
 }
)
Expand All@@ -180,10 +180,11 @@ def __hash__(self):
 r"^tests/cli",
 ],
 SelectiveUnitTestTypes.PROVIDERS: [
 "^airflow/providers/",
 "^tests/providers/",
 r"^airflow/providers/",
 r"^tests/system/providers/",
 r"^tests/providers/",
 ],
 SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www"],
 SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"],
 }
)

Expand DownExpand Up@@ -479,6 +480,10 @@ def needs_api_codegen(self) -> bool:
 def run_www_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.WWW_FILES)

 @cached_property
 def run_amazon_tests(self) -> bool:
 return "amazon" in self.test_types or "Providers" in self.test_types.split(" ")

 @cached_property
 def run_kubernetes_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.KUBERNETES_FILES)
Expand Down

67 changes: 67 additions & 0 deletions
dev/breeze/tests/test_selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,6 +41,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "false",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "",
Expand All@@ -59,6 +60,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always",
Expand All@@ -80,6 +82,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always Providers[amazon,common.sql,google,postgres]",
Expand All@@ -98,6 +101,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -117,6 +121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -139,6 +144,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -163,6 +169,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -187,6 +194,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -212,6 +220,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "needs-helm-tests": "true",
 "run-tests": "true",
 "docs-build": "true",
 "run-amazon-tests": "false",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always",
Expand All@@ -230,6 +239,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
Expand All@@ -249,13 +259,70 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
 },
 id="Everything should run and upgrading to newer requirements as dependencies change",
 )
 ),
 pytest.param(
 ("airflow/providers/amazon/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "run-amazon-tests": "true",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run including amazon tests if amazon provider files changed",
 ),
 pytest.param(
 ("tests/providers/airbyte/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[airbyte,http]",
 },
 id="Providers tests run including amazon tests if amazon tests provider files changed",
 ),
 pytest.param(
 ("airflow/providers/amazon/file.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run without amazon tests if no amazon file changed",
 ),
 ],
)
def test_expected_output_pull_request_main(
Expand Down

4 changes: 2 additions & 2 deletions
images/breeze/output-commands-hash.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,5 +60,5 @@ stop:e5aa686b4e53707ced4039d8414d5cd6
testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c
testing:helm-tests:94a442e7f3f63b34c4831a84d165690a
testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98
testing:tests:ce6d80b3073bafd1f10c50812432885a
testing:0f292a29b2a97fbd95778a72249ee997
testing:tests:ddda3dc31e0002d67fad26b682ff689a
testing:0b3792aed114983d6c9d12741561ec1c
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Run separate CI job for latest boto/botocore version when needed #30144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

potiuk
merged 1 commit into

from

Mar 16, 2023
 Merged


Run separate CI job for latest boto/botocore version when needed #30144
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


45 changes: 45 additions & 0 deletions
.github/workflows/ci.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,7 @@ jobs:
 providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }}
 skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }}
 run-tests: ${{ steps.selective-checks.outputs.run-tests }}
 run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
 run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
 run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
 basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
Expand DownExpand Up@@ -906,6 +907,50 @@ jobs:
 - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
 uses: ./.github/actions/post_tests

 tests-postgres-boto:
 timeout-minutes: 130
 name: >
 LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}},
 Py${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 runs-on: "${{needs.build-info.outputs.runs-on}}"
 needs: [build-info, test-pytest-collection]
 env:
 RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
 TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
 PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
 FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
 DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
 BACKEND: "postgres"
 PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
 PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
 POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 UPGRADE_BOTO: "true"
 JOB_ID: >
 postgres-${{needs.build-info.outputs.default-python-version}}-
 ${{needs.build-info.outputs.default-postgres-version}}
 COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
 if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
 steps:
 - name: Cleanup repo
 shell: bash
 run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
 - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
 uses: actions/checkout@v3
 with:
 persist-credentials: false
 - name: >
 Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
 uses: ./.github/actions/prepare_breeze_and_image
 - name: >
 Tests: ${{needs.build-info.outputs.default-python-version}}:${{needs.build-info.outputs.test-types}}
 run: breeze testing tests --run-in-parallel
 - name: >
 Post Tests: ${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 uses: ./.github/actions/post_tests

 tests-mysql:
 timeout-minutes: 130
 name: >
Expand Down

7 changes: 7 additions & 0 deletions
Dockerfile.ci

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1016,6 +1016,13 @@ else
 exit 1
 fi
fi
if [[ ${UPGRADE_BOTO=} == "true" ]]; then
 echo
 echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
 echo
 pip uninstall aiobotocore -y || true
 pip install --upgrade boto3 botocore
fi
readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
 ALL_TESTS ALL_PRESELECTED_TESTS

Expand Down

8 changes: 8 additions & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -357,6 +357,12 @@ def run_tests_in_parallel(
 is_flag=True,
 envvar="FULL_TESTS_NEEDED",
)
@click.option(
 "--upgrade-boto",
 help="Remove aiobotocore and upgrade botocore and boto to the latest version.",
 is_flag=True,
 envvar="UPGRADE_BOTO",
)
@option_verbose
@option_dry_run
@click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED)
Expand All@@ -380,6 +386,7 @@ def tests(
 full_tests_needed: bool,
 mount_sources: str,
 extra_pytest_args: tuple,
 upgrade_boto: bool,
):
 docker_filesystem = get_filesystem_type("/var/lib/docker")
 get_console().print(f"Docker filesystem: {docker_filesystem}")
Expand All@@ -394,6 +401,7 @@ def tests(
 mount_sources=mount_sources,
 forward_ports=False,
 test_type=test_type,
 upgrade_boto=upgrade_boto,
 )
 rebuild_or_pull_ci_image_if_needed(command_params=exec_shell_params)
 cleanup_python_generated_files()
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands_config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@
 "options": [
 "--image-tag",
 "--mount-sources",
 "--upgrade-boto",
 ],
 },
 ],
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/params/shell_params.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,6 +112,7 @@ class ShellParams:
 version_suffix_for_pypi: str = ""
 dry_run: bool = False
 verbose: bool = False
 upgrade_boto: bool = False

 def clone_with_test(self, test_type: str) -> ShellParams:
 new_params = deepcopy(self)
Expand Down

2 changes: 2 additions & 0 deletions
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -618,6 +618,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false")
 set_value_to_default_if_not_set(env, "TEST_TYPE", "")
 set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60")
 set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false")
 set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false")
 set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false")
 set_value_to_default_if_not_set(env, "VERBOSE", "false")
Expand DownExpand Up@@ -660,6 +661,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 "SKIP_PROVIDER_TESTS": "skip_provider_tests",
@potiukpotiukMar 16, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is unrelated - but I found it along the way. Separate PR follows.
 "SQLITE_URL": "sqlite_url",
 "START_AIRFLOW": "start_airflow",
 "UPGRADE_BOTO": "upgrade_boto",
 "USE_AIRFLOW_VERSION": "use_airflow_version",
 "USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
 "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
Expand Down

23 changes: 14 additions & 9 deletions
dev/breeze/src/airflow_breeze/utils/selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,14 +154,14 @@ def __hash__(self):
 r"\.py$",
 ],
 FileGroupForCi.ALL_SOURCE_FILES: [
 "^.pre-commit-config.yaml$",
 "^airflow",
 "^chart",
 "^tests",
 "^kubernetes_tests",
r"^.pre-commit-config.yaml$",
r"^airflow",
r"^chart",
r"^tests",
r"^kubernetes_tests",
 ],
 FileGroupForCi.SYSTEM_TEST_FILES: [
 "^tests/system/",
r"^tests/system/",
 ],
 }
)
Expand All@@ -180,10 +180,11 @@ def __hash__(self):
 r"^tests/cli",
 ],
 SelectiveUnitTestTypes.PROVIDERS: [
 "^airflow/providers/",
 "^tests/providers/",
 r"^airflow/providers/",
 r"^tests/system/providers/",
 r"^tests/providers/",
 ],
 SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www"],
 SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"],
 }
)

Expand DownExpand Up@@ -479,6 +480,10 @@ def needs_api_codegen(self) -> bool:
 def run_www_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.WWW_FILES)

 @cached_property
 def run_amazon_tests(self) -> bool:
 return "amazon" in self.test_types or "Providers" in self.test_types.split(" ")

 @cached_property
 def run_kubernetes_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.KUBERNETES_FILES)
Expand Down

67 changes: 67 additions & 0 deletions
dev/breeze/tests/test_selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,6 +41,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "false",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "",
Expand All@@ -59,6 +60,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always",
Expand All@@ -80,6 +82,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always Providers[amazon,common.sql,google,postgres]",
Expand All@@ -98,6 +101,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -117,6 +121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -139,6 +144,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -163,6 +169,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -187,6 +194,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -212,6 +220,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "needs-helm-tests": "true",
 "run-tests": "true",
 "docs-build": "true",
 "run-amazon-tests": "false",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always",
Expand All@@ -230,6 +239,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
Expand All@@ -249,13 +259,70 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
 },
 id="Everything should run and upgrading to newer requirements as dependencies change",
 )
 ),
 pytest.param(
 ("airflow/providers/amazon/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "run-amazon-tests": "true",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run including amazon tests if amazon provider files changed",
 ),
 pytest.param(
 ("tests/providers/airbyte/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[airbyte,http]",
 },
 id="Providers tests run including amazon tests if amazon tests provider files changed",
 ),
 pytest.param(
 ("airflow/providers/amazon/file.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run without amazon tests if no amazon file changed",
 ),
 ],
)
def test_expected_output_pull_request_main(
Expand Down

4 changes: 2 additions & 2 deletions
images/breeze/output-commands-hash.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,5 +60,5 @@ stop:e5aa686b4e53707ced4039d8414d5cd6
testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c
testing:helm-tests:94a442e7f3f63b34c4831a84d165690a
testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98
testing:tests:ce6d80b3073bafd1f10c50812432885a
testing:0f292a29b2a97fbd95778a72249ee997
testing:tests:ddda3dc31e0002d67fad26b682ff689a
testing:0b3792aed114983d6c9d12741561ec1c
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Run separate CI job for latest boto/botocore version when needed #30144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

potiuk
merged 1 commit into

from

Mar 16, 2023
 Merged


Run separate CI job for latest boto/botocore version when needed #30144
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


45 changes: 45 additions & 0 deletions
.github/workflows/ci.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,7 @@ jobs:
 providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }}
 skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }}
 run-tests: ${{ steps.selective-checks.outputs.run-tests }}
 run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
 run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
 run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
 basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
Expand DownExpand Up@@ -906,6 +907,50 @@ jobs:
 - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
 uses: ./.github/actions/post_tests

 tests-postgres-boto:
 timeout-minutes: 130
 name: >
 LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}},
 Py${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 runs-on: "${{needs.build-info.outputs.runs-on}}"
 needs: [build-info, test-pytest-collection]
 env:
 RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
 TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
 PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
 FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
 DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
 BACKEND: "postgres"
 PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
 PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
 POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 UPGRADE_BOTO: "true"
 JOB_ID: >
 postgres-${{needs.build-info.outputs.default-python-version}}-
 ${{needs.build-info.outputs.default-postgres-version}}
 COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
 if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
 steps:
 - name: Cleanup repo
 shell: bash
 run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
 - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
 uses: actions/checkout@v3
 with:
 persist-credentials: false
 - name: >
 Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
 uses: ./.github/actions/prepare_breeze_and_image
 - name: >
 Tests: ${{needs.build-info.outputs.default-python-version}}:${{needs.build-info.outputs.test-types}}
 run: breeze testing tests --run-in-parallel
 - name: >
 Post Tests: ${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 uses: ./.github/actions/post_tests

 tests-mysql:
 timeout-minutes: 130
 name: >
Expand Down

7 changes: 7 additions & 0 deletions
Dockerfile.ci

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1016,6 +1016,13 @@ else
 exit 1
 fi
fi
if [[ ${UPGRADE_BOTO=} == "true" ]]; then
 echo
 echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
 echo
 pip uninstall aiobotocore -y || true
 pip install --upgrade boto3 botocore
fi
readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
 ALL_TESTS ALL_PRESELECTED_TESTS

Expand Down

8 changes: 8 additions & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -357,6 +357,12 @@ def run_tests_in_parallel(
 is_flag=True,
 envvar="FULL_TESTS_NEEDED",
)
@click.option(
 "--upgrade-boto",
 help="Remove aiobotocore and upgrade botocore and boto to the latest version.",
 is_flag=True,
 envvar="UPGRADE_BOTO",
)
@option_verbose
@option_dry_run
@click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED)
Expand All@@ -380,6 +386,7 @@ def tests(
 full_tests_needed: bool,
 mount_sources: str,
 extra_pytest_args: tuple,
 upgrade_boto: bool,
):
 docker_filesystem = get_filesystem_type("/var/lib/docker")
 get_console().print(f"Docker filesystem: {docker_filesystem}")
Expand All@@ -394,6 +401,7 @@ def tests(
 mount_sources=mount_sources,
 forward_ports=False,
 test_type=test_type,
 upgrade_boto=upgrade_boto,
 )
 rebuild_or_pull_ci_image_if_needed(command_params=exec_shell_params)
 cleanup_python_generated_files()
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands_config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@
 "options": [
 "--image-tag",
 "--mount-sources",
 "--upgrade-boto",
 ],
 },
 ],
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/params/shell_params.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,6 +112,7 @@ class ShellParams:
 version_suffix_for_pypi: str = ""
 dry_run: bool = False
 verbose: bool = False
 upgrade_boto: bool = False

 def clone_with_test(self, test_type: str) -> ShellParams:
 new_params = deepcopy(self)
Expand Down

2 changes: 2 additions & 0 deletions
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -618,6 +618,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false")
 set_value_to_default_if_not_set(env, "TEST_TYPE", "")
 set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60")
 set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false")
 set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false")
 set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false")
 set_value_to_default_if_not_set(env, "VERBOSE", "false")
Expand DownExpand Up@@ -660,6 +661,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 "SKIP_PROVIDER_TESTS": "skip_provider_tests",
@potiukpotiukMar 16, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is unrelated - but I found it along the way. Separate PR follows.
 "SQLITE_URL": "sqlite_url",
 "START_AIRFLOW": "start_airflow",
 "UPGRADE_BOTO": "upgrade_boto",
 "USE_AIRFLOW_VERSION": "use_airflow_version",
 "USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
 "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
Expand Down

23 changes: 14 additions & 9 deletions
dev/breeze/src/airflow_breeze/utils/selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,14 +154,14 @@ def __hash__(self):
 r"\.py$",
 ],
 FileGroupForCi.ALL_SOURCE_FILES: [
 "^.pre-commit-config.yaml$",
 "^airflow",
 "^chart",
 "^tests",
 "^kubernetes_tests",
r"^.pre-commit-config.yaml$",
r"^airflow",
r"^chart",
r"^tests",
r"^kubernetes_tests",
 ],
 FileGroupForCi.SYSTEM_TEST_FILES: [
 "^tests/system/",
r"^tests/system/",
 ],
 }
)
Expand All@@ -180,10 +180,11 @@ def __hash__(self):
 r"^tests/cli",
 ],
 SelectiveUnitTestTypes.PROVIDERS: [
 "^airflow/providers/",
 "^tests/providers/",
 r"^airflow/providers/",
 r"^tests/system/providers/",
 r"^tests/providers/",
 ],
 SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www"],
 SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"],
 }
)

Expand DownExpand Up@@ -479,6 +480,10 @@ def needs_api_codegen(self) -> bool:
 def run_www_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.WWW_FILES)

 @cached_property
 def run_amazon_tests(self) -> bool:
 return "amazon" in self.test_types or "Providers" in self.test_types.split(" ")

 @cached_property
 def run_kubernetes_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.KUBERNETES_FILES)
Expand Down

67 changes: 67 additions & 0 deletions
dev/breeze/tests/test_selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,6 +41,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "false",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "",
Expand All@@ -59,6 +60,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always",
Expand All@@ -80,6 +82,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always Providers[amazon,common.sql,google,postgres]",
Expand All@@ -98,6 +101,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -117,6 +121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -139,6 +144,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -163,6 +169,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -187,6 +194,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -212,6 +220,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "needs-helm-tests": "true",
 "run-tests": "true",
 "docs-build": "true",
 "run-amazon-tests": "false",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always",
Expand All@@ -230,6 +239,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
Expand All@@ -249,13 +259,70 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
 },
 id="Everything should run and upgrading to newer requirements as dependencies change",
 )
 ),
 pytest.param(
 ("airflow/providers/amazon/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "run-amazon-tests": "true",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run including amazon tests if amazon provider files changed",
 ),
 pytest.param(
 ("tests/providers/airbyte/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[airbyte,http]",
 },
 id="Providers tests run including amazon tests if amazon tests provider files changed",
 ),
 pytest.param(
 ("airflow/providers/amazon/file.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run without amazon tests if no amazon file changed",
 ),
 ],
)
def test_expected_output_pull_request_main(
Expand Down

4 changes: 2 additions & 2 deletions
images/breeze/output-commands-hash.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,5 +60,5 @@ stop:e5aa686b4e53707ced4039d8414d5cd6
testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c
testing:helm-tests:94a442e7f3f63b34c4831a84d165690a
testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98
testing:tests:ce6d80b3073bafd1f10c50812432885a
testing:0f292a29b2a97fbd95778a72249ee997
testing:tests:ddda3dc31e0002d67fad26b682ff689a
testing:0b3792aed114983d6c9d12741561ec1c
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Run separate CI job for latest boto/botocore version when needed #30144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

potiuk
merged 1 commit into

from

Mar 16, 2023
 Merged


Run separate CI job for latest boto/botocore version when needed #30144
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


45 changes: 45 additions & 0 deletions
.github/workflows/ci.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,7 @@ jobs:
 providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }}
 skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }}
 run-tests: ${{ steps.selective-checks.outputs.run-tests }}
 run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
 run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
 run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
 basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
Expand DownExpand Up@@ -906,6 +907,50 @@ jobs:
 - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
 uses: ./.github/actions/post_tests

 tests-postgres-boto:
 timeout-minutes: 130
 name: >
 LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}},
 Py${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 runs-on: "${{needs.build-info.outputs.runs-on}}"
 needs: [build-info, test-pytest-collection]
 env:
 RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
 TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
 PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
 FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
 DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
 BACKEND: "postgres"
 PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
 PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
 POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 UPGRADE_BOTO: "true"
 JOB_ID: >
 postgres-${{needs.build-info.outputs.default-python-version}}-
 ${{needs.build-info.outputs.default-postgres-version}}
 COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
 if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
 steps:
 - name: Cleanup repo
 shell: bash
 run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
 - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
 uses: actions/checkout@v3
 with:
 persist-credentials: false
 - name: >
 Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
 uses: ./.github/actions/prepare_breeze_and_image
 - name: >
 Tests: ${{needs.build-info.outputs.default-python-version}}:${{needs.build-info.outputs.test-types}}
 run: breeze testing tests --run-in-parallel
 - name: >
 Post Tests: ${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 uses: ./.github/actions/post_tests

 tests-mysql:
 timeout-minutes: 130
 name: >
Expand Down

7 changes: 7 additions & 0 deletions
Dockerfile.ci

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1016,6 +1016,13 @@ else
 exit 1
 fi
fi
if [[ ${UPGRADE_BOTO=} == "true" ]]; then
 echo
 echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
 echo
 pip uninstall aiobotocore -y || true
 pip install --upgrade boto3 botocore
fi
readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
 ALL_TESTS ALL_PRESELECTED_TESTS

Expand Down

8 changes: 8 additions & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -357,6 +357,12 @@ def run_tests_in_parallel(
 is_flag=True,
 envvar="FULL_TESTS_NEEDED",
)
@click.option(
 "--upgrade-boto",
 help="Remove aiobotocore and upgrade botocore and boto to the latest version.",
 is_flag=True,
 envvar="UPGRADE_BOTO",
)
@option_verbose
@option_dry_run
@click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED)
Expand All@@ -380,6 +386,7 @@ def tests(
 full_tests_needed: bool,
 mount_sources: str,
 extra_pytest_args: tuple,
 upgrade_boto: bool,
):
 docker_filesystem = get_filesystem_type("/var/lib/docker")
 get_console().print(f"Docker filesystem: {docker_filesystem}")
Expand All@@ -394,6 +401,7 @@ def tests(
 mount_sources=mount_sources,
 forward_ports=False,
 test_type=test_type,
 upgrade_boto=upgrade_boto,
 )
 rebuild_or_pull_ci_image_if_needed(command_params=exec_shell_params)
 cleanup_python_generated_files()
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands_config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@
 "options": [
 "--image-tag",
 "--mount-sources",
 "--upgrade-boto",
 ],
 },
 ],
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/params/shell_params.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,6 +112,7 @@ class ShellParams:
 version_suffix_for_pypi: str = ""
 dry_run: bool = False
 verbose: bool = False
 upgrade_boto: bool = False

 def clone_with_test(self, test_type: str) -> ShellParams:
 new_params = deepcopy(self)
Expand Down

2 changes: 2 additions & 0 deletions
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -618,6 +618,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false")
 set_value_to_default_if_not_set(env, "TEST_TYPE", "")
 set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60")
 set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false")
 set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false")
 set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false")
 set_value_to_default_if_not_set(env, "VERBOSE", "false")
Expand DownExpand Up@@ -660,6 +661,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 "SKIP_PROVIDER_TESTS": "skip_provider_tests",
@potiukpotiukMar 16, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is unrelated - but I found it along the way. Separate PR follows.
 "SQLITE_URL": "sqlite_url",
 "START_AIRFLOW": "start_airflow",
 "UPGRADE_BOTO": "upgrade_boto",
 "USE_AIRFLOW_VERSION": "use_airflow_version",
 "USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
 "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
Expand Down

23 changes: 14 additions & 9 deletions
dev/breeze/src/airflow_breeze/utils/selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,14 +154,14 @@ def __hash__(self):
 r"\.py$",
 ],
 FileGroupForCi.ALL_SOURCE_FILES: [
 "^.pre-commit-config.yaml$",
 "^airflow",
 "^chart",
 "^tests",
 "^kubernetes_tests",
r"^.pre-commit-config.yaml$",
r"^airflow",
r"^chart",
r"^tests",
r"^kubernetes_tests",
 ],
 FileGroupForCi.SYSTEM_TEST_FILES: [
 "^tests/system/",
r"^tests/system/",
 ],
 }
)
Expand All@@ -180,10 +180,11 @@ def __hash__(self):
 r"^tests/cli",
 ],
 SelectiveUnitTestTypes.PROVIDERS: [
 "^airflow/providers/",
 "^tests/providers/",
 r"^airflow/providers/",
 r"^tests/system/providers/",
 r"^tests/providers/",
 ],
 SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www"],
 SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"],
 }
)

Expand DownExpand Up@@ -479,6 +480,10 @@ def needs_api_codegen(self) -> bool:
 def run_www_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.WWW_FILES)

 @cached_property
 def run_amazon_tests(self) -> bool:
 return "amazon" in self.test_types or "Providers" in self.test_types.split(" ")

 @cached_property
 def run_kubernetes_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.KUBERNETES_FILES)
Expand Down

67 changes: 67 additions & 0 deletions
dev/breeze/tests/test_selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,6 +41,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "false",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "",
Expand All@@ -59,6 +60,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always",
Expand All@@ -80,6 +82,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always Providers[amazon,common.sql,google,postgres]",
Expand All@@ -98,6 +101,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -117,6 +121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -139,6 +144,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -163,6 +169,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -187,6 +194,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -212,6 +220,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "needs-helm-tests": "true",
 "run-tests": "true",
 "docs-build": "true",
 "run-amazon-tests": "false",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always",
Expand All@@ -230,6 +239,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
Expand All@@ -249,13 +259,70 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
 },
 id="Everything should run and upgrading to newer requirements as dependencies change",
 )
 ),
 pytest.param(
 ("airflow/providers/amazon/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "run-amazon-tests": "true",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run including amazon tests if amazon provider files changed",
 ),
 pytest.param(
 ("tests/providers/airbyte/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[airbyte,http]",
 },
 id="Providers tests run including amazon tests if amazon tests provider files changed",
 ),
 pytest.param(
 ("airflow/providers/amazon/file.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run without amazon tests if no amazon file changed",
 ),
 ],
)
def test_expected_output_pull_request_main(
Expand Down

4 changes: 2 additions & 2 deletions
images/breeze/output-commands-hash.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,5 +60,5 @@ stop:e5aa686b4e53707ced4039d8414d5cd6
testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c
testing:helm-tests:94a442e7f3f63b34c4831a84d165690a
testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98
testing:tests:ce6d80b3073bafd1f10c50812432885a
testing:0f292a29b2a97fbd95778a72249ee997
testing:tests:ddda3dc31e0002d67fad26b682ff689a
testing:0b3792aed114983d6c9d12741561ec1c
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Run separate CI job for latest boto/botocore version when needed #30144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

potiuk
merged 1 commit into

from

Mar 16, 2023
 Merged


Run separate CI job for latest boto/botocore version when needed #30144
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


45 changes: 45 additions & 0 deletions
.github/workflows/ci.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -151,6 +151,7 @@ jobs:
 providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }}
 skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }}
 run-tests: ${{ steps.selective-checks.outputs.run-tests }}
 run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
 run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
 run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
 basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
Expand DownExpand Up@@ -906,6 +907,50 @@ jobs:
 - name: "Post Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.test-types}}"
 uses: ./.github/actions/post_tests

 tests-postgres-boto:
 timeout-minutes: 130
 name: >
 LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}},
 Py${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 runs-on: "${{needs.build-info.outputs.runs-on}}"
 needs: [build-info, test-pytest-collection]
 env:
 RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
 TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
 PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
 FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
 DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
 BACKEND: "postgres"
 PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
 PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
 POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
 UPGRADE_BOTO: "true"
 JOB_ID: >
 postgres-${{needs.build-info.outputs.default-python-version}}-
 ${{needs.build-info.outputs.default-postgres-version}}
 COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
 if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
 steps:
 - name: Cleanup repo
 shell: bash
 run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
 - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
 uses: actions/checkout@v3
 with:
 persist-credentials: false
 - name: >
 Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
 uses: ./.github/actions/prepare_breeze_and_image
 - name: >
 Tests: ${{needs.build-info.outputs.default-python-version}}:${{needs.build-info.outputs.test-types}}
 run: breeze testing tests --run-in-parallel
 - name: >
 Post Tests: ${{needs.build-info.outputs.default-python-version}}:
 ${{needs.build-info.outputs.test-types}}
 uses: ./.github/actions/post_tests

 tests-mysql:
 timeout-minutes: 130
 name: >
Expand Down

7 changes: 7 additions & 0 deletions
Dockerfile.ci

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1016,6 +1016,13 @@ else
 exit 1
 fi
fi
if [[ ${UPGRADE_BOTO=} == "true" ]]; then
 echo
 echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
 echo
 pip uninstall aiobotocore -y || true
 pip install --upgrade boto3 botocore
fi
readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
 ALL_TESTS ALL_PRESELECTED_TESTS

Expand Down

8 changes: 8 additions & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -357,6 +357,12 @@ def run_tests_in_parallel(
 is_flag=True,
 envvar="FULL_TESTS_NEEDED",
)
@click.option(
 "--upgrade-boto",
 help="Remove aiobotocore and upgrade botocore and boto to the latest version.",
 is_flag=True,
 envvar="UPGRADE_BOTO",
)
@option_verbose
@option_dry_run
@click.argument("extra_pytest_args", nargs=-1, type=click.UNPROCESSED)
Expand All@@ -380,6 +386,7 @@ def tests(
 full_tests_needed: bool,
 mount_sources: str,
 extra_pytest_args: tuple,
 upgrade_boto: bool,
):
 docker_filesystem = get_filesystem_type("/var/lib/docker")
 get_console().print(f"Docker filesystem: {docker_filesystem}")
Expand All@@ -394,6 +401,7 @@ def tests(
 mount_sources=mount_sources,
 forward_ports=False,
 test_type=test_type,
 upgrade_boto=upgrade_boto,
 )
 rebuild_or_pull_ci_image_if_needed(command_params=exec_shell_params)
 cleanup_python_generated_files()
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/commands/testing_commands_config.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@
 "options": [
 "--image-tag",
 "--mount-sources",
 "--upgrade-boto",
 ],
 },
 ],
Expand Down

1 change: 1 addition & 0 deletions
dev/breeze/src/airflow_breeze/params/shell_params.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -112,6 +112,7 @@ class ShellParams:
 version_suffix_for_pypi: str = ""
 dry_run: bool = False
 verbose: bool = False
 upgrade_boto: bool = False

 def clone_with_test(self, test_type: str) -> ShellParams:
 new_params = deepcopy(self)
Expand Down

2 changes: 2 additions & 0 deletions
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -618,6 +618,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false")
 set_value_to_default_if_not_set(env, "TEST_TYPE", "")
 set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60")
 set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false")
 set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false")
 set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false")
 set_value_to_default_if_not_set(env, "VERBOSE", "false")
Expand DownExpand Up@@ -660,6 +661,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
 "SKIP_PROVIDER_TESTS": "skip_provider_tests",
@potiukpotiukMar 16, 2023

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is unrelated - but I found it along the way. Separate PR follows.
 "SQLITE_URL": "sqlite_url",
 "START_AIRFLOW": "start_airflow",
 "UPGRADE_BOTO": "upgrade_boto",
 "USE_AIRFLOW_VERSION": "use_airflow_version",
 "USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
 "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
Expand Down

23 changes: 14 additions & 9 deletions
dev/breeze/src/airflow_breeze/utils/selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -154,14 +154,14 @@ def __hash__(self):
 r"\.py$",
 ],
 FileGroupForCi.ALL_SOURCE_FILES: [
 "^.pre-commit-config.yaml$",
 "^airflow",
 "^chart",
 "^tests",
 "^kubernetes_tests",
r"^.pre-commit-config.yaml$",
r"^airflow",
r"^chart",
r"^tests",
r"^kubernetes_tests",
 ],
 FileGroupForCi.SYSTEM_TEST_FILES: [
 "^tests/system/",
r"^tests/system/",
 ],
 }
)
Expand All@@ -180,10 +180,11 @@ def __hash__(self):
 r"^tests/cli",
 ],
 SelectiveUnitTestTypes.PROVIDERS: [
 "^airflow/providers/",
 "^tests/providers/",
 r"^airflow/providers/",
 r"^tests/system/providers/",
 r"^tests/providers/",
 ],
 SelectiveUnitTestTypes.WWW: ["^airflow/www", "^tests/www"],
 SelectiveUnitTestTypes.WWW: [r"^airflow/www", r"^tests/www"],
 }
)

Expand DownExpand Up@@ -479,6 +480,10 @@ def needs_api_codegen(self) -> bool:
 def run_www_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.WWW_FILES)

 @cached_property
 def run_amazon_tests(self) -> bool:
 return "amazon" in self.test_types or "Providers" in self.test_types.split(" ")

 @cached_property
 def run_kubernetes_tests(self) -> bool:
 return self._should_be_run(FileGroupForCi.KUBERNETES_FILES)
Expand Down

67 changes: 67 additions & 0 deletions
dev/breeze/tests/test_selective_checks.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,6 +41,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "false",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "",
Expand All@@ -59,6 +60,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always",
Expand All@@ -80,6 +82,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "API Always Providers[amazon,common.sql,google,postgres]",
Expand All@@ -98,6 +101,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -117,6 +121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "false",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -139,6 +144,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -163,6 +169,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -187,6 +194,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "true",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
Expand All@@ -212,6 +220,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "needs-helm-tests": "true",
 "run-tests": "true",
 "docs-build": "true",
 "run-amazon-tests": "false",
 "run-kubernetes-tests": "true",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always",
Expand All@@ -230,6 +239,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
Expand All@@ -249,13 +259,70 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
 "image-build": "true",
 "needs-helm-tests": "true",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "upgrade-to-newer-dependencies": "true",
 "test-types": "API Always CLI Core Other Providers WWW",
 },
 id="Everything should run and upgrading to newer requirements as dependencies change",
 )
 ),
 pytest.param(
 ("airflow/providers/amazon/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "run-amazon-tests": "true",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run including amazon tests if amazon provider files changed",
 ),
 pytest.param(
 ("tests/providers/airbyte/__init__.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "false",
 "docs-build": "false",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[airbyte,http]",
 },
 id="Providers tests run including amazon tests if amazon tests provider files changed",
 ),
 pytest.param(
 ("airflow/providers/amazon/file.py",),
 {
 "all-python-versions": "['3.7']",
 "all-python-versions-list-as-string": "3.7",
 "python-versions": "['3.7']",
 "python-versions-list-as-string": "3.7",
 "image-build": "true",
 "needs-helm-tests": "false",
 "run-tests": "true",
 "run-amazon-tests": "true",
 "docs-build": "true",
 "run-kubernetes-tests": "false",
 "upgrade-to-newer-dependencies": "false",
 "test-types": "Always Providers[amazon,apache.hive,cncf.kubernetes,"
 "common.sql,exasol,ftp,google,imap,mongo,mysql,postgres,salesforce,ssh]",
 },
 id="Providers tests run without amazon tests if no amazon file changed",
 ),
 ],
)
def test_expected_output_pull_request_main(
Expand Down

4 changes: 2 additions & 2 deletions
images/breeze/output-commands-hash.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,5 +60,5 @@ stop:e5aa686b4e53707ced4039d8414d5cd6
testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c
testing:helm-tests:94a442e7f3f63b34c4831a84d165690a
testing:integration-tests:225ddb6243cce5fc64f4824b87adfd98
testing:tests:ce6d80b3073bafd1f10c50812432885a
testing:0f292a29b2a97fbd95778a72249ee997
testing:tests:ddda3dc31e0002d67fad26b682ff689a
testing:0b3792aed114983d6c9d12741561ec1c
Loading