Run stubtest on stubs on different platforms

.github/workflows/daily.yml

@@ -44,9 +44,10 @@ jobs:
  stubtest-third-party:
  name: Check third party stubs with stubtest
  if: ${{ github.repository == 'python/typeshed' || github.event_name == 'workflow_dispatch' }}
- runs-on: ubuntu-20.04
+ runs-on: ${{ matrix.os }}
  strategy:
  matrix:
+ os: ["ubuntu-latest", "windows-latest", "macos-latest"]
  shard-index: [0, 1, 2, 3]
  fail-fast: false
  steps:
@@ -56,12 +57,17 @@ jobs:
  python-version: "3.9"
  - name: Install dependencies
  run: pip install -r requirements-tests.txt
- - name: Install apt packages
- run: |
- sudo apt update
- sudo apt install -y $(python tests/get_apt_packages.py)
  - name: Run stubtest
- run: xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+ shell: bash
+ run: |
+ if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
+ sudo apt update
+ sudo apt install -y $(python tests/get_packages.py)
+
+ xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+ else
+ python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+ fi
 
  stub-uploader:
  name: Run the stub_uploader tests

.github/workflows/stubtest_stdlib.yml

@@ -1,4 +1,4 @@
-name: Run stubtest
+name: Stdlib stubtest
 
 on:
  workflow_dispatch:

.github/workflows/stubtest_third_party.yml

@@ -0,0 +1,70 @@
+name: Third-party stubtest
+
+on:
+ pull_request:
+ paths-ignore:
+ - '**/*.md'
+ - 'scripts/**'
+
+permissions:
+ contents: read
+
+env:
+ PIP_DISABLE_PIP_VERSION_CHECK: 1
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ stubtest-third-party:
+ name: Check third party stubs with stubtest
+
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+ fail-fast: false
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ - uses: actions/setup-python@v4
+ with:
+ python-version: "3.9"
+ - name: Install dependencies
+ run: pip install -r requirements-tests.txt
+ - name: Run stubtest
+ shell: bash
+ run: |
+ STUBS=$(
+ git diff --name-only origin/${{ github.base_ref }} HEAD |
+ # Use the daily.yml workflow to run stubtest on all third party stubs
+ egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done)
+ )
+
+ if [ -n "$STUBS" ]; then
+ echo "Testing $STUBS..."
+ PACKAGES=$(python tests/get_packages.py $STUBS)
+
+ if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
+ if [ -n "$PACKAGES" ]; then
+ echo "Installing apt packages: $PACKAGES"
+ sudo apt update && sudo apt install -y $PACKAGES
+ fi
+ xvfb-run python tests/stubtest_third_party.py $STUBS
+ fi
+
+ if [ "${{ matrix.os }}" = "macos-latest" ]; then
+ # Could install brew packages here if we run into stubs that need it
+ python tests/stubtest_third_party.py $STUBS
+ fi
+
+ if [ "${{ matrix.os }}" = "windows-latest" ]; then
+ # Could install choco packages here if we run into stubs that need it
+ python tests/stubtest_third_party.py $STUBS
+ fi
+ else
+ echo "Nothing to test"
+ fi

.github/workflows/tests.yml

@@ -146,36 +146,3 @@ jobs:
  cd stub_uploader
  pip install -r requirements.txt
  python -m pytest tests
-
- stubtest-third-party:
- name: Check third party stubs with stubtest
- runs-on: ubuntu-20.04
- if: github.event_name == 'pull_request'
- steps:
- - uses: actions/checkout@v3
- with:
- fetch-depth: 0
- - uses: actions/setup-python@v4
- with:
- python-version: "3.9"
- - name: Install dependencies
- run: pip install -r requirements-tests.txt
- - name: Run stubtest
- run: |
- STUBS=$(
- git diff --name-only origin/${{ github.base_ref }} HEAD |
- # Uncomment the following to (very slowly) run on all third party stubs:
- # git ls-files |
- egrep ^stubs/ | cut -d "/" -f 2 | sort -u | (while read stub; do [ -d stubs/$stub ] && echo $stub || true; done)
- )
- if test -n "$STUBS"; then
- echo "Testing $STUBS..."
- APT_PACKAGES=$(python tests/get_apt_packages.py $STUBS)
- if test -n "$APT_PACKAGES"; then
- echo "Installing apt packages: $APT_PACKAGES"
- sudo apt update && sudo apt install -y $APT_PACKAGES
- fi
- xvfb-run python tests/stubtest_third_party.py $STUBS
- else
- echo "Nothing to test"
- fi

CONTRIBUTING.md

@@ -192,6 +192,11 @@ This has the following keys:
 * `apt_dependencies` (default: `[]`): A list of Ubuntu APT packages
  that need to be installed for stubtest to run successfully. These are
  usually packages needed to pip install the implementation distribution.
+* `platforms` (default: `["linux"]`): A list of OSes on which to run stubtest.
+ Can contain `win32`, `linux`, and `darwin` values.
+ If not specified, stubtest is run only on `linux`.
+ Only add extra OSes to the test
+ if there are platform-specific branches in a stubs package.
 
 The format of all `METADATA.toml` files can be checked by running
 `python3 ./tests/check_consistent.py`.

stubs/D3DShot/METADATA.toml

@@ -2,6 +2,7 @@ version = "0.1.*"
 requires = ["types-Pillow"]
 
 [tool.stubtest]
-# The library only works on Windows; we currently only run stubtest on Ubuntu for third-party stubs in CI.
-# See #8660
+# TODO: figure out how to run stubtest for this package
+# (the package pins Pillow in a problematic way)
 skip = true
+platforms = ["win32"]

stubs/JACK-Client/METADATA.toml

@@ -1,4 +1,5 @@
 version = "0.5.*"
 
 [tool.stubtest]
+platforms = ["linux"]
 apt_dependencies = ["libjack-dev"]

stubs/pyaudio/METADATA.toml

@@ -1,4 +1,5 @@
 version = "0.2.*"
 
 [tool.stubtest]
+platforms = ["linux"]
 apt_dependencies = ["portaudio19-dev"]

stubs/pycurl/METADATA.toml

@@ -1,4 +1,5 @@
 version = "7.45.*"
 
 [tool.stubtest]
+platforms = ["linux"]
 apt_dependencies = ["libcurl4-openssl-dev"]

stubs/pywin32/METADATA.toml

@@ -1,6 +1,5 @@
 version = "305.*"
+
 [tool.stubtest]
+platforms = ["win32"]
 ignore_missing_stub = false
-# The library only works on Windows; we currently only run stubtest on Ubuntu for third-party stubs in CI.
-# See #8660
-skip = true

tests/check_consistent.py

@@ -18,7 +18,7 @@
 from utils import VERSIONS_RE, get_all_testcase_directories, get_gitignore_spec, spec_matches_path, strip_comments
 
 metadata_keys = {"version", "requires", "extra_description", "obsolete_since", "no_longer_updated", "tool"}
-tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub"}}
+tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub", "platforms"}}
 extension_descriptions = {".pyi": "stub", ".py": ".py"}
 
 

tests/get_packages.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+import tomli
+
+platform = sys.platform
+distributions = sys.argv[1:]
+if not distributions:
+ distributions = os.listdir("stubs")
+
+metadata_mapping = {
+ "linux": "apt_dependencies",
+ # We could add others here if we run into stubs that need it:
+ # "darwin": "brew_dependencies",
+ # "win32": "choco_dependencies",
+}
+
+if platform in metadata_mapping:
+ for distribution in distributions:
+ with open(f"stubs/{distribution}/METADATA.toml", "rb") as file:
+ for package in tomli.load(file).get("tool", {}).get("stubtest", {}).get(metadata_mapping[platform], []):
+ print(package)

tests/stubtest_third_party.py

@@ -34,6 +34,11 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool:
  print(colored("skipping", "yellow"))
  return True
 
+ platforms_to_test = stubtest_meta.get("platforms", ["linux"])
+ if sys.platform not in platforms_to_test:
+ print(colored(f"skipping, unsupported platform: {sys.platform}, supported: {platforms_to_test}", "yellow"))
+ return True
+
  with tempfile.TemporaryDirectory() as tmp:
  venv_dir = Path(tmp)
  venv.create(venv_dir, with_pip=True, clear=True)
@@ -57,8 +62,8 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool:
  # If @tests/requirements-stubtest.txt exists, run "pip install" on it.
  req_path = dist / "@tests" / "requirements-stubtest.txt"
  if req_path.exists():
+ pip_cmd = [pip_exe, "install", "-r", str(req_path)]
  try:
- pip_cmd = [pip_exe, "install", "-r", str(req_path)]
  subprocess.run(pip_cmd, check=True, capture_output=True)
  except subprocess.CalledProcessError as e:
  print_command_failure("Failed to install requirements", e)
@@ -102,6 +107,9 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool:
  allowlist_path = dist / "@tests/stubtest_allowlist.txt"
  if allowlist_path.exists():
  stubtest_cmd.extend(["--allowlist", str(allowlist_path)])
+ platform_allowlist = dist / f"@tests/stubtest_allowlist_{sys.platform}.txt"
+ if platform_allowlist.exists():
+ stubtest_cmd.extend(["--allowlist", str(platform_allowlist)])
 
  try:
  subprocess.run(stubtest_cmd, env=stubtest_env, check=True, capture_output=True)