[Windows] Run Python unit test CI on Windows

.ci/scripts/unittest-windows.ps1

@@ -0,0 +1,36 @@
+param (
+ [string]$editable
+)
+
+Set-PSDebug -Trace 1
+$ErrorActionPreference = 'Stop'
+$PSNativeCommandUseErrorActionPreference = $true
+
+conda create --yes --quiet -n et python=3.12
+conda activate et
+
+# Activate the VS environment - this is required for Dynamo to work, as it uses MSVC.
+# There are a bunch of environment variables that it requires.
+# See https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line.
+& "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\Tools\Launch-VsDevShell.ps1" -Arch amd64
+
+# Install test dependencies
+pip install -r .ci/docker/requirements-ci.txt
+
+if ($editable -eq 'true') {
+ install_executorch.bat --editable
+} else {
+ install_executorch.bat
+}
+if ($LASTEXITCODE -ne 0) {
+ Write-Host "Installation was unsuccessful. Exit code: $LASTEXITCODE."
+ exit $LASTEXITCODE
+}
+
+# Run pytest with coverage
+# pytest -n auto --cov=./ --cov-report=xml
+pytest -v --full-trace -c pytest-windows.ini
+if ($LASTEXITCODE -ne 0) {
+ Write-Host "Pytest invocation was unsuccessful. Exit code: $LASTEXITCODE."
+ exit $LASTEXITCODE
+}

.github/workflows/_unittest.yml

@@ -19,6 +19,7 @@ on:
  required: false
  type: string
  description: Install ExecuTorch in editable mode or not.
+ default: 'false'
  python-version:
  required: false
  type: string
@@ -52,3 +53,14 @@ jobs:
  # This is needed to get the prebuilt PyTorch wheel from S3
  ${CONDA_RUN} --no-capture-output pip install awscli==1.37.21
  .ci/scripts/unittest-macos.sh --build-tool "${{ inputs.build-tool }}" --build-mode "${{ inputs.build-mode }}" --editable "${{ inputs.editable }}"
+
+ windows:
+ if: ${{ inputs.build-tool == 'cmake' }}
+ uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
+ with:
+ submodules: 'recursive'
+ ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+ timeout: 120
+ script: |
+ conda init powershell
+ powershell .ci/scripts/unittest-windows.ps1 -editable "${{ inputs.editable }}"

.gitignore

@@ -65,3 +65,7 @@ xcuserdata/
 
 # Android
 *.aar
+
+# Windows
+*.dll
+*.pyd

backends/xnnpack/CMakeLists.txt

@@ -62,7 +62,7 @@ endforeach()
 if(WIN32 AND NOT CMAKE_CROSSCOMPILING)
  set(MV_COMMAND
  powershell -Command
- "Move-Item -Path ${_xnnpack_flatbuffer__outputs} -Destination ${_xnnpack_schema__outputs}"
+ "Move-Item -Path ${_xnnpack_flatbuffer__outputs} -Destination ${_xnnpack_schema__outputs} -Force"
  )
 else()
  set(MV_COMMAND mv ${_xnnpack_flatbuffer__outputs} ${_xnnpack_schema__outputs})

conftest.py

@@ -0,0 +1,12 @@
+import sys
+
+collect_ignore_glob: list[str] = []
+
+# Skip Apple tests on Windows. Note that some Core ML tests can run on Linux, as the AOT flow
+# is available. Tests will manage this internally. However, the coremltools import is not available
+# on Windows and causes collection to fail. The easiest way to manage this seems to be to just
+# skip collection for this subdirectory on unsupported platforms.
+if sys.platform == "win32":
+ collect_ignore_glob += [
+ "backends/apple/**",
+ ]

examples/apple/coreml/scripts/build_executor_runner.sh

@@ -93,7 +93,7 @@ find "$CMAKE_BUILD_DIR_PATH/" -name 'libcoreml_inmemoryfs.a' -exec cp -f "{}" "$
 find "$CMAKE_BUILD_DIR_PATH/" -name 'libcoremldelegate.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH/libcoremldelegate.a" \;
 find "$CMAKE_BUILD_DIR_PATH/" -name 'libportable_ops_lib.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH/libportable_ops_lib.a" \;
 find "$CMAKE_BUILD_DIR_PATH/" -name 'libportable_kernels.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH/libportable_kernels.a" \;
-cp -f "$CMAKE_BUILD_DIR_PATH/third-party/flatcc_external_project/lib/libflatccrt.a" "$LIBRARIES_DIR_PATH/libflatccrt.a"
+cp -f "$CMAKE_BUILD_DIR_PATH/third-party/flatcc_ep/lib/libflatccrt.a" "$LIBRARIES_DIR_PATH/libflatccrt.a"
 
 # Build the runner
 echo "ExecuTorch: Building runner"

examples/models/llama/source_transformation/custom_kv_cache.py

@@ -269,7 +269,7 @@ def replace_kv_cache_with_quantized_kv_cache(module):
  executorch_package_path = executorch.__path__[-1]
  libs = list(
  glob.glob(
- f"{executorch_package_path}/**/libquantized_ops_aot_lib.*",
+ f"{executorch_package_path}/**/*quantized_ops_aot_lib.*",
  recursive=True,
  )
  )

exir/tracer.py

@@ -48,6 +48,7 @@
 from torch._decomp import get_decompositions
 from torch._dynamo.guards import Guard
 from torch._functorch.eager_transforms import _maybe_unwrap_functional_tensor
+
 from torch.export import default_decompositions
 from torch.func import functionalize
 from torch.fx.operator_schemas import normalize_function

extension/llm/custom_ops/custom_ops.py

@@ -33,7 +33,7 @@
  package_path = Path(__file__).parent.resolve()
  logging.info(f"Looking for libcustom_ops_aot_lib.so in {package_path}")
 
- libs = list(package_path.glob("**/libcustom_ops_aot_lib.*"))
+ libs = list(package_path.glob("**/*custom_ops_aot_lib.*"))
 
  assert len(libs) == 1, f"Expected 1 library but got {len(libs)}"
  logging.info(f"Loading custom ops library: {libs[0]}")

extension/llm/custom_ops/op_tile_crop_aot.py

@@ -13,7 +13,7 @@
  tile_crop = torch.ops.preprocess.tile_crop.default
  assert tile_crop is not None
 except:
- libs = list(Path(__file__).parent.resolve().glob("libcustom_ops_aot_lib.*"))
+ libs = list(Path(__file__).parent.resolve().glob("*custom_ops_aot_lib.*"))
  assert len(libs) == 1, f"Expected 1 library but got {len(libs)}"
  logging.info(f"Loading custom ops library: {libs[0]}")
  torch.ops.load_library(libs[0])

install_executorch.py

@@ -194,14 +194,6 @@ def main(args):
  clean()
  return
 
- cmake_args = [os.getenv("CMAKE_ARGS", "")]
- # Use ClangCL on Windows.
- # ClangCL is an alias to Clang that configures it to work in an MSVC-compatible
- # mode. Using it on Windows to avoid compiler compatibility issues for MSVC.
- if os.name == "nt":
- cmake_args.append("-T ClangCL")
- os.environ["CMAKE_ARGS"] = " ".join(cmake_args)
-
  check_and_update_submodules()
  # This option is used in CI to make sure that PyTorch build from the pinned commit
  # is used instead of nightly. CI jobs wouldn't be able to catch regression from the

install_requirements.py

@@ -112,8 +112,13 @@ def install_requirements(use_pytorch_nightly):
 
  LOCAL_REQUIREMENTS = [
  "third-party/ao", # We need the latest kernels for fast iteration, so not relying on pypi.
- "extension/llm/tokenizers", # TODO(larryliu0820): Setup a pypi package for this.
- ]
+ ] + (
+ [
+ "extension/llm/tokenizers", # TODO(larryliu0820): Setup a pypi package for this.
+ ]
+ if sys.platform != "win32"
+ else []
+ ) # TODO(gjcomer): Re-enable when buildable on Windows.
 
  # Install packages directly from local copy instead of pypi.
  # This is usually not recommended.

kernels/quantized/__init__.py

@@ -7,7 +7,7 @@
 try:
  from pathlib import Path
 
- libs = list(Path(__file__).parent.resolve().glob("**/libquantized_ops_aot_lib.*"))
+ libs = list(Path(__file__).parent.resolve().glob("**/*quantized_ops_aot_lib.*"))
  del Path
  assert len(libs) == 1, f"Expected 1 library but got {len(libs)}"
  import torch as _torch

pytest-windows.ini

@@ -0,0 +1,115 @@
+# NOTE: This file is a copy of pytest.ini, but with additional tests disabled for Windows. This
+# is intended to be a short-term solution to allow for incrementally enabling tests on Windows.
+# This file is intended to be deleted once the enablement is complete.
+
+[pytest]
+addopts =
+ # show summary of all tests that did not pass
+ -rEfX
+ # Make tracebacks shorter
+ --tb=native
+ # capture only Python print and C++ py::print, but not C output (low-level Python errors)
+ --capture=sys
+ # don't suppress warnings, but don't shove them all to the end either
+ -p no:warnings
+ # Ignore backends/arm tests you need to run examples/arm/setup.sh to install some tool to make them work
+ # For GitHub testing this is setup/executed in the unittest-arm job see .github/workflows/pull.yml for more info.
+ --ignore-glob=backends/arm/**/*
+ # explicitly list out tests that are running successfully in oss
+ .ci/scripts/tests
+ examples/models/test
+ devtools/
+ --ignore=devtools/visualization/visualization_utils_test.py
+ # examples
+ # examples/models/llava/test TODO: enable this
+ # exir
+ exir/_serialize/test
+ exir/backend/test
+ exir/dialects/backend/test
+ exir/dialects/edge/test
+ exir/dialects/test
+ exir/emit/test
+ exir/program/test
+ exir/tests/
+ # executorch/export
+ export/tests
+ --ignore=export/tests/test_export_stages.py
+ # kernels/
+ kernels/prim_ops/test
+ kernels/quantized
+ # Because this test depends on test only cpp ops lib
+ # Will add test only cmake targets to re-enable this test
+ # but maybe it is a bit of anti-pattern
+ --ignore=kernels/quantized/test/test_quant_dequant_per_token.py
+ kernels/test/test_case_gen.py
+ # backends/test
+ # This effort is WIP and will be enabled in CI once testing infra
+ # is stable and signal to noise ratio is good (no irrelevant failures).
+ # See https://github.com/pytorch/executorch/discussions/11140
+ --ignore=backends/test
+ backends/test/harness/tests
+ backends/test/suite/tests
+ # backends/xnnpack
+ backends/xnnpack/test/ops
+ --ignore=backends/xnnpack/test/ops/test_bmm.py
+ --ignore=backends/xnnpack/test/ops/test_conv2d.py
+ --ignore=backends/xnnpack/test/ops/test_linear.py
+ --ignore=backends/xnnpack/test/ops/test_sdpa.py
+ backends/xnnpack/test/passes
+ backends/xnnpack/test/recipes
+ backends/xnnpack/test/serialization
+ # backends/apple/coreml
+ backends/apple/coreml/test
+ # extension/
+ extension/llm/custom_ops/test_sdpa_with_kv_cache.py
+ extension/llm/custom_ops/test_update_cache.py
+ extension/llm/custom_ops/test_quantized_sdpa.py
+ extension/pybindings/test
+ extension/training/pybindings/test
+ # Runtime
+ runtime
+ # Tools
+ codegen/test
+ tools/cmake
+ # test TODO: fix these tests
+ # test/end2end/test_end2end.py
+ --ignore=backends/xnnpack/test/ops/linear.py
+ --ignore=backends/xnnpack/test/models/llama2_et_example.py
+ # T200992559: Add torchao to ET as core dependency
+ --ignore=examples/models/llama/tests/test_pre_quantization_transforms.py
+ --ignore=exir/backend/test/demos
+ --ignore=exir/backend/test/test_backends.py
+ --ignore=exir/backend/test/test_backends_lifted.py
+ --ignore=exir/backend/test/test_partitioner.py
+ --ignore=exir/tests/test_common.py
+ --ignore=exir/tests/test_memory_format_ops_pass_aten.py
+ --ignore=exir/tests/test_memory_planning.py
+ --ignore=exir/tests/test_op_convert.py
+ --ignore=exir/tests/test_passes.py
+ --ignore=exir/tests/test_quant_fusion_pass.py
+ --ignore=exir/tests/test_quantization.py
+ --ignore=exir/tests/test_verification.py
+ # Tests that are (temporarily) disabled for Windows
+ # TODO(gjcomer) Re-enable the LLM tests when tokenizers library is available on Windows.
+ #examples/models/llama3_2_vision/preprocess
+ #examples/models/llama3_2_vision/vision_encoder/test
+ #examples/models/llama3_2_vision/text_decoder/test
+ #examples/models/llama/tests
+ #examples/models/llama/config
+ #extension/llm/modules/test
+ #extension/llm/export
+ --deselect=extension/pybindings/test/test_pybindings.py::PybindingsTest::test_method_quantized_ops
+ --deselect=extension/pybindings/test/test_pybindings.py::PybindingsTest::test_quantized_ops
+ --deselect=runtime/test/test_runtime.py::RuntimeTest::test_load_program_with_path
+ --deselect=exir/backend/test/test_compatibility.py::TestCompatibility::test_compatibility_in_runtime
+ --deselect=exir/backend/test/test_compatibility.py::TestCompatibility::test_compatibility_in_runtime_edge_program_manager
+ --deselect=exir/backend/test/test_lowered_backend_module.py::TestBackendAPI::test_emit_lowered_backend_module_end_to_end
+ --deselect=exir/backend/test/test_to_backend_multi_method.py::TestToBackendMultiMethod::test_multi_method_end_to_end
+ --deselect=extension/llm/custom_ops/test_sdpa_with_kv_cache.py::SDPATestForSpeculativeDecode::test_sdpa_with_cache_seq_len_130
+ --deselect=devtools/inspector/tests/inspector_test.py::TestInspector::test_etrecord_populates_correct_edge_dialect_aot_intermediate_outputs
+ --deselect=devtools/inspector/tests/inspector_test.py::TestInspector::test_etrecord_populates_correct_export_program_aot_intermediate_outputs
+
+# run the same tests multiple times to determine their
+# flakiness status. Default to 50 re-runs
+flake-finder = true
+flake-runs = 50

setup.py

@@ -672,6 +672,10 @@ def run(self): # noqa C901
  f"-DCMAKE_BUILD_TYPE={cmake_build_type}",
  ]
 
+ # Use ClangCL on Windows.
+ if _is_windows():
+ cmake_configuration_args += ["-T ClangCL"]
+
  # Allow adding extra cmake args through the environment. Used by some
  # tests and demos to expand the set of targets included in the pip
  # package.
@@ -769,7 +773,7 @@ def run(self): # noqa C901
  # platform-specific files using InstallerBuildExt.
  ext_modules=[
  BuiltFile(
- src_dir="%CMAKE_CACHE_DIR%/third-party/flatbuffers_external_project/bin/",
+ src_dir="%CMAKE_CACHE_DIR%/third-party/flatc_proj/bin/",
  src_name="flatc",
  dst="executorch/data/bin/",
  is_executable=True,
@@ -795,7 +799,8 @@ def run(self): # noqa C901
  dependent_cmake_flags=["EXECUTORCH_BUILD_EXTENSION_TRAINING"],
  ),
  BuiltExtension(
- src="codegen/tools/selective_build.*",
+ src_dir="%CMAKE_CACHE_DIR%/codegen/tools/%BUILD_TYPE%/",
+ src="selective_build.cp*" if _is_windows() else "selective_build.*",
  modpath="executorch.codegen.tools.selective_build",
  dependent_cmake_flags=["EXECUTORCH_BUILD_PYBIND"],
  ),

third-party/CMakeLists.txt

@@ -28,8 +28,8 @@ endif()
 # We use ExternalProject to build flatc from source to force it target the host.
 # Otherwise, flatc will target the project's toolchain (i.e. iOS, or Android).
 ExternalProject_Add(
-flatbuffers_external_project
- PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_external_project
+flatbuffers_ep
+ PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatc_proj
  SOURCE_DIR ${PROJECT_SOURCE_DIR}/third-party/flatbuffers
  CMAKE_ARGS -DFLATBUFFERS_BUILD_FLATC=ON
  -DFLATBUFFERS_INSTALL=ON
@@ -46,9 +46,9 @@ ExternalProject_Add(
  BUILD_BYPRODUCTS <INSTALL_DIR>/bin/flatc
  ${_executorch_external_project_additional_args}
 )
-ExternalProject_Get_Property(flatbuffers_external_project INSTALL_DIR)
+ExternalProject_Get_Property(flatbuffers_ep INSTALL_DIR)
 add_executable(flatc IMPORTED GLOBAL)
-add_dependencies(flatc flatbuffers_external_project)
+add_dependencies(flatc flatbuffers_ep)
 if(WIN32 AND NOT CMAKE_CROSSCOMPILING)
  # flatbuffers does not use CMAKE_BUILD_TYPE. Internally, the build forces Release
  # config, but from CMake's perspective the build type is always Debug.
@@ -79,8 +79,8 @@ endif()
 # Similar to flatbuffers, we want to build flatcc for the host. See inline comments
 # in the flatbuffers ExternalProject_Add for more details.
 ExternalProject_Add(
-flatcc_external_project
- PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatcc_external_project
+flatcc_ep
+ PREFIX ${CMAKE_CURRENT_BINARY_DIR}/flatcc_ep
  SOURCE_DIR ${PROJECT_SOURCE_DIR}/third-party/flatcc
  CMAKE_ARGS -DFLATCC_RTONLY=OFF
  -DFLATCC_TEST=OFF
@@ -98,9 +98,9 @@ ExternalProject_Add(
  {_executorch_external_project_additional_args}
 )
 file(REMOVE_RECURSE ${PROJECT_SOURCE_DIR}/third-party/flatcc/lib)
-ExternalProject_Get_Property(flatcc_external_project INSTALL_DIR)
+ExternalProject_Get_Property(flatcc_ep INSTALL_DIR)
 add_executable(flatcc_cli IMPORTED GLOBAL)
-add_dependencies(flatcc_cli flatcc_external_project)
+add_dependencies(flatcc_cli flatcc_ep)
 if(WIN32 AND NOT CMAKE_CROSSCOMPILING)
  set_target_properties(flatcc_cli PROPERTIES IMPORTED_LOCATION ${INSTALL_DIR}/bin/flatcc.exe)
 else()