diff --git a/.github/workflows/_test_arduino_library.yml b/.github/workflows/_test_arduino_library.yml new file mode 100644 index 00000000000..c1a971cefbc --- /dev/null +++ b/.github/workflows/_test_arduino_library.yml @@ -0,0 +1,79 @@ +name: Test Arduino library + +permissions: + id-token: write + contents: read + +on: + workflow_call: + inputs: + timeout: + description: 'Job timeout in minutes' + required: false + type: number + default: 90 + +jobs: + run: + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + with: + job-name: arduino-library + runner: linux.2xlarge + docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: ${{ inputs.timeout }} + script: | + # The generic Linux job chooses to use base env, not the one setup by the image + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" + + source .ci/scripts/utils.sh + install_executorch "--use-pt-pinned-commit" + + # Schema headers come from a cmake build; the library build script + # refuses to run without them. + cmake -B cmake-out -DCMAKE_BUILD_TYPE=Release + cmake --build cmake-out --target program_schema -j$(nproc) + + cd examples/arduino + ./build_arduino_library.sh + + # A model exported against a different ExecuTorch commit than the one + # that built the library loads fine, resolves every operator, and then + # fails inside Method::execute. Catch that here rather than on a board. + python verify_models.py arduino_lib/ExecuTorch + + # arduino-cli, the Uno Q core, and the Serial dependency the core + # hard-errors without. + export ARDUINO_DIRECTORIES_USER="${RUNNER_TEMP}/arduino" + mkdir -p "${ARDUINO_DIRECTORIES_USER}" + curl -fsSL https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh \ + | BINDIR="${RUNNER_TEMP}/bin" sh + export PATH="${RUNNER_TEMP}/bin:${PATH}" + arduino-cli core update-index + arduino-cli core install arduino:zephyr + arduino-cli lib install Arduino_RouterBridge + + # link_mode=static is not optional. The board defaults to Dynamic, + # which builds the sketch as a Zephyr loadable extension; a library + # this size never starts that way and prints nothing at all. + FQBN="arduino:zephyr:unoq:link_mode=static" + for sketch in arduino_lib/ExecuTorch/examples/*/; do + echo "::group::compile $(basename "${sketch}")" + arduino-cli compile --fqbn "${FQBN}" \ + --libraries arduino_lib "${sketch}" + echo "::endgroup::" + done + + # The library is published through the Arduino Library Manager, which + # applies these rules at submission time. + curl -fsSL https://raw.githubusercontent.com/arduino/arduino-lint/main/etc/install.sh \ + | BINDIR="${RUNNER_TEMP}/bin" sh + cp ../../LICENSE arduino_lib/ExecuTorch/LICENSE + cp README.md arduino_lib/ExecuTorch/README.md + (cd arduino_lib/ExecuTorch && arduino-lint \ + --project-type library --library-manager submit --compliance strict) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index ddad7eebf61..1bf3c7242b7 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -61,6 +61,22 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/test_wheel_package_qnn.sh "${{ matrix.python-version }}" + test-arduino-library: + needs: changed-files + if: | + github.event_name != 'pull_request' || + contains(needs.changed-files.outputs.changed-files, '.github/workflows/_test_arduino_library.yml') || + contains(needs.changed-files.outputs.changed-files, 'backends/cortex_m/') || + contains(needs.changed-files.outputs.changed-files, 'examples/arduino/') || + contains(needs.changed-files.outputs.changed-files, 'kernels/portable/') || + contains(needs.changed-files.outputs.changed-files, 'runtime/') || + contains(needs.changed-files.outputs.changed-files, 'schema/') + name: test-arduino-library + uses: ./.github/workflows/_test_arduino_library.yml + permissions: + id-token: write + contents: read + test-minimal-wheel-linux: needs: changed-files if: | diff --git a/examples/arduino/verify_models.py b/examples/arduino/verify_models.py new file mode 100644 index 00000000000..f25569ee465 --- /dev/null +++ b/examples/arduino/verify_models.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Check the example models against the library that will run them. + + python verify_models.py arduino_lib/ExecuTorch + +A .pte records, per operator call, how many values it puts on the stack. The +generated kernel wrappers check that count and reject anything else. The two +only agree when the model and the library came from the same ExecuTorch +commit, because Cortex-M operator schemas change between releases -- `scratch` +was added to the conv operators in #19636 and #19825. + +A mismatch is invisible until far too late: the program loads, every operator +resolves, and then Method::execute returns InvalidProgram (0x23) naming +nothing useful. This compares the two directly, in about a second, with no +board and no toolchain. +""" + +import argparse +import pathlib +import re +import sys + +from executorch.exir._serialize._program import deserialize_pte_binary + +KERNEL_RE = re.compile(r'Kernel\(\s*"([^"]+)"(.*?)stack\.size\(\) == (\d+)', re.S) + + +def library_expectations(library: pathlib.Path) -> dict[str, int]: + """Stack size each registered kernel wrapper demands, by operator name.""" + generated = list((library / "src/executorch/codegen").glob("Register*Kernels*.cpp")) + if not generated: + sys.exit(f"no kernel registration found under {library}/src/executorch/codegen") + expectations: dict[str, int] = {} + for source in generated: + for match in KERNEL_RE.finditer(source.read_text()): + expectations[match.group(1)] = int(match.group(3)) + return expectations + + +def model_calls(pte: pathlib.Path) -> list[tuple[str, int]]: + """Operator name and stack size for every kernel call in a .pte.""" + parsed = deserialize_pte_binary(pte.read_bytes()) + plan = getattr(parsed, "program", parsed).execution_plan[0] + calls = [] + for chain in plan.chains: + for instruction in chain.instructions: + args = getattr(instruction.instr_args, "args", None) + index = getattr(instruction.instr_args, "op_index", None) + if args is None or index is None: + continue # not a kernel call + op = plan.operators[index] + name = f"{op.name}.{op.overload}" if op.overload else op.name + calls.append((name, len(args))) + return calls + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("library", type=pathlib.Path, help="Generated library root") + args = parser.parse_args() + + expectations = library_expectations(args.library) + models = sorted(args.library.glob("examples/*/model.pte")) + if not models: + # The build converts each model.pte to model.h and removes it, so run + # this against the source tree rather than the packaged library. + models = sorted(pathlib.Path(__file__).parent.glob("examples/*/model.pte")) + if not models: + sys.exit("no example models found") + + failures = 0 + for pte in models: + problems = [] + for name, provided in model_calls(pte): + expected = expectations.get(name) + if expected is None: + problems.append(f"{name}: not registered in the library") + elif expected != provided: + problems.append( + f"{name}: model supplies {provided}, library expects {expected}" + ) + if problems: + failures += len(problems) + print(f"FAIL {pte.parent.name}") + for problem in dict.fromkeys(problems): + print(f" {problem}") + else: + print(f"ok {pte.parent.name}") + + if failures: + print( + "\nThe models and the library came from different ExecuTorch commits.\n" + "Re-export the models from the same checkout that built the library;\n" + "see the pin in extras/PROVENANCE.txt." + ) + return 1 + print(f"\n{len(models)} models match the library") + return 0 + + +if __name__ == "__main__": + sys.exit(main())