diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d6f36924ef..40775a257f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,7 +118,6 @@ set(VCLIBS_DEBUG_OPTIONS "$<$:/Od>") set(VCLIBS_RELEASE_OPTIONS "$<$:/O2;/Os>") if(BUILD_TESTING) - enable_testing() add_subdirectory(tests) endif() diff --git a/README.md b/README.md index 354a729e36e..681caeb8183 100644 --- a/README.md +++ b/README.md @@ -269,34 +269,33 @@ C:\Users\username\Desktop>dumpbin /DEPENDENTS .\example.exe | findstr msvcp * Otherwise, use [LLVM's installer][] and choose to add LLVM to your `PATH` during installation. 4. Follow the instructions below. -## Running All The Tests +## Running The Tests -After configuring and building the project, running `ctest` from the build output directory will run all the tests. -CTest will only display the standard error output of tests that failed. In order to get more details from CTest's -`lit` invocations, run the tests with `ctest -V`. +Our tests are currently split across three test suites that are located at `tests\std`, `tests\tr1`, and +`llvm-project\libcxx\test\std`. The test runner `${PROJECT_BINARY_DIR}\tests\utils\stl-lit\stl-lit.py` accepts paths to +directories in the test suites and runs all tests located in the subtree rooted at those paths. This can mean executing +the entirety of a single test suite, running all tests under a category in `libcxx`, or running a single test in `std` +and `tr1`. -## Running A Subset Of The Tests - -`${PROJECT_BINARY_DIR}\tests\utils\stl-lit\stl-lit.py` can be invoked on a subdirectory of a test suite and will execute -all the tests under that subdirectory. This can mean executing the entirety of a single test suite, running all tests -under a category in libcxx, or running a single test in `std` and `tr1`. +Some useful `stl-lit.py` options: +* `-v` (verbose) tells `stl-lit.py` to show us output from failed test cases. +* `-Dnotags=ASAN` disables the "extra ASan configs" that we typically run only in CI. This is useful to limit runtime + for full validation runs, but often omitted when running just a few test cases to enable the extra ASan coverage. ## Examples These examples assume that your current directory is `C:\Dev\STL\out\x64`. -* This command will run all of the test suites with verbose output. - + `ctest -V` -* This command will also run all of the test suites. - + `python tests\utils\stl-lit\stl-lit.py ..\..\llvm-project\libcxx\test ..\..\tests\std ..\..\tests\tr1` -* This command will run all of the std test suite. - + `python tests\utils\stl-lit\stl-lit.py ..\..\tests\std` +* This command will run all of the test suites: + + `python tests\utils\stl-lit\stl-lit.py -Dnotags=ASAN ..\..\llvm-project\libcxx\test ..\..\tests\std ..\..\tests\tr1` +* This command will run only the std test suite. + + `python tests\utils\stl-lit\stl-lit.py -Dnotags=ASAN ..\..\tests\std` * If you want to run a subset of a test suite, you need to point it to the right place in the sources. The following -will run the single test found under VSO_0000000_any_calling_conventions. - + `python tests\utils\stl-lit\stl-lit.py ..\..\tests\std\tests\VSO_0000000_any_calling_conventions` +will run the single test found under `VSO_0000000_any_calling_conventions`. + + `python tests\utils\stl-lit\stl-lit.py -Dnotags=ASAN ..\..\tests\std\tests\VSO_0000000_any_calling_conventions` * You can invoke `stl-lit` with any arbitrary subdirectory of a test suite. In libcxx this allows you to have finer control over what category of tests you would like to run. The following will run all the libcxx map tests. - + `python tests\utils\stl-lit\stl-lit.py ..\..\llvm-project\libcxx\test\std\containers\associative\map` + + `python tests\utils\stl-lit\stl-lit.py -Dnotags=ASAN ..\..\llvm-project\libcxx\test\std\containers\associative\map` * You can also use the `--filter` option to include tests whose names match a regular expression. The following command will run tests with "atomic_wait" in their names in both the std and libcxx test suites. + `python tests\utils\stl-lit\stl-lit.py ..\..\llvm-project\libcxx\test ..\..\tests\std --filter=atomic_wait` @@ -305,31 +304,8 @@ control over what category of tests you would like to run. The following will ru ## Interpreting The Results Of Tests -### CTest - -When running the tests via CTest, all of the test suites are considered to be a single test. If any single test in a -test suite fails, CTest will simply report that the `stl` test failed. - -Example: -``` -0% tests passed, 1 tests failed out of 1 - -Total Test time (real) = 2441.55 sec - -The following tests FAILED: - 1 - stl (Failed) -``` - -The primary utility of CTest in this case is to conveniently invoke `stl-lit.py` with the correct set of arguments. - -CTest will output everything that was sent to stderr for each of the failed test suites, which can be used to identify -which individual test within the test suite failed. It can sometimes be helpful to run CTest with the `-V` option in -order to see the stdout of the tests. - -### stl-lit - -When running the tests directly via the generated `stl-lit.py` script the result of each test will be printed. The -format of each result is `{Result Code}: {Test Suite Name} :: {Test Name}:{Configuration Number}`. +`stl-lit.py` prints the result of each test. The format of each result is +`{Result Code}: {Test Suite Name} :: {Test Name}:{Configuration Number}`. Example: ``` diff --git a/azure-devops/asan-pipeline.yml b/azure-devops/asan-pipeline.yml index 5107cd41b4a..f98953de548 100644 --- a/azure-devops/asan-pipeline.yml +++ b/azure-devops/asan-pipeline.yml @@ -27,7 +27,7 @@ stages: hostArch: x64 targetArch: x64 asanBuild: true - ctestOptions: '--tests-regex stlasan' + testTargets: STL-ASan-CI - stage: Build_And_Test_x86 displayName: 'Build and Test x86' @@ -41,6 +41,6 @@ stages: hostArch: x86 targetArch: x86 asanBuild: true - ctestOptions: '--tests-regex stlasan' + testTargets: STL-ASan-CI # no coverage for ARM and ARM64 diff --git a/azure-devops/build-and-test.yml b/azure-devops/build-and-test.yml index 6798be4d1b7..edc9134cd93 100644 --- a/azure-devops/build-and-test.yml +++ b/azure-devops/build-and-test.yml @@ -15,9 +15,9 @@ parameters: - name: buildBenchmarks type: boolean default: false -- name: ctestOptions +- name: testTargets type: string - default: '--exclude-regex stlasan' + default: 'STL-CI' - name: numShards type: number default: 8 @@ -61,5 +61,5 @@ jobs: parameters: hostArch: ${{ parameters.hostArch }} targetArch: ${{ parameters.targetArch }} - ctestOptions: ${{ parameters.ctestOptions }} + testTargets: ${{ parameters.testTargets }} skipTesting: ${{ parameters.skipTesting }} diff --git a/azure-devops/run-tests.yml b/azure-devops/run-tests.yml index 27ce640f32f..a6e2c1363d3 100644 --- a/azure-devops/run-tests.yml +++ b/azure-devops/run-tests.yml @@ -6,7 +6,7 @@ parameters: type: string - name: targetArch type: string -- name: ctestOptions +- name: testTargets type: string - name: skipTesting type: boolean @@ -14,7 +14,7 @@ steps: - script: | call "%ProgramFiles%\Microsoft Visual Studio\2022\Preview\Common7\Tools\VsDevCmd.bat" ^ -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo - ctest --verbose ${{ parameters.ctestOptions }} + ninja --verbose -k 0 ${{ parameters.testTargets }} displayName: 'Build and Run Tests' timeoutInMinutes: 30 condition: and(succeeded(), not(${{ parameters.skipTesting }})) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9dc352a0f25..980ddcf194b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -26,19 +26,11 @@ add_subdirectory(utils/stl-lit) find_package(Python "3.13" REQUIRED COMPONENTS Interpreter) if(NOT DEFINED LIT_FLAGS) - list(APPEND LIT_FLAGS "-o" "${CMAKE_CURRENT_BINARY_DIR}/test_results.json") + set(LIT_FLAGS "-o" "${CMAKE_CURRENT_BINARY_DIR}/test_results.json") endif() +set(STL_LIT_COMMAND ${Python_EXECUTABLE} ${STL_LIT_OUTPUT} ${LIT_FLAGS}) get_property(STL_LIT_TEST_DIRS GLOBAL PROPERTY STL_LIT_TEST_DIRS) -list(APPEND STL_LIT_COMMAND "${STL_LIT_OUTPUT}" - "${LIT_FLAGS}" - "-D" "notags=ASAN" - "${STL_LIT_TEST_DIRS}") -list(APPEND STLASAN_LIT_COMMAND "${STL_LIT_OUTPUT}" - "${LIT_FLAGS}" - "-D" "tags=ASAN" - "${STL_LIT_TEST_DIRS}") - -add_test(NAME stl COMMAND ${Python_EXECUTABLE} ${STL_LIT_COMMAND} COMMAND_EXPAND_LISTS) -add_test(NAME stlasan COMMAND ${Python_EXECUTABLE} ${STLASAN_LIT_COMMAND} COMMAND_EXPAND_LISTS) -set_tests_properties(stl stlasan PROPERTIES RUN_SERIAL ON) + +add_custom_target(STL-CI COMMAND ${STL_LIT_COMMAND} -Dnotags=ASAN ${STL_LIT_TEST_DIRS} USES_TERMINAL) +add_custom_target(STL-ASan-CI COMMAND ${STL_LIT_COMMAND} -Dtags=ASAN ${STL_LIT_TEST_DIRS} USES_TERMINAL) diff --git a/tests/utils/stl/test/params.py b/tests/utils/stl/test/params.py index d032839beaf..c3dd091d5d5 100644 --- a/tests/utils/stl/test/params.py +++ b/tests/utils/stl/test/params.py @@ -49,8 +49,10 @@ def beNice(prio: str) -> list[ConfigAction]: } psutil.Process().nice(priority_map[prio]) except ImportError: - import sys - print(f'NOTE: Module "psutil" is not installed, so the priority setting "{prio}" has no effect.', file=sys.stderr) + if not hasattr(beNice, 'suppress'): + import sys + print(f'NOTE: Module "psutil" is not installed, so the priority setting "{prio}" has no effect.', file=sys.stderr) + beNice.suppress = True return []