From 37faeda17c015c64125801358df5323ad935d943 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 09:11:30 +0000 Subject: [PATCH 1/5] Make the test runners quieter for non-interactive consumers Adds two opt-in environment variables to the runner scripts, both unset by default so existing output is unchanged: * NO_COLOR (https://no-color.org/) stops the runners from forcing ANSI color codes on. parallel-lint and PHPUnit forced them unconditionally, which meant escape sequences in every captured log. * WP_CLI_TEST_QUIET switches the reporters to their most compact form: PHP_CodeSniffer to one line per violation with no progress ticker, PHPStan to one line per error with no progress bar and no result table, and Behat to omitting step definition snippets. Also documents narrowing a Behat run to a single scenario, --stop-on-failure and composer behat-rerun. Refs https://github.com/wp-cli/wp-cli/issues/6161 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VcGmbu6CYzGGhP3jXuWDkJ --- .readme-partials/USING.md | 33 +++++++++++++++++++++++++++++++++ README.md | 33 +++++++++++++++++++++++++++++++++ bin/run-behat-tests | 13 +++++++++++++ bin/run-linter-tests | 10 +++++++++- bin/run-php-unit-tests | 14 ++++++++++++-- bin/run-phpcs-tests | 12 +++++++++++- bin/run-phpstan-tests | 19 +++++++++++++++++-- 7 files changed, 128 insertions(+), 6 deletions(-) diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md index 3b2528f76..32d94e3b2 100644 --- a/.readme-partials/USING.md +++ b/.readme-partials/USING.md @@ -104,6 +104,39 @@ composer behat -- features/cli-info.feature Prepending with the double dash is needed because the arguments would otherwise be sent to Composer itself, not the tool that Composer executes. +The same mechanism works for narrowing a run down further, or for bailing out early: +```bash +# A single scenario, identified by the line it starts on. +composer behat -- features/cli-info.feature:12 + +# Every scenario carrying a given tag. +composer behat -- --tags=@require-wp-5.0 + +# Stop at the first failing scenario instead of running the whole suite. +composer behat -- --stop-on-failure + +# Re-run only the scenarios that failed the last time. +composer behat-rerun +``` + +### Controlling the amount of output + +Two environment variables make the test tools less chatty. Both are unset by default, which leaves the output exactly as it has always been. + + - `NO_COLOR` (the [no-color.org](https://no-color.org/) convention) stops the runners from forcing ANSI color codes on, and leaves the decision to each tool's own terminal detection. Set this when capturing output to a file or a pipe, where the escape sequences are noise. + - `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table, and Behat stops printing step definition snippets for undefined steps. + +```bash +NO_COLOR=1 WP_CLI_TEST_QUIET=1 composer phpstan +``` + +This is worth setting permanently in environments that read the output back rather than display it, such as an AI coding agent's shell: + +```bash +export NO_COLOR=1 +export WP_CLI_TEST_QUIET=1 +``` + ### Controlling the test environment #### WordPress Version diff --git a/README.md b/README.md index 6eeff87ab..6a9b6171f 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,39 @@ composer behat -- features/cli-info.feature Prepending with the double dash is needed because the arguments would otherwise be sent to Composer itself, not the tool that Composer executes. +The same mechanism works for narrowing a run down further, or for bailing out early: +```bash +# A single scenario, identified by the line it starts on. +composer behat -- features/cli-info.feature:12 + +# Every scenario carrying a given tag. +composer behat -- --tags=@require-wp-5.0 + +# Stop at the first failing scenario instead of running the whole suite. +composer behat -- --stop-on-failure + +# Re-run only the scenarios that failed the last time. +composer behat-rerun +``` + +### Controlling the amount of output + +Two environment variables make the test tools less chatty. Both are unset by default, which leaves the output exactly as it has always been. + + - `NO_COLOR` (the [no-color.org](https://no-color.org/) convention) stops the runners from forcing ANSI color codes on, and leaves the decision to each tool's own terminal detection. Set this when capturing output to a file or a pipe, where the escape sequences are noise. + - `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table, and Behat stops printing step definition snippets for undefined steps. + +```bash +NO_COLOR=1 WP_CLI_TEST_QUIET=1 composer phpstan +``` + +This is worth setting permanently in environments that read the output back rather than display it, such as an AI coding agent's shell: + +```bash +export NO_COLOR=1 +export WP_CLI_TEST_QUIET=1 +``` + ### Controlling the test environment #### WordPress Version diff --git a/bin/run-behat-tests b/bin/run-behat-tests index b276c7cbb..6d959bafa 100755 --- a/bin/run-behat-tests +++ b/bin/run-behat-tests @@ -141,6 +141,19 @@ if [[ "${WP_CLI_TEST_COVERAGE}" == "true" ]] && vendor/bin/behat --help 2>/dev/n BEHAT_EXTRA_ARGS+=('--xdebug') fi +# Honor the NO_COLOR convention (https://no-color.org/). +if [ -n "${NO_COLOR}" ]; then + BEHAT_EXTRA_ARGS+=('--no-colors') +fi + +# Drop the step definition snippets that Behat prints for undefined steps. They +# are long, and they are only actionable when you are writing new step +# definitions in this package. Useful for CI logs and for AI coding agents, +# which pay for every token of the report they read back. +if [ -n "${WP_CLI_TEST_QUIET}" ]; then + BEHAT_EXTRA_ARGS+=('--no-snippets') +fi + # Run the functional tests. FORMAT_ARGS=(--format progress) for arg in "$@"; do diff --git a/bin/run-linter-tests b/bin/run-linter-tests index add8ec336..d2c374ca7 100755 --- a/bin/run-linter-tests +++ b/bin/run-linter-tests @@ -1,3 +1,11 @@ #!/bin/sh -vendor/bin/parallel-lint -j 10 --colors --exclude vendor . "$@" +# Honor the NO_COLOR convention (https://no-color.org/) by leaving the color +# decision to the tool's own TTY detection instead of forcing it. +COLOR_ARGS="--colors" +if [ -n "${NO_COLOR}" ]; then + COLOR_ARGS="" +fi + +# shellcheck disable=SC2086 # Intentional word splitting of the optional arguments. +vendor/bin/parallel-lint -j 10 $COLOR_ARGS --exclude vendor . "$@" diff --git a/bin/run-php-unit-tests b/bin/run-php-unit-tests index 6182d0746..3d89befab 100755 --- a/bin/run-php-unit-tests +++ b/bin/run-php-unit-tests @@ -10,9 +10,19 @@ then EXTRA_ARGS="--display-warnings --fail-on-warning --display-notices --fail-on-notice --display-deprecations --fail-on-deprecation" fi + # Honor the NO_COLOR convention (https://no-color.org/). This has to be an + # explicit "never" rather than an omitted flag, because a project's + # phpunit.xml may well turn colors on by itself. + COLOR_ARGS="--color=always" + if [ -n "${NO_COLOR}" ]; then + COLOR_ARGS="--color=never" + fi + if [ -f "./vendor/wp-cli/wp-cli-tests/tests/bootstrap.php" ]; then - vendor/bin/phpunit --color=always "$@" $EXTRA_ARGS --bootstrap ./vendor/wp-cli/wp-cli-tests/tests/bootstrap.php + # shellcheck disable=SC2086 # Intentional word splitting of the optional arguments. + vendor/bin/phpunit $COLOR_ARGS "$@" $EXTRA_ARGS --bootstrap ./vendor/wp-cli/wp-cli-tests/tests/bootstrap.php else - vendor/bin/phpunit --color=always "$@" $EXTRA_ARGS + # shellcheck disable=SC2086 # Intentional word splitting of the optional arguments. + vendor/bin/phpunit $COLOR_ARGS "$@" $EXTRA_ARGS fi fi diff --git a/bin/run-phpcs-tests b/bin/run-phpcs-tests index 82d96b4e5..cff48a9bf 100755 --- a/bin/run-phpcs-tests +++ b/bin/run-phpcs-tests @@ -3,5 +3,15 @@ # Run the code style check only if a configuration file exists. if [ -f ".phpcs.xml" ] || [ -f "phpcs.xml" ] || [ -f ".phpcs.xml.dist" ] || [ -f "phpcs.xml.dist" ] then - vendor/bin/phpcs "$@" + EXTRA_ARGS="" + + # Compact, one-line-per-violation output without the progress ticker. + # Useful for CI logs and for AI coding agents, which pay for every token of + # the report they read back. + if [ -n "${WP_CLI_TEST_QUIET}" ]; then + EXTRA_ARGS="-q --report=emacs" + fi + + # shellcheck disable=SC2086 # Intentional word splitting of the optional arguments. + vendor/bin/phpcs $EXTRA_ARGS "$@" fi diff --git a/bin/run-phpstan-tests b/bin/run-phpstan-tests index 21f276900..235242ae8 100755 --- a/bin/run-phpstan-tests +++ b/bin/run-phpstan-tests @@ -1,7 +1,22 @@ #!/bin/sh -# Run the code style check only if a configuration file exists. +# Run the static analysis only if a configuration file exists. if [ -f "phpstan.dist.neon" ] || [ -f "phpstan.neon.dist" ] || [ -f "phpstan.neon" ] then - vendor/bin/phpstan --memory-limit=2048M analyse "$@" + EXTRA_ARGS="" + + # Honor the NO_COLOR convention (https://no-color.org/). + if [ -n "${NO_COLOR}" ]; then + EXTRA_ARGS="--no-ansi" + fi + + # Replace the redrawing progress bar and the box-drawing result table with + # plain `file:line:message` lines. Useful for CI logs and for AI coding + # agents, which pay for every token of the report they read back. + if [ -n "${WP_CLI_TEST_QUIET}" ]; then + EXTRA_ARGS="${EXTRA_ARGS} --no-progress --error-format=raw" + fi + + # shellcheck disable=SC2086 # Intentional word splitting of the optional arguments. + vendor/bin/phpstan --memory-limit=2048M analyse $EXTRA_ARGS "$@" fi From f38e2e6226115aadba2abf8d54b1ded5dd2b2a89 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 09:11:30 +0000 Subject: [PATCH 2/5] Cache the WP_VERSION lookups performed by the Behat runner Every `composer behat` invocation resolved WP_VERSION over the network: one request to api.wordpress.org for `latest`, and a second one to the wp-versions artifact when the version has no patch number. That cost applies equally to a full suite run and to re-running one scenario for the fifth time while iterating on a fix. The answers now go into a cache in the system temp directory with a configurable lifetime, defaulting to a day. Two side effects worth noting: * A run without connectivity falls back to the last known answer rather than continuing with an empty WP_VERSION, which silently disabled the filtering of version-specific tags. * When there is nothing to fall back to, that case is now reported instead of being silent. Refs https://github.com/wp-cli/wp-cli/issues/6161 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VcGmbu6CYzGGhP3jXuWDkJ --- .readme-partials/USING.md | 6 ++++ README.md | 6 ++++ bin/run-behat-tests | 71 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md index 32d94e3b2..dfa697125 100644 --- a/.readme-partials/USING.md +++ b/.readme-partials/USING.md @@ -152,6 +152,12 @@ Here's how to run your tests against the latest trunk version of WordPress: WP_VERSION=trunk composer behat ``` +Resolving `latest`, or a `X.Y` version without a patch number, needs a network +request. The answer is cached in the system temp directory for a day, so that +repeated runs do not repeat the lookup, and so that a run without connectivity +falls back to the last known answer. `WP_CLI_TEST_WP_VERSION_CACHE_TTL` sets the +lifetime of that cache in seconds; `0` looks the version up every time. + #### WordPress Archive Instead of downloading WordPress from WordPress.org, you can run the tests against an arbitrary diff --git a/README.md b/README.md index 6a9b6171f..022bf2f7d 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,12 @@ Here's how to run your tests against the latest trunk version of WordPress: WP_VERSION=trunk composer behat ``` +Resolving `latest`, or a `X.Y` version without a patch number, needs a network +request. The answer is cached in the system temp directory for a day, so that +repeated runs do not repeat the lookup, and so that a run without connectivity +falls back to the last known answer. `WP_CLI_TEST_WP_VERSION_CACHE_TTL` sets the +lifetime of that cache in seconds; `0` looks the version up every time. + #### WordPress Archive Instead of downloading WordPress from WordPress.org, you can run the tests against an arbitrary diff --git a/bin/run-behat-tests b/bin/run-behat-tests index 6d959bafa..068a283bd 100755 --- a/bin/run-behat-tests +++ b/bin/run-behat-tests @@ -97,9 +97,64 @@ if [ -n "${WP_CLI_TEST_CORE_ZIP-}" ] && [ -z "${WP_VERSION-}" ]; then export WP_VERSION=trunk fi +# Resolving WP_VERSION costs up to two network round trips on every invocation. +# Cache the answers, so that re-running a single scenario while iterating does +# not repeat them, and so that a run without connectivity can fall back to the +# last known answer instead of ending up with no version at all. +# +# Set WP_CLI_TEST_WP_VERSION_CACHE_TTL to 0 to always refetch. +WP_VERSION_CACHE_DIR="${TMPDIR:-/tmp}/wp-cli-test-wp-version-cache" +WP_VERSION_CACHE_TTL="${WP_CLI_TEST_WP_VERSION_CACHE_TTL:-86400}" + +# Print a cache entry if it exists and is younger than the given number of +# seconds. A negative TTL accepts an entry of any age. +read_version_cache() { + local cache_file="${WP_VERSION_CACHE_DIR}/$1" + local ttl="$2" + local age + + [ -s "${cache_file}" ] || return 1 + + if [ "${ttl}" -ge 0 ]; then + # PHP rather than `find -newermt`, which is not portable across + # GNU and BSD userlands. The Behat runner needs PHP anyway. + age=$(php -r 'echo time() - filemtime( $argv[1] );' "${cache_file}" 2>/dev/null) + case ${age} in + ''|*[!0-9]*) return 1;; + esac + [ "${age}" -lt "${ttl}" ] || return 1 + fi + + cat "${cache_file}" +} + +write_version_cache() { + mkdir -p "${WP_VERSION_CACHE_DIR}" 2>/dev/null || return 0 + printf '%s' "$2" > "${WP_VERSION_CACHE_DIR}/$1" 2>/dev/null || true +} + # Turn WP_VERSION into an actual number to make sure our tags work correctly. if [ "${WP_VERSION-latest}" = "latest" ]; then - export WP_VERSION=$(curl -s https://api.wordpress.org/core/version-check/1.7/ | jq -r ".offers[0].current") + LATEST_WP_VERSION=$(read_version_cache latest "${WP_VERSION_CACHE_TTL}") + + if [ -z "${LATEST_WP_VERSION}" ]; then + LATEST_WP_VERSION=$(curl -s https://api.wordpress.org/core/version-check/1.7/ | jq -r ".offers[0].current // empty") + + if [ -n "${LATEST_WP_VERSION}" ]; then + write_version_cache latest "${LATEST_WP_VERSION}" + else + # Prefer a stale answer over no answer. + LATEST_WP_VERSION=$(read_version_cache latest -1) + + if [ -n "${LATEST_WP_VERSION}" ]; then + echo "Warning: Could not reach api.wordpress.org, falling back to the cached latest WordPress version ${LATEST_WP_VERSION}." + else + echo "Warning: Could not determine the latest WordPress version. Version-specific tags will not be filtered." + fi + fi + fi + + export WP_VERSION="${LATEST_WP_VERSION}" fi # Normalize WP_VERSION=X.Y.0 to X.Y (WordPress uses X.Y for the initial release, not X.Y.0). @@ -107,7 +162,19 @@ fi if [[ "${WP_VERSION}" =~ ^([0-9]+\.[0-9]+)\.0$ ]]; then export WP_VERSION="${BASH_REMATCH[1]}" elif [[ "${WP_VERSION}" =~ ^[0-9]+\.[0-9]+$ ]]; then - WP_VERSIONS_JSON=$(curl -s https://raw.githubusercontent.com/wp-cli/wp-cli-tests/artifacts/wp-versions.json) + WP_VERSIONS_JSON=$(read_version_cache wp-versions.json "${WP_VERSION_CACHE_TTL}") + + if [ -z "${WP_VERSIONS_JSON}" ]; then + WP_VERSIONS_JSON=$(curl -s https://raw.githubusercontent.com/wp-cli/wp-cli-tests/artifacts/wp-versions.json) + + # Only cache a well-formed response; a proxy error page is not one. + if echo "${WP_VERSIONS_JSON}" | jq -e 'type == "object"' > /dev/null 2>&1; then + write_version_cache wp-versions.json "${WP_VERSIONS_JSON}" + else + WP_VERSIONS_JSON=$(read_version_cache wp-versions.json -1) + fi + fi + if [ -n "${WP_VERSIONS_JSON}" ]; then RESOLVED_VERSION=$(echo "${WP_VERSIONS_JSON}" | jq -r --arg prefix "${WP_VERSION}." 'keys | map(select(startswith($prefix))) | sort_by(split(".") | map(tonumber)) | last // empty') if [ -n "${RESOLVED_VERSION}" ]; then From 1d6afbc6399e99f88b35236f1bd33c8a8791df4f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 09:27:40 +0000 Subject: [PATCH 3/5] Add a Gherkin linter to the test suite The feature files are linted on every pull request, but the check lives entirely in the reusable CI workflow and its ruleset lives in wp-cli/.github, so contributors cannot run it locally at all. A green `composer test` is therefore not a green build, and the way to find out is to push. Moves the check to where the other suites are: `.gherkin-lintrc` ships with this package as the shared default, a project can override it by committing its own, and `composer lint-gherkin` runs it. CI can then call the same script rather than reimplementing the invocation. Uses gherkin-lint-plus, pinned, and overridable through WP_CLI_TEST_GHERKIN_LINT_VERSION. Being a Node package, it is invoked through npx and skips with a message where npx is absent, rather than failing a suite that is otherwise entirely PHP. The linter colors its report unconditionally and offers no plain output format, so NO_COLOR strips the escape sequences from its output. Refs https://github.com/wp-cli/wp-cli/issues/6161 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VcGmbu6CYzGGhP3jXuWDkJ --- .gherkin-lintrc | 32 +++++++++++++++ .readme-partials/USING.md | 15 +++++++ README.md | 15 +++++++ bin/run-gherkin-lint-tests | 82 ++++++++++++++++++++++++++++++++++++++ composer.json | 3 ++ 5 files changed, 147 insertions(+) create mode 100644 .gherkin-lintrc create mode 100755 bin/run-gherkin-lint-tests diff --git a/.gherkin-lintrc b/.gherkin-lintrc new file mode 100644 index 000000000..a0e5d631d --- /dev/null +++ b/.gherkin-lintrc @@ -0,0 +1,32 @@ +{ + "file-name": [ + "on", + { + "style": "kebab-case" + } + ], + "indentation": [ + "on", + { + "Feature": 0, + "Background": 2, + "Scenario": 2, + "Examples": 4, + "Step": 4, + "given": 4, + "example": 6, + "and": 4 + } + ], + "no-dupe-feature-names": "on", + "no-dupe-scenario-names": "off", + "no-empty-file": "on", + "no-files-without-scenarios": "on", + "no-multiple-empty-lines": "off", + "no-partially-commented-tag-lines": "on", + "no-trailing-spaces": "off", + "no-unnamed-features": "on", + "no-unnamed-scenarios": "on", + "no-scenario-outlines-without-examples": "on", + "use-and": "on" +} diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md index dfa697125..3a0d29ffa 100644 --- a/.readme-partials/USING.md +++ b/.readme-partials/USING.md @@ -90,6 +90,7 @@ You can use the following commands to control the tests: * `composer prepare-tests` - Set up the database that is needed for running the functional tests. This is only needed once. * `composer test` - Run all test suites. * `composer lint` - Run only the linting test suite. +* `composer lint-gherkin` - Run only the Gherkin linter over the feature files. * `composer phpcs` - Run only the code sniffer test suite. * `composer phpcbf` - Run only the code sniffer cleanup. * `composer phpunit` - Run only the unit test suite. @@ -119,6 +120,18 @@ composer behat -- --stop-on-failure composer behat-rerun ``` +### Linting the feature files + +`composer lint-gherkin` checks `features/` with +[gherkin-lint-plus](https://www.npmjs.com/package/gherkin-lint-plus), against the +`.gherkin-lintrc` ruleset shipped with this package. A project that needs +different rules can override it by committing its own `.gherkin-lintrc`. + +The linter is a Node package, so it is run through `npx` and needs Node.js 20 or +later. Where `npx` is not available the check reports that it is skipping, rather +than failing a suite that is otherwise entirely PHP. Pin a different release of +the linter with `WP_CLI_TEST_GHERKIN_LINT_VERSION`. + ### Controlling the amount of output Two environment variables make the test tools less chatty. Both are unset by default, which leaves the output exactly as it has always been. @@ -126,6 +139,8 @@ Two environment variables make the test tools less chatty. Both are unset by def - `NO_COLOR` (the [no-color.org](https://no-color.org/) convention) stops the runners from forcing ANSI color codes on, and leaves the decision to each tool's own terminal detection. Set this when capturing output to a file or a pipe, where the escape sequences are noise. - `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table, and Behat stops printing step definition snippets for undefined steps. +`NO_COLOR` also covers the Gherkin linter, which colors its report unconditionally and has no plain output format of its own. + ```bash NO_COLOR=1 WP_CLI_TEST_QUIET=1 composer phpstan ``` diff --git a/README.md b/README.md index 022bf2f7d..199c89200 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ You can use the following commands to control the tests: * `composer prepare-tests` - Set up the database that is needed for running the functional tests. This is only needed once. * `composer test` - Run all test suites. * `composer lint` - Run only the linting test suite. +* `composer lint-gherkin` - Run only the Gherkin linter over the feature files. * `composer phpcs` - Run only the code sniffer test suite. * `composer phpcbf` - Run only the code sniffer cleanup. * `composer phpunit` - Run only the unit test suite. @@ -130,6 +131,18 @@ composer behat -- --stop-on-failure composer behat-rerun ``` +### Linting the feature files + +`composer lint-gherkin` checks `features/` with +[gherkin-lint-plus](https://www.npmjs.com/package/gherkin-lint-plus), against the +`.gherkin-lintrc` ruleset shipped with this package. A project that needs +different rules can override it by committing its own `.gherkin-lintrc`. + +The linter is a Node package, so it is run through `npx` and needs Node.js 20 or +later. Where `npx` is not available the check reports that it is skipping, rather +than failing a suite that is otherwise entirely PHP. Pin a different release of +the linter with `WP_CLI_TEST_GHERKIN_LINT_VERSION`. + ### Controlling the amount of output Two environment variables make the test tools less chatty. Both are unset by default, which leaves the output exactly as it has always been. @@ -137,6 +150,8 @@ Two environment variables make the test tools less chatty. Both are unset by def - `NO_COLOR` (the [no-color.org](https://no-color.org/) convention) stops the runners from forcing ANSI color codes on, and leaves the decision to each tool's own terminal detection. Set this when capturing output to a file or a pipe, where the escape sequences are noise. - `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table, and Behat stops printing step definition snippets for undefined steps. +`NO_COLOR` also covers the Gherkin linter, which colors its report unconditionally and has no plain output format of its own. + ```bash NO_COLOR=1 WP_CLI_TEST_QUIET=1 composer phpstan ``` diff --git a/bin/run-gherkin-lint-tests b/bin/run-gherkin-lint-tests new file mode 100755 index 000000000..78fc82bd7 --- /dev/null +++ b/bin/run-gherkin-lint-tests @@ -0,0 +1,82 @@ +#!/bin/sh + +# Run the Gherkin linter only if there are feature files to lint. +if [ ! -d "features" ] +then + exit 0; +fi + +# The linter is a Node package, which a PHP project cannot assume is present. +# Skip rather than fail, the same way the Behat runner skips a package without a +# behat.yml. The check still runs in CI, where Node is always available. +if ! command -v npx > /dev/null 2>&1 +then + echo 'Did not detect the "npx" command, skipping the Gherkin linting.' + echo "It is part of Node.js 20 or later, see https://nodejs.org/ for installation instructions." + exit 0; +fi + +# To retrieve the WP-CLI tests package root folder, we start with this scripts +# location. +SOURCE="$0" + +# Resolve $SOURCE until the file is no longer a symlink. +while [ -h "$SOURCE" ]; do + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$( readlink "$SOURCE" )" + # If $SOURCE was a relative symlink, we need to resolve it relative to the + # path where the symlink file was located. + case $SOURCE in + /*) ;; + *) SOURCE="$DIR/$SOURCE";; + esac +done + +# Fetch the root folder of the WP-CLI tests package. +WP_CLI_TESTS_ROOT="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" + +# A project can override the shared ruleset by committing its own .gherkin-lintrc. +CONFIG_FILE=".gherkin-lintrc" +if [ ! -f "${CONFIG_FILE}" ] +then + CONFIG_FILE="${WP_CLI_TESTS_ROOT}/.gherkin-lintrc" +fi + +# Pinned so that a new release of the linter cannot turn a green build red +# without a commit in this repository. +GHERKIN_LINT_VERSION="${WP_CLI_TEST_GHERKIN_LINT_VERSION:-1.0.2}" + +# Without an explicit path the linter walks the whole working directory, vendor +# included. The feature files are the target. +if [ $# -eq 0 ] +then + set -- features +fi + +run_linter() { + npx --yes "gherkin-lint-plus@${GHERKIN_LINT_VERSION}" --config "${CONFIG_FILE}" "$@" +} + +if [ -n "${NO_COLOR}" ] +then + # The linter colors its report unconditionally: it honors neither NO_COLOR + # nor the absence of a terminal, and "stylish" is its only output format. + # It writes the report to STDERR, so strip the escape sequences from that + # stream while leaving STDOUT and the exit code alone. + STDERR_FILE=$( mktemp ) + + run_linter "$@" 2> "${STDERR_FILE}" + STATUS=$? + + if [ -s "${STDERR_FILE}" ] + then + ESC=$( printf '\033' ) + sed "s/${ESC}\[[0-9;]*m//g" "${STDERR_FILE}" >&2 + fi + + rm -f "${STDERR_FILE}" + + exit ${STATUS}; +fi + +run_linter "$@" diff --git a/composer.json b/composer.json index c32507878..c0d596827 100644 --- a/composer.json +++ b/composer.json @@ -79,6 +79,7 @@ "bin/install-package-tests", "bin/rerun-behat-tests", "bin/run-behat-tests", + "bin/run-gherkin-lint-tests", "bin/run-linter-tests", "bin/run-php-unit-tests", "bin/run-phpcs-tests", @@ -89,6 +90,7 @@ "behat": "run-behat-tests", "behat-rerun": "rerun-behat-tests", "lint": "run-linter-tests", + "lint-gherkin": "run-gherkin-lint-tests", "phpcs": "run-phpcs-tests", "phpcbf": "run-phpcbf-cleanup", "phpstan": "run-phpstan-tests", @@ -96,6 +98,7 @@ "prepare-tests": "install-package-tests", "test": [ "@lint", + "@lint-gherkin", "@phpcs", "@phpstan", "@phpunit", From 203f0caaf6bcf477ad49a78ee857f21fa4dc876e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 09:44:01 +0000 Subject: [PATCH 4/5] Move the Gherkin linter pin into package.json Drops the WP_CLI_TEST_GHERKIN_LINT_VERSION override, which was configuration nobody asked for, and puts the pinned version somewhere a dependency bot can see it. A version string inside a shell script is invisible to Dependabot; a devDependency in package.json is not. The package.json exists only to hold that pin: it is private, has no scripts, and nothing runs `npm install` against it. The runner reads the version out of it and fails loudly if it is missing, rather than quietly falling through to whatever the latest release happens to be. Note that picking these updates up needs an npm entry in the dependabot.yml that wp-cli/.github syncs out. Refs https://github.com/wp-cli/wp-cli/issues/6161 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VcGmbu6CYzGGhP3jXuWDkJ --- .readme-partials/USING.md | 4 ++-- README.md | 4 ++-- bin/run-gherkin-lint-tests | 14 +++++++++++--- package.json | 9 +++++++++ 4 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 package.json diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md index 3a0d29ffa..91f6f67b6 100644 --- a/.readme-partials/USING.md +++ b/.readme-partials/USING.md @@ -129,8 +129,8 @@ different rules can override it by committing its own `.gherkin-lintrc`. The linter is a Node package, so it is run through `npx` and needs Node.js 20 or later. Where `npx` is not available the check reports that it is skipping, rather -than failing a suite that is otherwise entirely PHP. Pin a different release of -the linter with `WP_CLI_TEST_GHERKIN_LINT_VERSION`. +than failing a suite that is otherwise entirely PHP. Its version is pinned in +this package's `package.json`, which exists only to hold that pin. ### Controlling the amount of output diff --git a/README.md b/README.md index 199c89200..d871c2163 100644 --- a/README.md +++ b/README.md @@ -140,8 +140,8 @@ different rules can override it by committing its own `.gherkin-lintrc`. The linter is a Node package, so it is run through `npx` and needs Node.js 20 or later. Where `npx` is not available the check reports that it is skipping, rather -than failing a suite that is otherwise entirely PHP. Pin a different release of -the linter with `WP_CLI_TEST_GHERKIN_LINT_VERSION`. +than failing a suite that is otherwise entirely PHP. Its version is pinned in +this package's `package.json`, which exists only to hold that pin. ### Controlling the amount of output diff --git a/bin/run-gherkin-lint-tests b/bin/run-gherkin-lint-tests index 78fc82bd7..794eb5e38 100755 --- a/bin/run-gherkin-lint-tests +++ b/bin/run-gherkin-lint-tests @@ -42,9 +42,17 @@ then CONFIG_FILE="${WP_CLI_TESTS_ROOT}/.gherkin-lintrc" fi -# Pinned so that a new release of the linter cannot turn a green build red -# without a commit in this repository. -GHERKIN_LINT_VERSION="${WP_CLI_TEST_GHERKIN_LINT_VERSION:-1.0.2}" +# The linter is pinned so that one of its releases cannot turn a green build red +# without a commit here. The pin lives in package.json rather than in this file, +# because a version string inside a shell script is invisible to Dependabot. +PACKAGE_JSON="${WP_CLI_TESTS_ROOT}/package.json" +GHERKIN_LINT_VERSION=$( php -r '$package = json_decode( (string) file_get_contents( $argv[1] ), true ); echo isset( $package["devDependencies"]["gherkin-lint-plus"] ) ? $package["devDependencies"]["gherkin-lint-plus"] : "";' "${PACKAGE_JSON}" 2> /dev/null ) + +if [ -z "${GHERKIN_LINT_VERSION}" ] +then + echo "Could not read the pinned gherkin-lint-plus version from ${PACKAGE_JSON}." + exit 1; +fi # Without an explicit path the linter walks the whole working directory, vendor # included. The feature files are the target. diff --git a/package.json b/package.json new file mode 100644 index 000000000..c069a68bc --- /dev/null +++ b/package.json @@ -0,0 +1,9 @@ +{ + "name": "wp-cli-tests", + "description": "Pins the Node tooling used by the WP-CLI testing framework. Not published to npm.", + "private": true, + "license": "MIT", + "devDependencies": { + "gherkin-lint-plus": "1.0.2" + } +} From e844d3533a86e0a00763a74222cff08a9106ded0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 13:27:25 +0000 Subject: [PATCH 5/5] Address review feedback on the Behat runner Two points from review: * The wp-versions artifact already marks the current release with a "latest" status, so the separate request to api.wordpress.org was redundant. Both the "latest" and the X.Y resolution now come out of that one file, which means one cached artifact and at most one network request per run instead of two. * Behat's step definition snippets are not only printed when writing new step definitions; they are also how a typo in an existing step surfaces. That makes them a diagnostic rather than noise, and they only appear when something is already wrong, so suppressing them under WP_CLI_TEST_QUIET saved nothing in the passing case and cost information in the failing one. Dropped, which leaves WP_CLI_TEST_QUIET with no effect on Behat. Also adds lint-gherkin to the setup instructions, which listed the scripts a consuming package should wire up but not the new one. Refs https://github.com/wp-cli/wp-cli/issues/6161 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VcGmbu6CYzGGhP3jXuWDkJ --- .readme-partials/USING.md | 15 +++-- README.md | 15 +++-- bin/run-behat-tests | 123 ++++++++++++++++++-------------------- 3 files changed, 76 insertions(+), 77 deletions(-) diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md index 91f6f67b6..bd384b401 100644 --- a/.readme-partials/USING.md +++ b/.readme-partials/USING.md @@ -11,12 +11,14 @@ To make use of the WP-CLI testing framework, you need to complete the following "behat": "run-behat-tests", "behat-rerun": "rerun-behat-tests", "lint": "run-linter-tests", + "lint-gherkin": "run-gherkin-lint-tests", "phpcs": "run-phpcs-tests", "phpcbf": "run-phpcbf-cleanup", "phpunit": "run-php-unit-tests", "prepare-tests": "install-package-tests", "test": [ "@lint", + "@lint-gherkin", "@phpcs", "@phpunit", "@behat" @@ -137,7 +139,7 @@ this package's `package.json`, which exists only to hold that pin. Two environment variables make the test tools less chatty. Both are unset by default, which leaves the output exactly as it has always been. - `NO_COLOR` (the [no-color.org](https://no-color.org/) convention) stops the runners from forcing ANSI color codes on, and leaves the decision to each tool's own terminal detection. Set this when capturing output to a file or a pipe, where the escape sequences are noise. - - `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table, and Behat stops printing step definition snippets for undefined steps. + - `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table. Behat's own output is already minimal, so it is unaffected. `NO_COLOR` also covers the Gherkin linter, which colors its report unconditionally and has no plain output format of its own. @@ -167,11 +169,12 @@ Here's how to run your tests against the latest trunk version of WordPress: WP_VERSION=trunk composer behat ``` -Resolving `latest`, or a `X.Y` version without a patch number, needs a network -request. The answer is cached in the system temp directory for a day, so that -repeated runs do not repeat the lookup, and so that a run without connectivity -falls back to the last known answer. `WP_CLI_TEST_WP_VERSION_CACHE_TTL` sets the -lifetime of that cache in seconds; `0` looks the version up every time. +Resolving `latest`, or a `X.Y` version without a patch number, needs the +WordPress versions data, which is fetched once and cached in the system temp +directory for a day. Repeated runs do not repeat the request, and a run without +connectivity falls back to the last known copy. +`WP_CLI_TEST_WP_VERSION_CACHE_TTL` sets the lifetime of that cache in seconds; +`0` fetches it every time. #### WordPress Archive diff --git a/README.md b/README.md index d871c2163..be54b714d 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,14 @@ To make use of the WP-CLI testing framework, you need to complete the following "behat": "run-behat-tests", "behat-rerun": "rerun-behat-tests", "lint": "run-linter-tests", + "lint-gherkin": "run-gherkin-lint-tests", "phpcs": "run-phpcs-tests", "phpcbf": "run-phpcbf-cleanup", "phpunit": "run-php-unit-tests", "prepare-tests": "install-package-tests", "test": [ "@lint", + "@lint-gherkin", "@phpcs", "@phpunit", "@behat" @@ -148,7 +150,7 @@ this package's `package.json`, which exists only to hold that pin. Two environment variables make the test tools less chatty. Both are unset by default, which leaves the output exactly as it has always been. - `NO_COLOR` (the [no-color.org](https://no-color.org/) convention) stops the runners from forcing ANSI color codes on, and leaves the decision to each tool's own terminal detection. Set this when capturing output to a file or a pipe, where the escape sequences are noise. - - `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table, and Behat stops printing step definition snippets for undefined steps. + - `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table. Behat's own output is already minimal, so it is unaffected. `NO_COLOR` also covers the Gherkin linter, which colors its report unconditionally and has no plain output format of its own. @@ -178,11 +180,12 @@ Here's how to run your tests against the latest trunk version of WordPress: WP_VERSION=trunk composer behat ``` -Resolving `latest`, or a `X.Y` version without a patch number, needs a network -request. The answer is cached in the system temp directory for a day, so that -repeated runs do not repeat the lookup, and so that a run without connectivity -falls back to the last known answer. `WP_CLI_TEST_WP_VERSION_CACHE_TTL` sets the -lifetime of that cache in seconds; `0` looks the version up every time. +Resolving `latest`, or a `X.Y` version without a patch number, needs the +WordPress versions data, which is fetched once and cached in the system temp +directory for a day. Repeated runs do not repeat the request, and a run without +connectivity falls back to the last known copy. +`WP_CLI_TEST_WP_VERSION_CACHE_TTL` sets the lifetime of that cache in seconds; +`0` fetches it every time. #### WordPress Archive diff --git a/bin/run-behat-tests b/bin/run-behat-tests index 068a283bd..ccf133106 100755 --- a/bin/run-behat-tests +++ b/bin/run-behat-tests @@ -97,89 +97,90 @@ if [ -n "${WP_CLI_TEST_CORE_ZIP-}" ] && [ -z "${WP_VERSION-}" ]; then export WP_VERSION=trunk fi -# Resolving WP_VERSION costs up to two network round trips on every invocation. -# Cache the answers, so that re-running a single scenario while iterating does -# not repeat them, and so that a run without connectivity can fall back to the -# last known answer instead of ending up with no version at all. +# Everything WP_VERSION resolution needs is in one file: the wp-versions artifact +# maps every WordPress release to its status, with the current one marked +# "latest". Cache it, so that re-running a single scenario while iterating does +# not refetch it every time, and so that a run without connectivity can fall back +# to the last known answer instead of ending up with no version at all. # # Set WP_CLI_TEST_WP_VERSION_CACHE_TTL to 0 to always refetch. -WP_VERSION_CACHE_DIR="${TMPDIR:-/tmp}/wp-cli-test-wp-version-cache" -WP_VERSION_CACHE_TTL="${WP_CLI_TEST_WP_VERSION_CACHE_TTL:-86400}" - -# Print a cache entry if it exists and is younger than the given number of -# seconds. A negative TTL accepts an entry of any age. -read_version_cache() { - local cache_file="${WP_VERSION_CACHE_DIR}/$1" - local ttl="$2" +WP_VERSIONS_URL="https://raw.githubusercontent.com/wp-cli/wp-cli-tests/artifacts/wp-versions.json" +WP_VERSIONS_CACHE_FILE="${TMPDIR:-/tmp}/wp-cli-test-wp-version-cache/wp-versions.json" +WP_VERSIONS_CACHE_TTL="${WP_CLI_TEST_WP_VERSION_CACHE_TTL:-86400}" + +# Print the cached versions file if it is younger than the given number of +# seconds. A negative TTL accepts it at any age. +read_versions_cache() { + local ttl="$1" local age - [ -s "${cache_file}" ] || return 1 + [ -s "${WP_VERSIONS_CACHE_FILE}" ] || return 1 if [ "${ttl}" -ge 0 ]; then # PHP rather than `find -newermt`, which is not portable across # GNU and BSD userlands. The Behat runner needs PHP anyway. - age=$(php -r 'echo time() - filemtime( $argv[1] );' "${cache_file}" 2>/dev/null) + age=$(php -r 'echo time() - filemtime( $argv[1] );' "${WP_VERSIONS_CACHE_FILE}" 2>/dev/null) case ${age} in ''|*[!0-9]*) return 1;; esac [ "${age}" -lt "${ttl}" ] || return 1 fi - cat "${cache_file}" + cat "${WP_VERSIONS_CACHE_FILE}" } -write_version_cache() { - mkdir -p "${WP_VERSION_CACHE_DIR}" 2>/dev/null || return 0 - printf '%s' "$2" > "${WP_VERSION_CACHE_DIR}/$1" 2>/dev/null || true -} - -# Turn WP_VERSION into an actual number to make sure our tags work correctly. -if [ "${WP_VERSION-latest}" = "latest" ]; then - LATEST_WP_VERSION=$(read_version_cache latest "${WP_VERSION_CACHE_TTL}") +# Print the WordPress versions data, from the cache where possible. Warnings go +# to STDERR so that they cannot end up inside the returned JSON. +get_wp_versions() { + local json - if [ -z "${LATEST_WP_VERSION}" ]; then - LATEST_WP_VERSION=$(curl -s https://api.wordpress.org/core/version-check/1.7/ | jq -r ".offers[0].current // empty") + json=$( read_versions_cache "${WP_VERSIONS_CACHE_TTL}" ) + if [ -n "${json}" ]; then + printf '%s' "${json}" + return 0 + fi - if [ -n "${LATEST_WP_VERSION}" ]; then - write_version_cache latest "${LATEST_WP_VERSION}" - else - # Prefer a stale answer over no answer. - LATEST_WP_VERSION=$(read_version_cache latest -1) + json=$( curl -s "${WP_VERSIONS_URL}" ) - if [ -n "${LATEST_WP_VERSION}" ]; then - echo "Warning: Could not reach api.wordpress.org, falling back to the cached latest WordPress version ${LATEST_WP_VERSION}." - else - echo "Warning: Could not determine the latest WordPress version. Version-specific tags will not be filtered." - fi - fi + # Only cache a well-formed response; an error page is not one. + if echo "${json}" | jq -e 'type == "object" and length > 0' > /dev/null 2>&1; then + mkdir -p "$( dirname "${WP_VERSIONS_CACHE_FILE}" )" 2>/dev/null \ + && printf '%s' "${json}" > "${WP_VERSIONS_CACHE_FILE}" 2>/dev/null || true + printf '%s' "${json}" + return 0 fi - export WP_VERSION="${LATEST_WP_VERSION}" -fi + # Prefer a stale answer over no answer. + json=$( read_versions_cache -1 ) + if [ -n "${json}" ]; then + echo "Warning: Could not fetch the WordPress versions data, falling back to the cached copy." >&2 + printf '%s' "${json}" + return 0 + fi -# Normalize WP_VERSION=X.Y.0 to X.Y (WordPress uses X.Y for the initial release, not X.Y.0). -# If WP_VERSION=X.Y (major.minor only), resolve to the latest available patch release. -if [[ "${WP_VERSION}" =~ ^([0-9]+\.[0-9]+)\.0$ ]]; then - export WP_VERSION="${BASH_REMATCH[1]}" -elif [[ "${WP_VERSION}" =~ ^[0-9]+\.[0-9]+$ ]]; then - WP_VERSIONS_JSON=$(read_version_cache wp-versions.json "${WP_VERSION_CACHE_TTL}") + return 1 +} - if [ -z "${WP_VERSIONS_JSON}" ]; then - WP_VERSIONS_JSON=$(curl -s https://raw.githubusercontent.com/wp-cli/wp-cli-tests/artifacts/wp-versions.json) +# Turn WP_VERSION into an actual number to make sure our tags work correctly. +if [ "${WP_VERSION-latest}" = "latest" ]; then + WP_VERSION=$( get_wp_versions | jq -r 'to_entries | map( select( .value == "latest" ) ) | last | .key // empty' ) - # Only cache a well-formed response; a proxy error page is not one. - if echo "${WP_VERSIONS_JSON}" | jq -e 'type == "object"' > /dev/null 2>&1; then - write_version_cache wp-versions.json "${WP_VERSIONS_JSON}" - else - WP_VERSIONS_JSON=$(read_version_cache wp-versions.json -1) - fi + if [ -z "${WP_VERSION}" ]; then + echo "Warning: Could not determine the latest WordPress version. Version-specific tags will not be filtered." fi - if [ -n "${WP_VERSIONS_JSON}" ]; then - RESOLVED_VERSION=$(echo "${WP_VERSIONS_JSON}" | jq -r --arg prefix "${WP_VERSION}." 'keys | map(select(startswith($prefix))) | sort_by(split(".") | map(tonumber)) | last // empty') - if [ -n "${RESOLVED_VERSION}" ]; then - export WP_VERSION="${RESOLVED_VERSION}" - fi + export WP_VERSION +# Normalize WP_VERSION=X.Y.0 to X.Y (WordPress uses X.Y for the initial release, +# not X.Y.0). This asks for that specific release, so it must not fall through to +# the patch resolution below. +elif [[ "${WP_VERSION}" =~ ^([0-9]+\.[0-9]+)\.0$ ]]; then + export WP_VERSION="${BASH_REMATCH[1]}" +# If WP_VERSION=X.Y (major.minor only), resolve to the latest available patch release. +elif [[ "${WP_VERSION}" =~ ^[0-9]+\.[0-9]+$ ]]; then + RESOLVED_VERSION=$( get_wp_versions | jq -r --arg prefix "${WP_VERSION}." 'keys | map( select( startswith( $prefix ) ) ) | sort_by( split(".") | map( tonumber ) ) | last // empty' ) + + if [ -n "${RESOLVED_VERSION}" ]; then + export WP_VERSION="${RESOLVED_VERSION}" fi fi @@ -213,14 +214,6 @@ if [ -n "${NO_COLOR}" ]; then BEHAT_EXTRA_ARGS+=('--no-colors') fi -# Drop the step definition snippets that Behat prints for undefined steps. They -# are long, and they are only actionable when you are writing new step -# definitions in this package. Useful for CI logs and for AI coding agents, -# which pay for every token of the report they read back. -if [ -n "${WP_CLI_TEST_QUIET}" ]; then - BEHAT_EXTRA_ARGS+=('--no-snippets') -fi - # Run the functional tests. FORMAT_ARGS=(--format progress) for arg in "$@"; do