diff --git a/.gherkin-lintrc b/.gherkin-lintrc new file mode 100644 index 00000000..a0e5d631 --- /dev/null +++ b/.gherkin-lintrc @@ -0,0 +1,32 @@ +{ + "file-name": [ + "on", + { + "style": "kebab-case" + } + ], + "indentation": [ + "on", + { + "Feature": 0, + "Background": 2, + "Scenario": 2, + "Examples": 4, + "Step": 4, + "given": 4, + "example": 6, + "and": 4 + } + ], + "no-dupe-feature-names": "on", + "no-dupe-scenario-names": "off", + "no-empty-file": "on", + "no-files-without-scenarios": "on", + "no-multiple-empty-lines": "off", + "no-partially-commented-tag-lines": "on", + "no-trailing-spaces": "off", + "no-unnamed-features": "on", + "no-unnamed-scenarios": "on", + "no-scenario-outlines-without-examples": "on", + "use-and": "on" +} diff --git a/.readme-partials/USING.md b/.readme-partials/USING.md index 3b2528f7..bd384b40 100644 --- a/.readme-partials/USING.md +++ b/.readme-partials/USING.md @@ -11,12 +11,14 @@ To make use of the WP-CLI testing framework, you need to complete the following "behat": "run-behat-tests", "behat-rerun": "rerun-behat-tests", "lint": "run-linter-tests", + "lint-gherkin": "run-gherkin-lint-tests", "phpcs": "run-phpcs-tests", "phpcbf": "run-phpcbf-cleanup", "phpunit": "run-php-unit-tests", "prepare-tests": "install-package-tests", "test": [ "@lint", + "@lint-gherkin", "@phpcs", "@phpunit", "@behat" @@ -90,6 +92,7 @@ You can use the following commands to control the tests: * `composer prepare-tests` - Set up the database that is needed for running the functional tests. This is only needed once. * `composer test` - Run all test suites. * `composer lint` - Run only the linting test suite. +* `composer lint-gherkin` - Run only the Gherkin linter over the feature files. * `composer phpcs` - Run only the code sniffer test suite. * `composer phpcbf` - Run only the code sniffer cleanup. * `composer phpunit` - Run only the unit test suite. @@ -104,6 +107,53 @@ composer behat -- features/cli-info.feature Prepending with the double dash is needed because the arguments would otherwise be sent to Composer itself, not the tool that Composer executes. +The same mechanism works for narrowing a run down further, or for bailing out early: +```bash +# A single scenario, identified by the line it starts on. +composer behat -- features/cli-info.feature:12 + +# Every scenario carrying a given tag. +composer behat -- --tags=@require-wp-5.0 + +# Stop at the first failing scenario instead of running the whole suite. +composer behat -- --stop-on-failure + +# Re-run only the scenarios that failed the last time. +composer behat-rerun +``` + +### Linting the feature files + +`composer lint-gherkin` checks `features/` with +[gherkin-lint-plus](https://www.npmjs.com/package/gherkin-lint-plus), against the +`.gherkin-lintrc` ruleset shipped with this package. A project that needs +different rules can override it by committing its own `.gherkin-lintrc`. + +The linter is a Node package, so it is run through `npx` and needs Node.js 20 or +later. Where `npx` is not available the check reports that it is skipping, rather +than failing a suite that is otherwise entirely PHP. Its version is pinned in +this package's `package.json`, which exists only to hold that pin. + +### Controlling the amount of output + +Two environment variables make the test tools less chatty. Both are unset by default, which leaves the output exactly as it has always been. + + - `NO_COLOR` (the [no-color.org](https://no-color.org/) convention) stops the runners from forcing ANSI color codes on, and leaves the decision to each tool's own terminal detection. Set this when capturing output to a file or a pipe, where the escape sequences are noise. + - `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table. Behat's own output is already minimal, so it is unaffected. + +`NO_COLOR` also covers the Gherkin linter, which colors its report unconditionally and has no plain output format of its own. + +```bash +NO_COLOR=1 WP_CLI_TEST_QUIET=1 composer phpstan +``` + +This is worth setting permanently in environments that read the output back rather than display it, such as an AI coding agent's shell: + +```bash +export NO_COLOR=1 +export WP_CLI_TEST_QUIET=1 +``` + ### Controlling the test environment #### WordPress Version @@ -119,6 +169,13 @@ Here's how to run your tests against the latest trunk version of WordPress: WP_VERSION=trunk composer behat ``` +Resolving `latest`, or a `X.Y` version without a patch number, needs the +WordPress versions data, which is fetched once and cached in the system temp +directory for a day. Repeated runs do not repeat the request, and a run without +connectivity falls back to the last known copy. +`WP_CLI_TEST_WP_VERSION_CACHE_TTL` sets the lifetime of that cache in seconds; +`0` fetches it every time. + #### WordPress Archive Instead of downloading WordPress from WordPress.org, you can run the tests against an arbitrary diff --git a/README.md b/README.md index 6eeff87a..be54b714 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,14 @@ To make use of the WP-CLI testing framework, you need to complete the following "behat": "run-behat-tests", "behat-rerun": "rerun-behat-tests", "lint": "run-linter-tests", + "lint-gherkin": "run-gherkin-lint-tests", "phpcs": "run-phpcs-tests", "phpcbf": "run-phpcbf-cleanup", "phpunit": "run-php-unit-tests", "prepare-tests": "install-package-tests", "test": [ "@lint", + "@lint-gherkin", "@phpcs", "@phpunit", "@behat" @@ -101,6 +103,7 @@ You can use the following commands to control the tests: * `composer prepare-tests` - Set up the database that is needed for running the functional tests. This is only needed once. * `composer test` - Run all test suites. * `composer lint` - Run only the linting test suite. +* `composer lint-gherkin` - Run only the Gherkin linter over the feature files. * `composer phpcs` - Run only the code sniffer test suite. * `composer phpcbf` - Run only the code sniffer cleanup. * `composer phpunit` - Run only the unit test suite. @@ -115,6 +118,53 @@ composer behat -- features/cli-info.feature Prepending with the double dash is needed because the arguments would otherwise be sent to Composer itself, not the tool that Composer executes. +The same mechanism works for narrowing a run down further, or for bailing out early: +```bash +# A single scenario, identified by the line it starts on. +composer behat -- features/cli-info.feature:12 + +# Every scenario carrying a given tag. +composer behat -- --tags=@require-wp-5.0 + +# Stop at the first failing scenario instead of running the whole suite. +composer behat -- --stop-on-failure + +# Re-run only the scenarios that failed the last time. +composer behat-rerun +``` + +### Linting the feature files + +`composer lint-gherkin` checks `features/` with +[gherkin-lint-plus](https://www.npmjs.com/package/gherkin-lint-plus), against the +`.gherkin-lintrc` ruleset shipped with this package. A project that needs +different rules can override it by committing its own `.gherkin-lintrc`. + +The linter is a Node package, so it is run through `npx` and needs Node.js 20 or +later. Where `npx` is not available the check reports that it is skipping, rather +than failing a suite that is otherwise entirely PHP. Its version is pinned in +this package's `package.json`, which exists only to hold that pin. + +### Controlling the amount of output + +Two environment variables make the test tools less chatty. Both are unset by default, which leaves the output exactly as it has always been. + + - `NO_COLOR` (the [no-color.org](https://no-color.org/) convention) stops the runners from forcing ANSI color codes on, and leaves the decision to each tool's own terminal detection. Set this when capturing output to a file or a pipe, where the escape sequences are noise. + - `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table. Behat's own output is already minimal, so it is unaffected. + +`NO_COLOR` also covers the Gherkin linter, which colors its report unconditionally and has no plain output format of its own. + +```bash +NO_COLOR=1 WP_CLI_TEST_QUIET=1 composer phpstan +``` + +This is worth setting permanently in environments that read the output back rather than display it, such as an AI coding agent's shell: + +```bash +export NO_COLOR=1 +export WP_CLI_TEST_QUIET=1 +``` + ### Controlling the test environment #### WordPress Version @@ -130,6 +180,13 @@ Here's how to run your tests against the latest trunk version of WordPress: WP_VERSION=trunk composer behat ``` +Resolving `latest`, or a `X.Y` version without a patch number, needs the +WordPress versions data, which is fetched once and cached in the system temp +directory for a day. Repeated runs do not repeat the request, and a run without +connectivity falls back to the last known copy. +`WP_CLI_TEST_WP_VERSION_CACHE_TTL` sets the lifetime of that cache in seconds; +`0` fetches it every time. + #### WordPress Archive Instead of downloading WordPress from WordPress.org, you can run the tests against an arbitrary diff --git a/bin/run-behat-tests b/bin/run-behat-tests index b276c7cb..ccf13310 100755 --- a/bin/run-behat-tests +++ b/bin/run-behat-tests @@ -97,22 +97,90 @@ if [ -n "${WP_CLI_TEST_CORE_ZIP-}" ] && [ -z "${WP_VERSION-}" ]; then export WP_VERSION=trunk fi +# Everything WP_VERSION resolution needs is in one file: the wp-versions artifact +# maps every WordPress release to its status, with the current one marked +# "latest". Cache it, so that re-running a single scenario while iterating does +# not refetch it every time, and so that a run without connectivity can fall back +# to the last known answer instead of ending up with no version at all. +# +# Set WP_CLI_TEST_WP_VERSION_CACHE_TTL to 0 to always refetch. +WP_VERSIONS_URL="https://raw.githubusercontent.com/wp-cli/wp-cli-tests/artifacts/wp-versions.json" +WP_VERSIONS_CACHE_FILE="${TMPDIR:-/tmp}/wp-cli-test-wp-version-cache/wp-versions.json" +WP_VERSIONS_CACHE_TTL="${WP_CLI_TEST_WP_VERSION_CACHE_TTL:-86400}" + +# Print the cached versions file if it is younger than the given number of +# seconds. A negative TTL accepts it at any age. +read_versions_cache() { + local ttl="$1" + local age + + [ -s "${WP_VERSIONS_CACHE_FILE}" ] || return 1 + + if [ "${ttl}" -ge 0 ]; then + # PHP rather than `find -newermt`, which is not portable across + # GNU and BSD userlands. The Behat runner needs PHP anyway. + age=$(php -r 'echo time() - filemtime( $argv[1] );' "${WP_VERSIONS_CACHE_FILE}" 2>/dev/null) + case ${age} in + ''|*[!0-9]*) return 1;; + esac + [ "${age}" -lt "${ttl}" ] || return 1 + fi + + cat "${WP_VERSIONS_CACHE_FILE}" +} + +# Print the WordPress versions data, from the cache where possible. Warnings go +# to STDERR so that they cannot end up inside the returned JSON. +get_wp_versions() { + local json + + json=$( read_versions_cache "${WP_VERSIONS_CACHE_TTL}" ) + if [ -n "${json}" ]; then + printf '%s' "${json}" + return 0 + fi + + json=$( curl -s "${WP_VERSIONS_URL}" ) + + # Only cache a well-formed response; an error page is not one. + if echo "${json}" | jq -e 'type == "object" and length > 0' > /dev/null 2>&1; then + mkdir -p "$( dirname "${WP_VERSIONS_CACHE_FILE}" )" 2>/dev/null \ + && printf '%s' "${json}" > "${WP_VERSIONS_CACHE_FILE}" 2>/dev/null || true + printf '%s' "${json}" + return 0 + fi + + # Prefer a stale answer over no answer. + json=$( read_versions_cache -1 ) + if [ -n "${json}" ]; then + echo "Warning: Could not fetch the WordPress versions data, falling back to the cached copy." >&2 + printf '%s' "${json}" + return 0 + fi + + return 1 +} + # Turn WP_VERSION into an actual number to make sure our tags work correctly. if [ "${WP_VERSION-latest}" = "latest" ]; then - export WP_VERSION=$(curl -s https://api.wordpress.org/core/version-check/1.7/ | jq -r ".offers[0].current") -fi + WP_VERSION=$( get_wp_versions | jq -r 'to_entries | map( select( .value == "latest" ) ) | last | .key // empty' ) -# Normalize WP_VERSION=X.Y.0 to X.Y (WordPress uses X.Y for the initial release, not X.Y.0). -# If WP_VERSION=X.Y (major.minor only), resolve to the latest available patch release. -if [[ "${WP_VERSION}" =~ ^([0-9]+\.[0-9]+)\.0$ ]]; then + if [ -z "${WP_VERSION}" ]; then + echo "Warning: Could not determine the latest WordPress version. Version-specific tags will not be filtered." + fi + + export WP_VERSION +# Normalize WP_VERSION=X.Y.0 to X.Y (WordPress uses X.Y for the initial release, +# not X.Y.0). This asks for that specific release, so it must not fall through to +# the patch resolution below. +elif [[ "${WP_VERSION}" =~ ^([0-9]+\.[0-9]+)\.0$ ]]; then export WP_VERSION="${BASH_REMATCH[1]}" +# If WP_VERSION=X.Y (major.minor only), resolve to the latest available patch release. elif [[ "${WP_VERSION}" =~ ^[0-9]+\.[0-9]+$ ]]; then - WP_VERSIONS_JSON=$(curl -s https://raw.githubusercontent.com/wp-cli/wp-cli-tests/artifacts/wp-versions.json) - if [ -n "${WP_VERSIONS_JSON}" ]; then - RESOLVED_VERSION=$(echo "${WP_VERSIONS_JSON}" | jq -r --arg prefix "${WP_VERSION}." 'keys | map(select(startswith($prefix))) | sort_by(split(".") | map(tonumber)) | last // empty') - if [ -n "${RESOLVED_VERSION}" ]; then - export WP_VERSION="${RESOLVED_VERSION}" - fi + RESOLVED_VERSION=$( get_wp_versions | jq -r --arg prefix "${WP_VERSION}." 'keys | map( select( startswith( $prefix ) ) ) | sort_by( split(".") | map( tonumber ) ) | last // empty' ) + + if [ -n "${RESOLVED_VERSION}" ]; then + export WP_VERSION="${RESOLVED_VERSION}" fi fi @@ -141,6 +209,11 @@ if [[ "${WP_CLI_TEST_COVERAGE}" == "true" ]] && vendor/bin/behat --help 2>/dev/n BEHAT_EXTRA_ARGS+=('--xdebug') fi +# Honor the NO_COLOR convention (https://no-color.org/). +if [ -n "${NO_COLOR}" ]; then + BEHAT_EXTRA_ARGS+=('--no-colors') +fi + # Run the functional tests. FORMAT_ARGS=(--format progress) for arg in "$@"; do diff --git a/bin/run-gherkin-lint-tests b/bin/run-gherkin-lint-tests new file mode 100755 index 00000000..794eb5e3 --- /dev/null +++ b/bin/run-gherkin-lint-tests @@ -0,0 +1,90 @@ +#!/bin/sh + +# Run the Gherkin linter only if there are feature files to lint. +if [ ! -d "features" ] +then + exit 0; +fi + +# The linter is a Node package, which a PHP project cannot assume is present. +# Skip rather than fail, the same way the Behat runner skips a package without a +# behat.yml. The check still runs in CI, where Node is always available. +if ! command -v npx > /dev/null 2>&1 +then + echo 'Did not detect the "npx" command, skipping the Gherkin linting.' + echo "It is part of Node.js 20 or later, see https://nodejs.org/ for installation instructions." + exit 0; +fi + +# To retrieve the WP-CLI tests package root folder, we start with this scripts +# location. +SOURCE="$0" + +# Resolve $SOURCE until the file is no longer a symlink. +while [ -h "$SOURCE" ]; do + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$( readlink "$SOURCE" )" + # If $SOURCE was a relative symlink, we need to resolve it relative to the + # path where the symlink file was located. + case $SOURCE in + /*) ;; + *) SOURCE="$DIR/$SOURCE";; + esac +done + +# Fetch the root folder of the WP-CLI tests package. +WP_CLI_TESTS_ROOT="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" + +# A project can override the shared ruleset by committing its own .gherkin-lintrc. +CONFIG_FILE=".gherkin-lintrc" +if [ ! -f "${CONFIG_FILE}" ] +then + CONFIG_FILE="${WP_CLI_TESTS_ROOT}/.gherkin-lintrc" +fi + +# The linter is pinned so that one of its releases cannot turn a green build red +# without a commit here. The pin lives in package.json rather than in this file, +# because a version string inside a shell script is invisible to Dependabot. +PACKAGE_JSON="${WP_CLI_TESTS_ROOT}/package.json" +GHERKIN_LINT_VERSION=$( php -r '$package = json_decode( (string) file_get_contents( $argv[1] ), true ); echo isset( $package["devDependencies"]["gherkin-lint-plus"] ) ? $package["devDependencies"]["gherkin-lint-plus"] : "";' "${PACKAGE_JSON}" 2> /dev/null ) + +if [ -z "${GHERKIN_LINT_VERSION}" ] +then + echo "Could not read the pinned gherkin-lint-plus version from ${PACKAGE_JSON}." + exit 1; +fi + +# Without an explicit path the linter walks the whole working directory, vendor +# included. The feature files are the target. +if [ $# -eq 0 ] +then + set -- features +fi + +run_linter() { + npx --yes "gherkin-lint-plus@${GHERKIN_LINT_VERSION}" --config "${CONFIG_FILE}" "$@" +} + +if [ -n "${NO_COLOR}" ] +then + # The linter colors its report unconditionally: it honors neither NO_COLOR + # nor the absence of a terminal, and "stylish" is its only output format. + # It writes the report to STDERR, so strip the escape sequences from that + # stream while leaving STDOUT and the exit code alone. + STDERR_FILE=$( mktemp ) + + run_linter "$@" 2> "${STDERR_FILE}" + STATUS=$? + + if [ -s "${STDERR_FILE}" ] + then + ESC=$( printf '\033' ) + sed "s/${ESC}\[[0-9;]*m//g" "${STDERR_FILE}" >&2 + fi + + rm -f "${STDERR_FILE}" + + exit ${STATUS}; +fi + +run_linter "$@" diff --git a/bin/run-linter-tests b/bin/run-linter-tests index add8ec33..d2c374ca 100755 --- a/bin/run-linter-tests +++ b/bin/run-linter-tests @@ -1,3 +1,11 @@ #!/bin/sh -vendor/bin/parallel-lint -j 10 --colors --exclude vendor . "$@" +# Honor the NO_COLOR convention (https://no-color.org/) by leaving the color +# decision to the tool's own TTY detection instead of forcing it. +COLOR_ARGS="--colors" +if [ -n "${NO_COLOR}" ]; then + COLOR_ARGS="" +fi + +# shellcheck disable=SC2086 # Intentional word splitting of the optional arguments. +vendor/bin/parallel-lint -j 10 $COLOR_ARGS --exclude vendor . "$@" diff --git a/bin/run-php-unit-tests b/bin/run-php-unit-tests index 6182d074..3d89befa 100755 --- a/bin/run-php-unit-tests +++ b/bin/run-php-unit-tests @@ -10,9 +10,19 @@ then EXTRA_ARGS="--display-warnings --fail-on-warning --display-notices --fail-on-notice --display-deprecations --fail-on-deprecation" fi + # Honor the NO_COLOR convention (https://no-color.org/). This has to be an + # explicit "never" rather than an omitted flag, because a project's + # phpunit.xml may well turn colors on by itself. + COLOR_ARGS="--color=always" + if [ -n "${NO_COLOR}" ]; then + COLOR_ARGS="--color=never" + fi + if [ -f "./vendor/wp-cli/wp-cli-tests/tests/bootstrap.php" ]; then - vendor/bin/phpunit --color=always "$@" $EXTRA_ARGS --bootstrap ./vendor/wp-cli/wp-cli-tests/tests/bootstrap.php + # shellcheck disable=SC2086 # Intentional word splitting of the optional arguments. + vendor/bin/phpunit $COLOR_ARGS "$@" $EXTRA_ARGS --bootstrap ./vendor/wp-cli/wp-cli-tests/tests/bootstrap.php else - vendor/bin/phpunit --color=always "$@" $EXTRA_ARGS + # shellcheck disable=SC2086 # Intentional word splitting of the optional arguments. + vendor/bin/phpunit $COLOR_ARGS "$@" $EXTRA_ARGS fi fi diff --git a/bin/run-phpcs-tests b/bin/run-phpcs-tests index 82d96b4e..cff48a9b 100755 --- a/bin/run-phpcs-tests +++ b/bin/run-phpcs-tests @@ -3,5 +3,15 @@ # Run the code style check only if a configuration file exists. if [ -f ".phpcs.xml" ] || [ -f "phpcs.xml" ] || [ -f ".phpcs.xml.dist" ] || [ -f "phpcs.xml.dist" ] then - vendor/bin/phpcs "$@" + EXTRA_ARGS="" + + # Compact, one-line-per-violation output without the progress ticker. + # Useful for CI logs and for AI coding agents, which pay for every token of + # the report they read back. + if [ -n "${WP_CLI_TEST_QUIET}" ]; then + EXTRA_ARGS="-q --report=emacs" + fi + + # shellcheck disable=SC2086 # Intentional word splitting of the optional arguments. + vendor/bin/phpcs $EXTRA_ARGS "$@" fi diff --git a/bin/run-phpstan-tests b/bin/run-phpstan-tests index 21f27690..235242ae 100755 --- a/bin/run-phpstan-tests +++ b/bin/run-phpstan-tests @@ -1,7 +1,22 @@ #!/bin/sh -# Run the code style check only if a configuration file exists. +# Run the static analysis only if a configuration file exists. if [ -f "phpstan.dist.neon" ] || [ -f "phpstan.neon.dist" ] || [ -f "phpstan.neon" ] then - vendor/bin/phpstan --memory-limit=2048M analyse "$@" + EXTRA_ARGS="" + + # Honor the NO_COLOR convention (https://no-color.org/). + if [ -n "${NO_COLOR}" ]; then + EXTRA_ARGS="--no-ansi" + fi + + # Replace the redrawing progress bar and the box-drawing result table with + # plain `file:line:message` lines. Useful for CI logs and for AI coding + # agents, which pay for every token of the report they read back. + if [ -n "${WP_CLI_TEST_QUIET}" ]; then + EXTRA_ARGS="${EXTRA_ARGS} --no-progress --error-format=raw" + fi + + # shellcheck disable=SC2086 # Intentional word splitting of the optional arguments. + vendor/bin/phpstan --memory-limit=2048M analyse $EXTRA_ARGS "$@" fi diff --git a/composer.json b/composer.json index c3250787..c0d59682 100644 --- a/composer.json +++ b/composer.json @@ -79,6 +79,7 @@ "bin/install-package-tests", "bin/rerun-behat-tests", "bin/run-behat-tests", + "bin/run-gherkin-lint-tests", "bin/run-linter-tests", "bin/run-php-unit-tests", "bin/run-phpcs-tests", @@ -89,6 +90,7 @@ "behat": "run-behat-tests", "behat-rerun": "rerun-behat-tests", "lint": "run-linter-tests", + "lint-gherkin": "run-gherkin-lint-tests", "phpcs": "run-phpcs-tests", "phpcbf": "run-phpcbf-cleanup", "phpstan": "run-phpstan-tests", @@ -96,6 +98,7 @@ "prepare-tests": "install-package-tests", "test": [ "@lint", + "@lint-gherkin", "@phpcs", "@phpstan", "@phpunit", diff --git a/package.json b/package.json new file mode 100644 index 00000000..c069a68b --- /dev/null +++ b/package.json @@ -0,0 +1,9 @@ +{ + "name": "wp-cli-tests", + "description": "Pins the Node tooling used by the WP-CLI testing framework. Not published to npm.", + "private": true, + "license": "MIT", + "devDependencies": { + "gherkin-lint-plus": "1.0.2" + } +}