Add support for validating stubs against specific PHP versions

Author: isfedorov

CONTRIBUTING.md

@@ -34,6 +34,7 @@ function curl_copy_handle(#[LanguageLevelTypeAware(['8.0' => 'CurlHandle'], defa
  * The easiest way to run the full test suite locally is the bundled script (requires Docker — it installs dependencies, regenerates the stubs and reflection caches, and runs every test suite):
    * macOS / Linux: `./runTests.sh`
    * Windows: `runTests.bat`
+ * While iterating on stubs for one PHP version, add `--php-version` (e.g. `./runTests.sh --php-version 5.6`) to validate against that version only — checks that do not apply to it are not generated at all. CI still runs every version.
  * To run suites manually instead, see [How to run tests](README.md#how-to-run-tests) in the README.
  * If a stub legitimately cannot match reflection (for example a runtime-dependent constant value, or an entity available only on certain PHP versions), register it in `tests/Framework/Validator/KnownProblems/DefaultKnownProblemsProvider.php`.
 

README.md

@@ -45,6 +45,17 @@ By default the script validates against the **committed** reflection caches
 them. Pass `--refresh-reflection` (e.g. `./runTests.sh --refresh-reflection`) to regenerate them
 locally; this is slow and rarely needed, and the result should not be committed (see below).
 
+To validate against a single PHP version instead of all of them, pass `--php-version`:
+
+* macOS / Linux: `./runTests.sh --php-version 5.6`
+* Windows: `runTests.bat --php-version 5.6`
+
+Only the checks that apply to that version run — a check declared for PHP 8.2+ produces no test cases
+for 5.6, so there is nothing to skip through. The `Unit` and `Structure` suites are not tied to a PHP
+version and always run in full, and a suite with nothing to run for the selected version (the
+`PhpDoc` checks, for example, are declared for the latest version only) is reported as empty rather
+than as a failure. Use this while iterating locally; CI always runs every version.
+
 Both scripts require Docker (they use the `test_runner` image defined in `docker-compose.yml`).
 
 #### Running suites manually

runTests.bat

@@ -17,15 +17,74 @@ rem only by the update-reflection-cache.yml workflow. By default we validate aga
 rem caches - exactly what CI does - so a normal run never rewrites them. Pass --refresh-reflection
 rem to regenerate them locally (slow; requires the per-version Docker images).
 set "REFRESH_REFLECTION=false"
-for %%A in (%*) do (
-  if "%%~A"=="--refresh-reflection" (
+rem Empty means "every version from the PhpVersions enum", which is what CI runs.
+set "PHP_VERSION_UNDER_TEST="
+
+:parse_args
+if "%~1"=="" goto :args_done
+if /i "%~1"=="--refresh-reflection" (
     set "REFRESH_REFLECTION=true"
-  ) else (
-    echo Unknown argument: %%~A
-    echo Usage: %~nx0 [--refresh-reflection]
-    endlocal
-    exit /b 1
-  )
+    shift
+    goto :parse_args
+)
+if /i "%~1"=="--php-version" (
+    if "%~2"=="" (
+        echo Missing value for --php-version
+        goto :usage
+    )
+    set "PHP_VERSION_UNDER_TEST=%~2"
+    shift
+    shift
+    goto :parse_args
+)
+set "CURRENT_ARG=%~1"
+if /i "!CURRENT_ARG:~0,14!"=="--php-version=" (
+    for /f "tokens=1* delims==" %%a in ("!CURRENT_ARG!") do set "PHP_VERSION_UNDER_TEST=%%b"
+    if "!PHP_VERSION_UNDER_TEST!"=="" (
+        echo Missing value for --php-version
+        goto :usage
+    )
+    shift
+    goto :parse_args
+)
+echo Unknown argument: %~1
+goto :usage
+
+:args_done
+
+rem The version the validators run against is not a runner setting - it comes from the data providers,
+rem which iterate PhpVersions::cases() and bake the version into every data set name (see
+rem ValidatorTestBase::buildTestName(), e.g. checkClassExists_ArrayObject_5.6). Selecting a version
+rem therefore means filtering test names on that suffix. PHPUnit renders a named data set as
+rem testEntity"<name>", so one closing quote follows the version; the trailing "." in the regex matches
+rem it, which keeps the filter free of quotes that cmd would have to escape.
+rem Checks that do not apply to the selected version drop out on their own: the providers skip any
+rem descriptor whose PhpVersionRange excludes it, so no data set is generated in the first place.
+set "VERSION_FILTER_ARGS="
+if not "%PHP_VERSION_UNDER_TEST%"=="" (
+    rem Keep the accepted versions in sync with the canonical enum, like run-all-reflection-parsers.bat does.
+    set "PHP_ENUM_FILE=%SCRIPT_DIR%tests\Framework\Runner\PhpVersions.php"
+    if not exist "!PHP_ENUM_FILE!" (
+        echo Cannot find PhpVersions.php: !PHP_ENUM_FILE!
+        endlocal
+        exit /b 1
+    )
+    findstr /c:"= '%PHP_VERSION_UNDER_TEST%';" "!PHP_ENUM_FILE!" >nul
+    if errorlevel 1 (
+        echo Unknown PHP version: %PHP_VERSION_UNDER_TEST%
+        rem usebackq (command in backticks) so the single quotes inside the findstr pattern do not
+        rem terminate the command for cmd's parser.
+        for /f "usebackq tokens=2 delims='" %%V in (`findstr /r /c:"case PHP_[0-9_]* = '[0-9.]*';" "!PHP_ENUM_FILE!"`) do set "KNOWN_VERSIONS=!KNOWN_VERSIONS! %%V"
+        echo Valid versions:!KNOWN_VERSIONS!
+        endlocal
+        exit /b 1
+    )
+    rem Escape the dot for the regex: 5.6 -> 5\.6
+    set "VERSION_PATTERN=%PHP_VERSION_UNDER_TEST:.=\.%"
+    rem --do-not-fail-on-empty-test-suite: a suite can legitimately have nothing to run for the selected
+    rem version (the PhpDoc checks, for instance, are declared LATEST-only), and PHPUnit exits 1 on an
+    rem empty suite by default, which would abort the script.
+    set "VERSION_FILTER_ARGS=--filter _!VERSION_PATTERN!.$ --do-not-fail-on-empty-test-suite"
 )
 
 echo Installing composer packages...
@@ -38,12 +97,19 @@ if errorlevel 1 goto :fail
 
 if "%REFRESH_REFLECTION%"=="true" (
   echo Regenerating reflection caches...
-  call "%SCRIPT_DIR%tests\run-all-reflection-parsers.bat"
+  rem With a single version selected, only that version's cache needs rebuilding.
+  call "%SCRIPT_DIR%tests\run-all-reflection-parsers.bat" %PHP_VERSION_UNDER_TEST%
   if errorlevel 1 goto :fail
 ) else (
   echo Using committed reflection caches ^(pass --refresh-reflection to regenerate them^).
 )
 
+if not "%PHP_VERSION_UNDER_TEST%"=="" (
+  echo Validating stubs against PHP %PHP_VERSION_UNDER_TEST% only.
+)
+
+rem The Unit and Structure suites are not parameterised by PHP version (they test the framework itself
+rem and the layout of the stubs tree), so they always run in full.
 echo Running unit tests...
 call :dc run --rm test_runner vendor/bin/phpunit --testsuite Unit
 if errorlevel 1 goto :fail
@@ -53,11 +119,11 @@ call :dc run --rm test_runner vendor/bin/phpunit --testsuite Structure
 if errorlevel 1 goto :fail
 
 echo Running PHPDoc tests...
-call :dc run --rm test_runner vendor/bin/phpunit --testsuite PhpDoc
+call :dc run --rm test_runner vendor/bin/phpunit --testsuite PhpDoc !VERSION_FILTER_ARGS!
 if errorlevel 1 goto :fail
 
 echo Running validator tests...
-call :dc run --rm test_runner vendor/bin/phpunit --testsuite General
+call :dc run --rm test_runner vendor/bin/phpunit --testsuite General !VERSION_FILTER_ARGS!
 if errorlevel 1 goto :fail
 
 endlocal
@@ -67,6 +133,13 @@ exit /b 0
 docker compose -f "%COMPOSE_FILE%" %*
 exit /b %errorlevel%
 
+:usage
+echo Usage: %~nx0 [--refresh-reflection] [--php-version ^<version^>]
+echo   --refresh-reflection      Regenerate the reflection caches instead of using the committed ones.
+echo   --php-version ^<version^>   Validate stubs against a single PHP version (e.g. 5.6) instead of all of them.
+endlocal
+exit /b 1
+
 :fail
 echo.
 echo Aborting: a previous step failed.

runTests.sh

@@ -13,22 +13,84 @@ dc() {
   docker compose -f "$SCRIPT_DIR/docker-compose.yml" "$@"
 }
 
+usage() {
+  echo "Usage: $(basename "$0") [--refresh-reflection] [--php-version <version>]"
+  echo "  --refresh-reflection      Regenerate the reflection caches instead of using the committed ones."
+  echo "  --php-version <version>   Validate stubs against a single PHP version (e.g. 5.6) instead of all of them."
+}
+
 # Reflection caches (tests/cache/Reflection<version>.json) are committed ground truth, refreshed
 # only by the update-reflection-cache.yml workflow. By default we validate against the committed
 # caches — exactly what CI does — so a normal run never rewrites them. Pass --refresh-reflection to
 # regenerate them locally (slow; requires the per-version Docker images).
 REFRESH_REFLECTION=false
-for arg in "$@"; do
-  case "$arg" in
+# Empty means "every version from the PhpVersions enum", which is what CI runs.
+PHP_VERSION_UNDER_TEST=""
+while [ $# -gt 0 ]; do
+  case "$1" in
     --refresh-reflection) REFRESH_REFLECTION=true ;;
+    --php-version=*)
+      PHP_VERSION_UNDER_TEST="${1#*=}"
+      if [ -z "$PHP_VERSION_UNDER_TEST" ]; then
+        echo "Missing value for --php-version"
+        usage
+        exit 1
+      fi
+      ;;
+    --php-version)
+      if [ $# -lt 2 ]; then
+        echo "Missing value for --php-version"
+        usage
+        exit 1
+      fi
+      PHP_VERSION_UNDER_TEST="$2"
+      shift
+      ;;
     *)
-      echo "Unknown argument: $arg"
-      echo "Usage: $(basename "$0") [--refresh-reflection]"
+      echo "Unknown argument: $1"
+      usage
       exit 1
       ;;
   esac
+  shift
 done
 
+# The version the validators run against is not a runner setting — it comes from the data providers,
+# which iterate PhpVersions::cases() and bake the version into every data set name (see
+# ValidatorTestBase::buildTestName(), e.g. checkClassExists_ArrayObject_5.6). Selecting a version
+# therefore means filtering test names on that suffix. PHPUnit renders a named data set as
+# testEntity"<name>", so one closing quote follows the version; the trailing "." in the regex matches
+# it (a literal quote would need per-shell escaping, and runTests.bat has to build the same filter).
+# Checks that do not apply to the selected version drop out on their own: the providers skip any
+# descriptor whose PhpVersionRange excludes it, so no data set is generated in the first place.
+VERSION_FILTER_ARGS=()
+if [ -n "$PHP_VERSION_UNDER_TEST" ]; then
+  # Keep the accepted versions in sync with the canonical enum, like run-all-reflection-parsers.sh does.
+  PHP_ENUM_FILE="$SCRIPT_DIR/tests/Framework/Runner/PhpVersions.php"
+  if [ ! -f "$PHP_ENUM_FILE" ]; then
+    echo "Cannot find PhpVersions.php: $PHP_ENUM_FILE"
+    exit 1
+  fi
+  KNOWN_VERSIONS=($(sed -n "s/.*case PHP_[A-Z0-9_]* = '\([0-9.]*\)'.*/\1/p" "$PHP_ENUM_FILE"))
+  VERSION_IS_KNOWN=false
+  for known in "${KNOWN_VERSIONS[@]}"; do
+    if [ "$known" = "$PHP_VERSION_UNDER_TEST" ]; then
+      VERSION_IS_KNOWN=true
+      break
+    fi
+  done
+  if [ "$VERSION_IS_KNOWN" = false ]; then
+    echo "Unknown PHP version: $PHP_VERSION_UNDER_TEST"
+    echo "Valid versions: ${KNOWN_VERSIONS[*]}"
+    exit 1
+  fi
+
+  # --do-not-fail-on-empty-test-suite: a suite can legitimately have nothing to run for the selected
+  # version (the PhpDoc checks, for instance, are declared LATEST-only), and PHPUnit exits 1 on an
+  # empty suite by default, which would abort the script.
+  VERSION_FILTER_ARGS=(--filter "_${PHP_VERSION_UNDER_TEST//./\\.}.\$" --do-not-fail-on-empty-test-suite)
+fi
+
 echo "Installing composer packages..."
 dc run --rm test_runner composer install --ignore-platform-reqs
 
@@ -37,19 +99,26 @@ dc run --rm test_runner php tests/run-stubs-parser.php
 
 if [ "$REFRESH_REFLECTION" = true ]; then
   echo "Regenerating reflection caches..."
-  bash "$SCRIPT_DIR/tests/run-all-reflection-parsers.sh"
+  # With a single version selected, only that version's cache needs rebuilding.
+  bash "$SCRIPT_DIR/tests/run-all-reflection-parsers.sh" ${PHP_VERSION_UNDER_TEST:+"$PHP_VERSION_UNDER_TEST"}
 else
   echo "Using committed reflection caches (pass --refresh-reflection to regenerate them)."
 fi
 
+if [ -n "$PHP_VERSION_UNDER_TEST" ]; then
+  echo "Validating stubs against PHP $PHP_VERSION_UNDER_TEST only."
+fi
+
+# The Unit and Structure suites are not parameterised by PHP version (they test the framework itself
+# and the layout of the stubs tree), so they always run in full.
 echo "Running unit tests..."
 dc run --rm test_runner vendor/bin/phpunit --testsuite Unit
 
 echo "Running structure tests..."
 dc run --rm test_runner vendor/bin/phpunit --testsuite Structure
 
 echo "Running PHPDoc tests..."
-dc run --rm test_runner vendor/bin/phpunit --testsuite PhpDoc
+dc run --rm test_runner vendor/bin/phpunit --testsuite PhpDoc ${VERSION_FILTER_ARGS[@]+"${VERSION_FILTER_ARGS[@]}"}
 
 echo "Running validator tests..."
-dc run --rm test_runner vendor/bin/phpunit --testsuite General
+dc run --rm test_runner vendor/bin/phpunit --testsuite General ${VERSION_FILTER_ARGS[@]+"${VERSION_FILTER_ARGS[@]}"}