[perf] Run the workers with PHPStan Turbo (#8812)

Author: SanderMuller

.github/workflows/self_benchmark.yaml

@@ -0,0 +1,47 @@
+# Runs scripts/rector-ab-benchmark.php on Rector's own rules/ with the base branch and with the pull request,
+# and puts the A/B table in the job summary
+name: Self Benchmark
+
+on:
+    pull_request:
+        branches:
+            - main
+
+env:
+    COMPOSER_ROOT_VERSION: "dev-main"
+
+jobs:
+    self_benchmark:
+        runs-on: ubuntu-latest
+        timeout-minutes: 40
+
+        steps:
+            -
+                uses: actions/checkout@v5
+                with:
+                    ref: ${{ github.event.pull_request.base.sha }}
+                    path: base
+
+            -
+                uses: actions/checkout@v5
+                with:
+                    path: head
+
+            -
+                uses: shivammathur/setup-php@v2
+                with:
+                    php-version: 8.4
+                    coverage: none
+
+            -   run: composer install --no-progress --ansi --working-dir=base
+            -   run: composer install --no-progress --ansi --working-dir=head
+
+            # GNU time gives the CPU time and peak memory
+            -   run: sudo apt-get install -y time
+
+            # both sides process the same code: the rules/ directory of the pull request
+            -   run: php head/scripts/rector-ab-benchmark.php --bin=base/bin/rector --project=head/rules --runs=5 --save=base.json
+            -   run: php head/scripts/rector-ab-benchmark.php --bin=head/bin/rector --project=head/rules --runs=5 --baseline=base.json | tee results.txt
+
+            -   run: cat results.txt >> "$GITHUB_STEP_SUMMARY"
+                if: always()

bin/rector.php

@@ -4,6 +4,8 @@
 
 use Composer\XdebugHandler\XdebugHandler;
 use Nette\Utils\Json;
+use PHPStan\PharAutoloader;
+use PHPStan\Turbo\TurboExtensionEnabler;
 use Rector\Bootstrap\AutoloadFileParameterResolver;
 use Rector\Bootstrap\RectorConfigsResolver;
 use Rector\ChangesReporting\Output\JsonOutputFormatter;
@@ -30,6 +32,25 @@
 $autoloadIncluder = new AutoloadIncluder();
 $autoloadIncluder->includeDependencyOrRepositoryVendorAutoloadIfExists();
 
+// PHPStan Turbo replaces php-parser classes, so it is activated before preload.php declares them.
+// The main process only gets Turbo when php.ini loads it; the workers get it from WorkerCommandLineFactory.
+$phpstanBootstrapFile = __DIR__ . '/../../../phpstan/phpstan/bootstrap.php';
+if (! class_exists(PharAutoloader::class) && is_file($phpstanBootstrapFile)) {
+    // Rector installed as a dependency: the project autoloader that loads PHPStan comes after preload.php,
+    // so load PHPStan's bootstrap now, and mark it as loaded with the identifier Composer gives the file
+    $GLOBALS['__composer_autoload_files'][md5('phpstan/phpstan:bootstrap.php')] = true;
+    require_once $phpstanBootstrapFile;
+}
+
+if (class_exists(PharAutoloader::class)) {
+    $phpstanPharPath = dirname((string) new ReflectionClass(PharAutoloader::class)->getFileName()) . '/phpstan.phar';
+    TurboExtensionEnabler::trustOwnTypesIfSuitable($_SERVER['argv'], $phpstanPharPath);
+    TurboExtensionEnabler::activateIfCompatible($phpstanPharPath);
+    unset($phpstanPharPath);
+}
+
+unset($phpstanBootstrapFile);
+
 final class AutoloadIncluder
 {
     /**

build/target-repository/composer.json

@@ -9,7 +9,7 @@
     ],
     "require": {
         "php": "^7.4|^8.0",
-        "phpstan/phpstan": "^2.3.0"
+        "phpstan/phpstan": "^2.3.2"
     },
     "autoload": {
         "files": [

composer.json

@@ -23,7 +23,7 @@
         "nikic/php-parser": "^5.9",
         "ondram/ci-detector": "^4.2",
         "phpstan/phpdoc-parser": "^2.3.3",
-        "phpstan/phpstan": "^2.3.0",
+        "phpstan/phpstan": "^2.3.2",
         "react/child-process": "^0.6.5",
         "react/event-loop": "^1.6",
         "react/socket": "^1.17",

scripts/rector-ab-benchmark.php

@@ -26,6 +26,7 @@
  * (cache primed once, then reused):
  *
  * - median wall time
+ * - median CPU time, user + system with the workers included, when /usr/bin/time is available (Linux)
  * - peak resident memory, when /usr/bin/time is available (Linux)
  * - changed-file count AND a fingerprint of the sorted changed-file list. The fingerprint
  *   must stay identical across refs - a perf change that alters it changed behaviour, not
@@ -95,7 +96,7 @@ public function report(array $result, ?array $baseline): void
     }
 
     /**
-     * @return array{seconds: float, peakKb: int, changed: int, fingerprint: string}
+     * @return array{seconds: float, cpuSeconds: float, peakKb: int, changed: int, fingerprint: string}
      */
     private function measurePhase(string $phase, string $configPath, string $cacheDirectory, bool $cold): array
     {
@@ -105,6 +106,7 @@ private function measurePhase(string $phase, string $configPath, string $cacheDi
         }
 
         $durations = [];
+        $cpuDurations = [];
         $peakKb = 0;
         $changed = 0;
         $fingerprint = '';
@@ -115,6 +117,7 @@ private function measurePhase(string $phase, string $configPath, string $cacheDi
             $measurement = $this->runRector($configPath, $cacheDirectory, $cold);
 
             $durations[] = $measurement['seconds'];
+            $cpuDurations[] = $measurement['cpuSeconds'];
             $peakKb = max($peakKb, $measurement['peakKb']);
             $changed = $measurement['changed'];
             $fingerprint = $measurement['fingerprint'];
@@ -124,14 +127,15 @@ private function measurePhase(string $phase, string $configPath, string $cacheDi
 
         return [
             'seconds' => $this->median($durations),
+            'cpuSeconds' => $this->median($cpuDurations),
             'peakKb' => $peakKb,
             'changed' => $changed,
             'fingerprint' => $fingerprint,
         ];
     }
 
     /**
-     * @return array{seconds: float, peakKb: int, changed: int, fingerprint: string}
+     * @return array{seconds: float, cpuSeconds: float, peakKb: int, changed: int, fingerprint: string}
      */
     private function runRector(string $configPath, string $cacheDirectory, bool $cold): array
     {
@@ -159,12 +163,14 @@ private function runRector(string $configPath, string $cacheDirectory, bool $col
         $seconds = (hrtime(true) - $startedAt) / 1_000_000_000;
 
         $peakKb = $usesGnuTime ? $this->parsePeakKb($timeOutputPath) : 0;
+        $cpuSeconds = $usesGnuTime ? $this->parseCpuSeconds($timeOutputPath) : 0.0;
         @unlink($timeOutputPath);
 
         $changedFiles = $this->parseChangedFiles(is_string($jsonOutput) ? $jsonOutput : '');
 
         return [
             'seconds' => $seconds,
+            'cpuSeconds' => $cpuSeconds,
             'peakKb' => $peakKb,
             'changed' => count($changedFiles),
             'fingerprint' => substr(hash('xxh128', implode("\n", $changedFiles)), 0, 12),
@@ -235,6 +241,27 @@ private function parsePeakKb(string $timeErrorPath): int
         return 0;
     }
 
+    private function parseCpuSeconds(string $timeErrorPath): float
+    {
+        if (! is_file($timeErrorPath)) {
+            return 0.0;
+        }
+
+        $contents = file_get_contents($timeErrorPath);
+        if (! is_string($contents)) {
+            return 0.0;
+        }
+
+        $cpuSeconds = 0.0;
+        foreach (['User', 'System'] as $kind) {
+            if (preg_match('/' . $kind . ' time \(seconds\):\s*([\d.]+)/', $contents, $matches) === 1) {
+                $cpuSeconds += (float) $matches[1];
+            }
+        }
+
+        return $cpuSeconds;
+    }
+
     private function hasGnuTime(): bool
     {
         if (! is_file('/usr/bin/time')) {
@@ -266,17 +293,18 @@ private function median(array $values): float
     }
 
     /**
-     * @param array{seconds: float, peakKb: int, changed: int, fingerprint: string} $current
-     * @param array{seconds: float, peakKb: int, changed: int, fingerprint: string}|null $baseline
+     * @param array{seconds: float, cpuSeconds: float, peakKb: int, changed: int, fingerprint: string} $current
+     * @param array{seconds: float, cpuSeconds: float, peakKb: int, changed: int, fingerprint: string}|null $baseline
      */
     private function printPhase(string $phase, array $current, ?array $baseline): void
     {
         printf("## %s phase\n\n", $phase);
 
         if ($baseline === null) {
             printf(
-                "wall:    %.2f s\npeak:    %s\nchanged: %d (fingerprint %s)\n\n",
+                "wall:    %.2f s\ncpu:     %s\npeak:    %s\nchanged: %d (fingerprint %s)\n\n",
                 $current['seconds'],
+                $this->formatCpu($current['cpuSeconds']),
                 $this->formatMemory($current['peakKb']),
                 $current['changed'],
                 $current['fingerprint']
@@ -286,11 +314,13 @@ private function printPhase(string $phase, array $current, ?array $baseline): vo
         }
 
         $wallDelta = $this->percentDelta($baseline['seconds'], $current['seconds']);
+        $cpuDelta = $this->percentDelta($baseline['cpuSeconds'], $current['cpuSeconds']);
         $memoryDelta = $this->percentDelta((float) $baseline['peakKb'], (float) $current['peakKb']);
 
         printf("| metric | baseline | this run | delta |\n");
         printf("| --- | --: | --: | --: |\n");
         printf("| wall | %.2f s | %.2f s | %s |\n", $baseline['seconds'], $current['seconds'], $wallDelta);
+        printf("| cpu | %s | %s | %s |\n", $this->formatCpu($baseline['cpuSeconds']), $this->formatCpu($current['cpuSeconds']), $cpuDelta);
         printf("| peak | %s | %s | %s |\n", $this->formatMemory($baseline['peakKb']), $this->formatMemory($current['peakKb']), $memoryDelta);
         printf("| changed | %d | %d | %s |\n", $baseline['changed'], $current['changed'], $baseline['changed'] === $current['changed'] ? 'same' : 'DIFFERENT');
 
@@ -312,6 +342,15 @@ private function percentDelta(float $baseline, float $current): string
         return sprintf('%+.1f%% (%s)', $delta, $delta < 0 ? 'faster/less' : 'slower/more');
     }
 
+    private function formatCpu(float $seconds): string
+    {
+        if ($seconds <= 0.0) {
+            return 'n/a';
+        }
+
+        return sprintf('%.2f s', $seconds);
+    }
+
     private function formatMemory(int $kb): string
     {
         if ($kb === 0) {

src/Parallel/Command/WorkerCommandLineFactory.php

@@ -4,6 +4,7 @@
 
 namespace Rector\Parallel\Command;
 
+use PHPStan\Turbo\TurboExtensionSelector;
 use Rector\ChangesReporting\Output\JsonOutputFormatter;
 use Rector\Configuration\Option;
 use Rector\FileSystem\FilePathHelper;
@@ -46,7 +47,7 @@ public function create(
             $commandArguments = array_merge(['process'], $commandArguments);
         }
 
-        $args = [PHP_BINARY, $mainScript, ...$commandArguments];
+        $args = [PHP_BINARY, ...$this->createTurboExtensionArguments(), $mainScript, ...$commandArguments];
         $workerCommandArray = [];
 
         $mainCommand = $this->commandFromReflectionFactory->create($mainCommandClass);
@@ -140,6 +141,27 @@ private function shouldSkipOption(InputInterface $input, string $optionName): bo
         return $optionName === Option::OUTPUT_FORMAT || $optionName === Option::CONFIG;
     }
 
+    /**
+     * The workers get PHPStan Turbo on their command line, so the main process needs no restart to enable it.
+     *
+     * @return string[]
+     */
+    private function createTurboExtensionArguments(): array
+    {
+        $extensionPath = TurboExtensionSelector::findExtensionForWorkers();
+        if ($extensionPath === null) {
+            return [];
+        }
+
+        // quote the value, so PHP parses it as a string even when the path contains an operator like ~
+        return [
+            '-d',
+            "extension='" . $extensionPath . "'",
+            '-d',
+            TurboExtensionSelector::EXTENSION_PATH_INI . "='" . $extensionPath . "'",
+        ];
+    }
+
     /**
      * @return string[]
      */