diff --git a/bin/phpstan b/bin/phpstan index 40afd0f7bf8..f5d2b36ff70 100755 --- a/bin/phpstan +++ b/bin/phpstan @@ -2,6 +2,7 @@ add(new AnalyseCommand($reversedComposerAutoloaderProjectPaths, $analysisStartTime)); + $application->add(new BisectCommand()); $application->add(new WorkerCommand($reversedComposerAutoloaderProjectPaths)); $application->add(new ClearResultCacheCommand($reversedComposerAutoloaderProjectPaths)); $application->add(new FixerWorkerCommand($reversedComposerAutoloaderProjectPaths)); diff --git a/src/Command/Bisect/BinarySearch.php b/src/Command/Bisect/BinarySearch.php new file mode 100644 index 00000000000..a3260522f66 --- /dev/null +++ b/src/Command/Bisect/BinarySearch.php @@ -0,0 +1,36 @@ + $items Items ordered from oldest to newest (at least 2) + * @return BinarySearchStep + */ + public static function getStep(array $items): BinarySearchStep + { + $count = count($items); + if ($count < 2) { + throw new InvalidArgumentException('Binary search requires at least 2 items.'); + } + + $mid = (int) (($count - 1) / 2); + + return new BinarySearchStep( + $items[$mid], + array_slice($items, $mid + 1), + array_slice($items, 0, $mid + 1), + (int) ceil(log($count, 2)), + ); + } + +} diff --git a/src/Command/Bisect/BinarySearchStep.php b/src/Command/Bisect/BinarySearchStep.php new file mode 100644 index 00000000000..2b7570824cb --- /dev/null +++ b/src/Command/Bisect/BinarySearchStep.php @@ -0,0 +1,25 @@ + $ifGood Remaining items to search if this item is good + * @param list $ifBad Remaining items to search if this item is bad + */ + public function __construct( + public readonly mixed $item, + public readonly array $ifGood, + public readonly array $ifBad, + public readonly int $stepsRemaining, + ) + { + } + +} diff --git a/src/Command/BisectCommand.php b/src/Command/BisectCommand.php new file mode 100644 index 00000000000..8cb0c0382dd --- /dev/null +++ b/src/Command/BisectCommand.php @@ -0,0 +1,457 @@ +setName(self::NAME) + ->setDescription('Binary search for the first bad PHPStan commit between two releases') + ->setDefinition([ + new InputArgument('paths', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'Paths with source code to run analysis on'), + new InputOption('good', null, InputOption::VALUE_REQUIRED, 'Good (old) PHPStan release version (e.g. 2.1.0)'), + new InputOption('bad', null, InputOption::VALUE_REQUIRED, 'Bad (new) PHPStan release version (e.g. 2.1.5)'), + new InputOption('configuration', 'c', InputOption::VALUE_REQUIRED, 'Path to project configuration file'), + new InputOption(AnalyseCommand::OPTION_LEVEL, 'l', InputOption::VALUE_REQUIRED, 'Level of rule options - the higher the stricter'), + new InputOption('autoload-file', 'a', InputOption::VALUE_REQUIRED, 'Project\'s additional autoload file path'), + new InputOption('memory-limit', null, InputOption::VALUE_REQUIRED, 'Memory limit for analysis'), + ]); + } + + #[Override] + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + + $good = $input->getOption('good'); + if (!is_string($good)) { + if (!$input->isInteractive()) { + $io->error('Both --good and --bad release versions are required in non-interactive mode.'); + return 1; + } + $good = $io->ask('Enter the good (working) PHPStan release version (e.g. 2.1.0)'); + } + + $bad = $input->getOption('bad'); + if (!is_string($bad)) { + if (!$input->isInteractive()) { + $io->error('Both --good and --bad release versions are required in non-interactive mode.'); + return 1; + } + $bad = $io->ask('Enter the bad (broken) PHPStan release version (e.g. 2.1.5)'); + } + + if (!is_string($good) || !is_string($bad)) { + $io->error('Both good and bad release versions are required.'); + return 1; + } + + $token = $this->getGitHubToken(); + if ($token === null) { + $io->error([ + 'GitHub token not found.', + 'Please set the GITHUB_TOKEN or GH_TOKEN environment variable,', + 'or add a GitHub OAuth token to ~/.composer/auth.json.', + ]); + return 1; + } + + $client = new Client([ + RequestOptions::TIMEOUT => 30, + RequestOptions::CONNECT_TIMEOUT => 10, + 'headers' => [ + 'Authorization' => 'token ' . $token, + 'Accept' => 'application/vnd.github.v3+json', + ], + ]); + + $io->section(sprintf('Fetching commits between %s and %s...', $good, $bad)); + + try { + $commits = $this->getCommitsBetween($client, $good, $bad); + } catch (GuzzleException $e) { + $io->error(sprintf('Failed to fetch commits from GitHub: %s', $e->getMessage())); + return 1; + } + + if (count($commits) === 0) { + $io->error('No commits found between the specified releases.'); + return 1; + } + + $io->writeln(sprintf('Found %d commits between %s and %s.', count($commits), $good, $bad)); + + $rangeShas = []; + foreach ($commits as $commit) { + $rangeShas[$commit['sha']] = true; + } + + try { + $checksumShas = $this->getPharChecksumCommitShas($client, $bad, $rangeShas); + } catch (GuzzleException $e) { + $io->error(sprintf('Failed to fetch .phar-checksum commits from GitHub: %s', $e->getMessage())); + return 1; + } + + $commits = array_values(array_filter($commits, static fn (array $commit): bool => isset($checksumShas[$commit['sha']]))); + + if (count($commits) === 0) { + $io->error('No commits found that change phpstan.phar between the specified releases.'); + return 1; + } + + $io->writeln(sprintf('%d of them change phpstan.phar.', count($commits))); + + $tmpDir = sys_get_temp_dir() . '/phpstan-bisect'; + @mkdir($tmpDir, 0777, true); + + $analyseArgs = $this->buildAnalyseArgs($input); + + while (count($commits) > 1) { + $step = BinarySearch::getStep($commits); + $commit = $step->item; + $sha = $commit['sha']; + $shortSha = substr($sha, 0, 7); + $message = $commit['commit']['message']; + $firstLine = strtok($message, "\n") ?: $shortSha; + + $io->section(sprintf( + 'Testing commit %s (%s) [~%d step%s left]', + $shortSha, + $firstLine, + $step->stepsRemaining, + $step->stepsRemaining === 1 ? '' : 's', + )); + + $pharPath = $tmpDir . '/phpstan-' . $shortSha . '.phar'; + if (!is_file($pharPath)) { + $io->writeln('Downloading phpstan.phar...'); + try { + $this->downloadPharForCommit($client, $sha, $pharPath, $output); + } catch (GuzzleException $e) { + $io->error(sprintf('Failed to download phpstan.phar: %s', $e->getMessage())); + return 1; + } + } + + $io->writeln('Running analysis...'); + $io->newLine(); + $exitCode = $this->runAnalysis($pharPath, $analyseArgs); + $io->newLine(); + $io->writeln(sprintf('Analysis exited with code: %d', $exitCode)); + + if (!$input->isInteractive()) { + $io->error('Cannot continue bisect in non-interactive mode.'); + return 1; + } + + $answer = $io->choice( + 'Is this result good or bad?', + ['good', 'bad'], + ); + + $commits = $answer === 'good' ? $step->ifGood : $step->ifBad; + } + + $badCommit = $commits[0]; + $this->printResult($badCommit, $io); + + return 0; + } + + public function getGitHubToken(?string $composerHome = null): ?string + { + $envToken = getenv('GITHUB_TOKEN'); + if ($envToken !== false && $envToken !== '') { + return $envToken; + } + + $ghToken = getenv('GH_TOKEN'); + if ($ghToken !== false && $ghToken !== '') { + return $ghToken; + } + + if ($composerHome === null) { + $composerHome = getenv('COMPOSER_HOME'); + if ($composerHome === false) { + $home = getenv('HOME'); + if ($home === false) { + $home = getenv('USERPROFILE'); + } + if ($home === false) { + return null; + } + $composerHome = $home . '/.composer'; + } + } + + $authFile = $composerHome . '/auth.json'; + if (!is_file($authFile)) { + return null; + } + + try { + /** @var array{github-oauth?: array} $auth */ + $auth = Json::decode(FileReader::read($authFile), Json::FORCE_ARRAY); + return $auth['github-oauth']['github.com'] ?? null; + } catch (Throwable) { + return null; + } + } + + /** + * @return list + * @throws GuzzleException + */ + private function getCommitsBetween(Client $client, string $good, string $bad): array + { + $allCommits = []; + $page = 1; + $perPage = 100; + + while (true) { + $response = $client->get(sprintf( + 'https://api.github.com/repos/%s/%s/compare/%s...%s?per_page=%d&page=%d', + self::REPO_OWNER, + self::REPO_NAME, + urlencode($good), + urlencode($bad), + $perPage, + $page, + )); + + /** @var array{commits: list, total_commits: int} $data */ + $data = Json::decode($response->getBody()->getContents(), Json::FORCE_ARRAY); + $commits = $data['commits']; + $allCommits = array_merge($allCommits, $commits); + + if (count($commits) < $perPage || count($allCommits) >= $data['total_commits']) { + break; + } + + $page++; + } + + return $allCommits; + } + + /** + * @param array $rangeShas + * @return array + * @throws GuzzleException + */ + private function getPharChecksumCommitShas(Client $client, string $bad, array $rangeShas): array + { + $checksumShas = []; + $page = 1; + $perPage = 100; + + while (true) { + $response = $client->get(sprintf( + 'https://api.github.com/repos/%s/%s/commits?sha=%s&path=%s&per_page=%d&page=%d', + self::REPO_OWNER, + self::REPO_NAME, + urlencode($bad), + urlencode('.phar-checksum'), + $perPage, + $page, + )); + + /** @var list $commits */ + $commits = Json::decode($response->getBody()->getContents(), Json::FORCE_ARRAY); + + if (count($commits) === 0) { + break; + } + + $foundOutOfRange = false; + foreach ($commits as $commit) { + if (!isset($rangeShas[$commit['sha']])) { + $foundOutOfRange = true; + break; + } + + $checksumShas[$commit['sha']] = true; + } + + if ($foundOutOfRange || count($commits) < $perPage) { + break; + } + + $page++; + } + + return $checksumShas; + } + + /** + * @throws GuzzleException + */ + private function downloadPharForCommit(Client $client, string $sha, string $pharPath, OutputInterface $output): void + { + $url = sprintf( + 'https://raw.githubusercontent.com/%s/%s/%s/phpstan.phar', + self::REPO_OWNER, + self::REPO_NAME, + $sha, + ); + + $progressBar = new ProgressBar($output); + $bytes = 0; + + $client->get($url, [ + RequestOptions::SINK => $pharPath, + RequestOptions::TIMEOUT => 120, + RequestOptions::PROGRESS => static function (int $downloadTotal, int $downloadedBytes) use ($progressBar, &$bytes): void { + if ($downloadTotal === 0) { + return; + } + if ($progressBar->getMaxSteps() === 0) { + $progressBar->setFormat('file_download'); + $progressBar->setMessage(sprintf('%.2f MB', $downloadTotal / 1000000), 'fileSize'); + $progressBar->start($downloadTotal); + } + if ($downloadedBytes <= $bytes) { + return; + } + $bytes = $downloadedBytes; + $progressBar->setProgress($bytes); + }, + ]); + + $progressBar->finish(); + $output->writeln(''); + + chmod($pharPath, 0755); + } + + public function buildAnalyseArgs(InputInterface $input): string + { + $args = []; + + $config = $input->getOption('configuration'); + if (is_string($config)) { + $args[] = '-c ' . escapeshellarg($config); + } + + $level = $input->getOption(AnalyseCommand::OPTION_LEVEL); + if (is_string($level)) { + $args[] = '-l ' . escapeshellarg($level); + } + + $autoload = $input->getOption('autoload-file'); + if (is_string($autoload)) { + $args[] = '-a ' . escapeshellarg($autoload); + } + + $memory = $input->getOption('memory-limit'); + if (is_string($memory)) { + $args[] = '--memory-limit=' . escapeshellarg($memory); + } + + $args[] = '--no-progress'; + + $paths = $input->getArgument('paths'); + if (is_array($paths)) { + foreach ($paths as $path) { + if (!is_string($path)) { + continue; + } + + $args[] = escapeshellarg($path); + } + } + + return implode(' ', $args); + } + + private function runAnalysis(string $pharPath, string $analyseArgs): int + { + $command = sprintf( + '%s %s analyse %s', + escapeshellarg(PHP_BINARY), + escapeshellarg($pharPath), + $analyseArgs, + ); + + passthru($command, $exitCode); + return $exitCode; + } + + /** + * @param array{sha: string, commit: array{message: string}} $commit + */ + private function printResult(array $commit, SymfonyStyle $io): void + { + $sha = $commit['sha']; + $message = $commit['commit']['message']; + + $io->success('Found the first bad commit!'); + $io->writeln(sprintf('Commit: %s', $sha)); + $io->writeln(sprintf('URL: https://github.com/%s/%s/commit/%s', self::REPO_OWNER, self::REPO_NAME, $sha)); + $io->newLine(); + $io->writeln('Commit message:'); + $io->writeln($message); + + if (preg_match_all('#https://github\.com/phpstan/phpstan-src/commit/[a-f0-9]+(?:\s+.*)?#', $message, $matches) < 1) { + return; + } + + $io->newLine(); + $io->writeln('Related phpstan-src commits:'); + foreach ($matches[0] as $line) { + $io->writeln(sprintf(' %s', $line)); + } + } + +} diff --git a/tests/PHPStan/Command/Bisect/BinarySearchTest.php b/tests/PHPStan/Command/Bisect/BinarySearchTest.php new file mode 100644 index 00000000000..da3edd46590 --- /dev/null +++ b/tests/PHPStan/Command/Bisect/BinarySearchTest.php @@ -0,0 +1,150 @@ + $items + * @param list $expectedIfGood + * @param list $expectedIfBad + */ + #[DataProvider('dataGetStep')] + public function testGetStep( + array $items, + string $expectedItem, + array $expectedIfGood, + array $expectedIfBad, + int $expectedStepsRemaining, + ): void + { + $step = BinarySearch::getStep($items); + $this->assertSame($expectedItem, $step->item); + $this->assertSame($expectedIfGood, $step->ifGood); + $this->assertSame($expectedIfBad, $step->ifBad); + $this->assertSame($expectedStepsRemaining, $step->stepsRemaining); + } + + public static function dataGetStep(): iterable + { + yield 'two items' => [ + ['a', 'b'], + 'a', + ['b'], + ['a'], + 1, + ]; + + yield 'three items' => [ + ['a', 'b', 'c'], + 'b', + ['c'], + ['a', 'b'], + 2, + ]; + + yield 'four items' => [ + ['a', 'b', 'c', 'd'], + 'b', + ['c', 'd'], + ['a', 'b'], + 2, + ]; + + yield 'five items' => [ + ['a', 'b', 'c', 'd', 'e'], + 'c', + ['d', 'e'], + ['a', 'b', 'c'], + 3, + ]; + + yield 'eight items' => [ + ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'], + 'd', + ['e', 'f', 'g', 'h'], + ['a', 'b', 'c', 'd'], + 3, + ]; + + yield 'sixteen items' => [ + array_map(static fn (int $i): string => (string) $i, range(1, 16)), + '8', + ['9', '10', '11', '12', '13', '14', '15', '16'], + ['1', '2', '3', '4', '5', '6', '7', '8'], + 4, + ]; + } + + /** + * @param list $items + */ + #[DataProvider('dataTooFewItems')] + public function testGetStepWithTooFewItems(array $items): void + { + $this->expectException(InvalidArgumentException::class); + BinarySearch::getStep($items); + } + + public static function dataTooFewItems(): iterable + { + yield 'empty' => [[]]; + yield 'single item' => [['a']]; + } + + /** + * @param list $items + */ + #[DataProvider('dataFullBisect')] + public function testFullBisect(array $items, string $firstBadItem): void + { + $badIndex = array_search($firstBadItem, $items, true); + $this->assertNotFalse($badIndex); + + $current = $items; + $steps = 0; + $initialStep = BinarySearch::getStep($current); + + while (count($current) > 1) { + $step = BinarySearch::getStep($current); + $testIndex = array_search($step->item, $items, true); + $this->assertNotFalse($testIndex); + + $isBad = $testIndex >= $badIndex; + $current = $isBad ? $step->ifBad : $step->ifGood; + $steps++; + } + + $this->assertCount(1, $current); + $this->assertSame($firstBadItem, $current[0]); + $this->assertLessThanOrEqual($initialStep->stepsRemaining, $steps); + } + + public static function dataFullBisect(): iterable + { + $lists = [ + '2 items' => ['a', 'b'], + '3 items' => ['a', 'b', 'c'], + '5 items' => ['a', 'b', 'c', 'd', 'e'], + '8 items' => ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'], + '16 items' => array_map(static fn (int $i): string => 'commit-' . $i, range(1, 16)), + ]; + + foreach ($lists as $name => $items) { + foreach ($items as $badItem) { + yield sprintf('%s, first bad is %s', $name, $badItem) => [$items, $badItem]; + } + } + } + +}