diff --git a/.gitignore b/.gitignore index 740326ab1bbcd6c775cdc1b3d72c9d4e62191131..394dfe37405bf906cc5996c232e5a2997256ed98 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,8 @@ nbproject # # Ignore build stuff /.ddev/* +/Build/.phpunit.result.cache +/Build/FunctionalTests-Job-* /Build/bower_components/* /Build/node_modules/* /Build/JavaScript diff --git a/Build/Scripts/splitAcceptanceTests.php b/Build/Scripts/splitAcceptanceTests.php new file mode 100755 index 0000000000000000000000000000000000000000..d55041858cc33100c0207bf6158c7544b889cec2 --- /dev/null +++ b/Build/Scripts/splitAcceptanceTests.php @@ -0,0 +1,254 @@ +#!/usr/bin/env php +<?php +declare(strict_types=1); + +/* + * This file is part of the TYPO3 CMS project. + * + * It is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, either version 2 + * of the License, or any later version. + * + * For the full copyright and license information, please read the + * LICENSE.txt file that was distributed with this source code. + * + * The TYPO3 project - inspiring people to share! + */ + +use PhpParser\Comment\Doc; +use PhpParser\Node; +use PhpParser\NodeTraverser; +use PhpParser\NodeVisitor\NameResolver; +use PhpParser\NodeVisitorAbstract; +use PhpParser\ParserFactory; +use Symfony\Component\Console\Input\ArgvInput; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputDefinition; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\ConsoleOutput; +use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Finder\Finder; +use Symfony\Component\Finder\SplFileInfo; + +if (PHP_SAPI !== 'cli') { + die('Script must be called from command line.' . chr(10)); +} + +require __DIR__ . '/../../vendor/autoload.php'; + +/** + * This script is typically executed by runTests.sh. + * + * The script expects to be run from the core root: + * ./Build/Scripts/splitAcceptanceTests.php <numberOfChunks> + * + * Verbose output with 8 chunks: + * ./Build/Scripts/splitAcceptanceTests.php 8 -v + * + * It's purpose is to find all core Backend acceptance tests and split them into + * pieces. In CI, there are for example 8 jobs for the ac tests and each picks one + * chunk of tests. This way, acceptance tests are run in parallel + * and thus reduce the overall runtime of the test suite. + * + * codeception group files including their specific set of tests are written to: + * typo3/sysext/core/Tests/Acceptance/AcceptanceTests-Job-<counter> + */ +class SplitAcceptanceTests extends NodeVisitorAbstract +{ + /** + * Main entry method + */ + public function execute() + { + $input = new ArgvInput($_SERVER['argv'], $this->getInputDefinition()); + $output = new ConsoleOutput(); + + // Number of chunks and verbose output + $numberOfChunks = (int)$input->getArgument('numberOfChunks'); + + if ($numberOfChunks < 1 || $numberOfChunks > 99) { + throw new \InvalidArgumentException( + 'Main argument "numberOfChunks" must be at least 1 and maximum 99', + 1528319388 + ); + } + + if ($input->hasParameterOption('-v', true) || $input->hasParameterOption('--verbose', true)) { + $output->setVerbosity(OutputInterface::VERBOSITY_VERBOSE); + } + + // Find functional test files + $testFiles = (new Finder()) + ->files() + ->in(__DIR__ . '/../../typo3/sysext/core/Tests/Acceptance/Backend') + ->name('/Cest\.php$/') + ->sortByName() + ; + + $parser = (new ParserFactory())->create(ParserFactory::ONLY_PHP7); + $testStats = []; + foreach ($testFiles as $file) { + /** @var $file SplFileInfo */ + $relativeFilename = $file->getRealPath(); + preg_match('/.*typo3\/sysext\/(.*)$/', $relativeFilename, $matches); + $relativeFilename = $matches[1]; + + $ast = $parser->parse($file->getContents()); + $traverser = new NodeTraverser(); + $visitor = new NameResolver(); + $traverser->addVisitor($visitor); + $visitor = new AcceptanceTestCaseVisitor(); + $traverser->addVisitor($visitor); + $traverser->traverse($ast); + + $fqcn = $visitor->getFqcn(); + $tests = $visitor->getTests(); + if (!empty($tests)) { + $testStats[$relativeFilename] = 0; + } + + foreach ($tests as $test) { + if (isset($test['dataProvider'])) { + // Test uses a data provider - get number of data sets. Data provider methods in codeception + // are protected, so we reflect them and make them accessible to see how many test cases they contain. + $dataProviderMethodName = $test['dataProvider']; + $dataProviderMethod = new \ReflectionMethod($fqcn, $dataProviderMethodName); + $dataProviderMethod->setAccessible(true); + $numberOfDataSets = count($dataProviderMethod->invoke(new $fqcn())); + $testStats[$relativeFilename] += $numberOfDataSets; + } else { + // Just a single test + $testStats[$relativeFilename] += 1; + } + } + } + + // Sort test files by number of tests, descending + arsort($testStats); + + $numberOfTestsPerChunk = []; + for ($i = 1; $i <= $numberOfChunks; $i++) { + $numberOfTestsPerChunk[$i] = 0; + } + + foreach ($testStats as $testFile => $numberOfTestsInFile) { + // Sort list of tests per chunk by number of tests, pick lowest as + // the target of this test file + asort($numberOfTestsPerChunk); + reset($numberOfTestsPerChunk); + $jobFileNumber = key($numberOfTestsPerChunk); + + $content = str_replace('core/Tests/', '', $testFile) . "\n"; + file_put_contents(__DIR__ . '/../../typo3/sysext/core/Tests/Acceptance/' . 'AcceptanceTests-Job-' . $jobFileNumber, $content, FILE_APPEND); + + $numberOfTestsPerChunk[$jobFileNumber] = $numberOfTestsPerChunk[$jobFileNumber] + $numberOfTestsInFile; + } + + if ($output->isVerbose()) { + $output->writeln('Number of test files found: ' . count($testStats)); + $output->writeln('Number of tests found: ' . array_sum($testStats)); + $output->writeln('Number of chunks prepared: ' . $numberOfChunks); + ksort($numberOfTestsPerChunk); + foreach ($numberOfTestsPerChunk as $chunkNumber => $testNumber) { + $output->writeln('Number of tests in chunk ' . $chunkNumber . ': ' . $testNumber); + } + } + } + + /** + * Allowed script arguments + * + * @return InputDefinition argv input definition of symfony console + */ + private function getInputDefinition(): InputDefinition + { + return new InputDefinition([ + new InputArgument('numberOfChunks', InputArgument::REQUIRED, 'Number of chunks / jobs to create'), + new InputOption('--verbose', '-v', InputOption::VALUE_NONE, 'Enable verbose output'), + ]); + } +} + +/** + * nikic/php-parser node visitor to find test class namespace, + * count @test annotated methods and their possible @dataProvider's + */ +class AcceptanceTestCaseVisitor extends NodeVisitorAbstract +{ + /** + * @var array[] An array of arrays with test method names and optionally a data provider name + */ + private $tests = []; + + /** + * @var string Fully qualified test class name + */ + private $fqcn; + + /** + * Create a list of '@test' annotated methods in a test case + * file and see if single tests use data providers. + * + * @param Node $node + */ + public function enterNode(Node $node): void + { + if ($node instanceof Node\Stmt\Class_ + && !$node->isAnonymous() + ) { + // The test class full namespace + $this->fqcn = (string)$node->namespacedName; + } + + // A method is considered a test method, if: + if (// It is a method + $node instanceof \PhpParser\Node\Stmt\ClassMethod + // There is a method comment + && ($docComment = $node->getDocComment()) instanceof Doc + // The method is public + && $node->isPublic() + // The methods does not start with an "_" (eg. _before()) + && $node->name->name[0] !== '_' + ) { + // Found a test + $test = [ + 'methodName' => $node->name->name, + ]; + preg_match_all( + '/\s*\s@(?<annotations>[^\s.].*)\n/', + $docComment->getText(), + $matches + ); + foreach ($matches['annotations'] as $possibleDataProvider) { + // See if this test has a data provider attached + if (strpos($possibleDataProvider, 'dataProvider') === 0) { + $test['dataProvider'] = trim(ltrim($possibleDataProvider, 'dataProvider')); + } + } + $this->tests[] = $test; + } + } + + /** + * Return array of found tests and their data providers + * + * @return array + */ + public function getTests(): array + { + return $this->tests; + } + + /** + * Return Fully qualified class test name + * + * @return string + */ + public function getFqcn(): string + { + return $this->fqcn; + } +} + +$splitFunctionalTests = new SplitAcceptanceTests(); +exit($splitFunctionalTests->execute()); diff --git a/Build/Scripts/splitFunctionalTests.php b/Build/Scripts/splitFunctionalTests.php new file mode 100755 index 0000000000000000000000000000000000000000..322e8549b14b82e750fb69ac6cd25480a14e080e --- /dev/null +++ b/Build/Scripts/splitFunctionalTests.php @@ -0,0 +1,311 @@ +#!/usr/bin/env php +<?php +declare(strict_types=1); + +/* + * This file is part of the TYPO3 CMS project. + * + * It is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, either version 2 + * of the License, or any later version. + * + * For the full copyright and license information, please read the + * LICENSE.txt file that was distributed with this source code. + * + * The TYPO3 project - inspiring people to share! + */ + +use PhpParser\Comment\Doc; +use PhpParser\Node; +use PhpParser\NodeTraverser; +use PhpParser\NodeVisitor\NameResolver; +use PhpParser\NodeVisitorAbstract; +use PhpParser\ParserFactory; +use Symfony\Component\Console\Input\ArgvInput; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputDefinition; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\ConsoleOutput; +use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Finder\Finder; +use Symfony\Component\Finder\SplFileInfo; + +if (PHP_SAPI !== 'cli') { + die('Script must be called from command line.' . chr(10)); +} + +require __DIR__ . '/../../vendor/autoload.php'; + +/** + * This script is typically executed by runTests.sh. + * + * The script expects to be run from the core root: + * ./Build/Scripts/splitFunctionalTests.php <numberOfChunks> + * + * Verbose output with 8 chunks: + * ./Build/Scripts/splitFunctionalTests.php 8 -v + * + * It's purpose is to find all core functional tests and split them into + * pieces. In CI, there are for example 8 jobs for the functional tests and each + * picks one chunk of tests. This way, functional tests are run in parallel and + * thus reduce the overall runtime of the test suite. + * + * phpunit .xml config files including their specific set of tests are written to: + * Build/Scripts/FunctionalTests-Job-<counter>.xml + */ +class SplitFunctionalTests +{ + /** + * Main entry method + */ + public function execute() + { + $input = new ArgvInput($_SERVER['argv'], $this->getInputDefinition()); + $output = new ConsoleOutput(); + + // Number of chunks and verbose output + $numberOfChunks = (int)$input->getArgument('numberOfChunks'); + + if ($numberOfChunks < 1 || $numberOfChunks > 99) { + throw new \InvalidArgumentException( + 'Main argument "numberOfChunks" must be at least 1 and maximum 99', + 1528319388 + ); + } + + if ($input->hasParameterOption('-v', true) || $input->hasParameterOption('--verbose', true)) { + $output->setVerbosity(OutputInterface::VERBOSITY_VERBOSE); + } + + // Find functional test files + $testFiles = (new Finder()) + ->files() + ->in(__DIR__ . '/../../typo3/sysext/*/Tests/Functional') + ->name('/Test\.php$/') + ->sortByName() + ; + + $parser = (new ParserFactory())->create(ParserFactory::ONLY_PHP7); + $testStats = []; + foreach ($testFiles as $file) { + /** @var $file SplFileInfo */ + $relativeFilename = $file->getRealPath(); + preg_match('/.*typo3\/sysext\/(.*)$/', $relativeFilename, $matches); + $relativeFilename = '../typo3/sysext/' . $matches[1]; + + $ast = $parser->parse($file->getContents()); + $traverser = new NodeTraverser(); + $visitor = new NameResolver(); + $traverser->addVisitor($visitor); + $visitor = new FunctionalTestCaseVisitor(); + $traverser->addVisitor($visitor); + $traverser->traverse($ast); + + $fqcn = $visitor->getFqcn(); + $tests = $visitor->getTests(); + if (!empty($tests)) { + $testStats[$relativeFilename] = 0; + } + + foreach ($tests as $test) { + if (isset($test['dataProvider'])) { + // Test uses a data provider - get number of data sets + $dataProviderMethodName = $test['dataProvider']; + $methods = (new $fqcn())->$dataProviderMethodName(); + if ($methods instanceof Generator) { + $numberOfDataSets = iterator_count($methods); + } else { + $numberOfDataSets = count($methods); + } + $testStats[$relativeFilename] += $numberOfDataSets; + } else { + // Just a single test + $testStats[$relativeFilename] += 1; + } + } + } + + // Sort test files by number of tests, descending + arsort($testStats); + + $this->createPhpunitXmlHeader($numberOfChunks); + + $numberOfTestsPerChunk = []; + for ($i = 1; $i <= $numberOfChunks; $i++) { + $numberOfTestsPerChunk[$i] = 0; + } + + foreach ($testStats as $testFile => $numberOfTestsInFile) { + // Sort list of tests per chunk by number of tests, pick lowest as + // the target of this test file + asort($numberOfTestsPerChunk); + reset($numberOfTestsPerChunk); + $jobFileNumber = key($numberOfTestsPerChunk); + + $content = <<<EOF + <directory> + $testFile + </directory> + +EOF; + file_put_contents(__DIR__ . '/../' . 'FunctionalTests-Job-' . $jobFileNumber . '.xml', $content, FILE_APPEND); + + $numberOfTestsPerChunk[$jobFileNumber] = $numberOfTestsPerChunk[$jobFileNumber] + $numberOfTestsInFile; + } + + $this->createPhpunitXmlFooter($numberOfChunks); + + if ($output->isVerbose()) { + $output->writeln('Number of test files found: ' . count($testStats)); + $output->writeln('Number of tests found: ' . array_sum($testStats)); + $output->writeln('Number of chunks prepared: ' . $numberOfChunks); + ksort($numberOfTestsPerChunk); + foreach ($numberOfTestsPerChunk as $chunkNumber => $testNumber) { + $output->writeln('Number of tests in chunk ' . $chunkNumber . ': ' . $testNumber); + } + } + } + + /** + * Allowed script arguments + * + * @return InputDefinition argv input definition of symfony console + */ + private function getInputDefinition(): InputDefinition + { + return new InputDefinition([ + new InputArgument('numberOfChunks', InputArgument::REQUIRED, 'Number of chunks / jobs to create'), + new InputOption('--verbose', '-v', InputOption::VALUE_NONE, 'Enable verbose output'), + ]); + } + + /** + * "Header" part of a phpunit.xml functional config file + * + * @param int $numberOfChunks + */ + private function createPhpunitXmlHeader(int $numberOfChunks): void + { + $content = <<<EOF +<phpunit + backupGlobals="true" + bootstrap="../vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTestsBootstrap.php" + colors="true" + convertErrorsToExceptions="true" + convertWarningsToExceptions="true" + forceCoversAnnotation="false" + stopOnError="false" + stopOnFailure="false" + stopOnIncomplete="false" + stopOnSkipped="false" + verbose="false" + beStrictAboutTestsThatDoNotTestAnything="false" +> + <testsuites> + <testsuite name="Core tests"> + +EOF; + for ($i = 1; $i <= $numberOfChunks; $i++) { + file_put_contents(__DIR__ . '/../' . 'FunctionalTests-Job-' . $i . '.xml', $content); + } + } + + /** + * "Footer" part of a phpunit.xml functional config file + * + * @param int $numberOfChunks + */ + private function createPhpunitXmlFooter(int $numberOfChunks): void + { + $content = <<<EOF + </testsuite> + </testsuites> +</phpunit> + +EOF; + for ($i = 1; $i <= $numberOfChunks; $i++) { + file_put_contents(__DIR__ . '/../' . 'FunctionalTests-Job-' . $i . '.xml', $content, FILE_APPEND); + } + } +} + +/** + * nikic/php-parser node visitor to find test class namespace, + * count @test annotated methods and their possible @dataProvider's + */ +class FunctionalTestCaseVisitor extends NodeVisitorAbstract +{ + /** + * @var array[] An array of arrays with test method names and optionally a data provider name + */ + private $tests = []; + + /** + * @var string Fully qualified test class name + */ + private $fqcn; + + /** + * Create a list of '@test' annotated methods in a test case + * file and see if single tests use data providers. + * + * @param Node $node + */ + public function enterNode(Node $node): void + { + if ($node instanceof Node\Stmt\Class_ + && !$node->isAnonymous() + ) { + // The test class full namespace + $this->fqcn = (string)$node->namespacedName; + } + + if ($node instanceof Node\Stmt\ClassMethod + && ($docComment = $node->getDocComment()) instanceof Doc + ) { + preg_match_all( + '/\s*\s@(?<annotations>[^\s.].*)\n/', + $docComment->getText(), + $matches + ); + foreach ($matches['annotations'] as $possibleTest) { + if ($possibleTest === 'test') { + // Found a test + $test = [ + 'methodName' => $node->name->name, + ]; + foreach ($matches['annotations'] as $possibleDataProvider) { + // See if this test has a data provider attached + if (strpos($possibleDataProvider, 'dataProvider') === 0) { + $test['dataProvider'] = trim(ltrim($possibleDataProvider, 'dataProvider')); + } + } + $this->tests[] = $test; + } + } + } + } + + /** + * Return array of found tests and their data providers + * + * @return array + */ + public function getTests(): array + { + return $this->tests; + } + + /** + * Return Fully qualified class test name + * + * @return string + */ + public function getFqcn(): string + { + return $this->fqcn; + } +} + +$splitFunctionalTests = new SplitFunctionalTests(); +exit($splitFunctionalTests->execute()); diff --git a/Build/testing-docker/local/docker-compose.yml b/Build/testing-docker/local/docker-compose.yml index 1d10d3f07b1ca7f4e9468e2ad73abd12f076dc52..88c04b862c5801b29554d1c47eefb65d91ddbb7d 100644 --- a/Build/testing-docker/local/docker-compose.yml +++ b/Build/testing-docker/local/docker-compose.yml @@ -54,7 +54,7 @@ services: volumes: - ${CORE_ROOT}:${CORE_ROOT} working_dir: ${CORE_ROOT} - command: php vendor/typo3/testing-framework/Resources/Core/Build/Scripts/splitAcceptanceTests.php -v ${CHUNKS} + command: php Build/Scripts/splitAcceptanceTests.php -v ${CHUNKS} prepare_acceptance_backend_mariadb: image: alpine:3.8 @@ -819,7 +819,7 @@ services: volumes: - ${CORE_ROOT}:${CORE_ROOT} working_dir: ${CORE_ROOT} - command: php vendor/typo3/testing-framework/Resources/Core/Build/Scripts/splitFunctionalTests.php -v ${CHUNKS} + command: php Build/Scripts/splitFunctionalTests.php -v ${CHUNKS} prepare_functional_mariadb: image: alpine:3.8 @@ -863,7 +863,7 @@ services: php -v | grep '^PHP' if [ ${CHUNKS} -gt 0 ]; then echo \"Running chunk ${THISCHUNK}\" - COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\" + COMMAND=\"vendor/phpunit/phpunit/phpunit -c Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\" else COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\" fi @@ -917,7 +917,7 @@ services: php -v | grep '^PHP' if [ ${CHUNKS} -gt 0 ]; then echo \"Running chunk ${THISCHUNK}\" - COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\" + COMMAND=\"vendor/phpunit/phpunit/phpunit -c Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\" else COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} ${TEST_FILE}\" fi @@ -974,7 +974,7 @@ services: php -v | grep '^PHP' if [ ${CHUNKS} -gt 0 ]; then echo \"Running chunk ${THISCHUNK}\" - COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE}\" + COMMAND=\"vendor/phpunit/phpunit/phpunit -c Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE}\" else COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE}\" fi @@ -1028,7 +1028,7 @@ services: php -v | grep '^PHP' if [ ${CHUNKS} -gt 0 ]; then echo \"Running chunk ${THISCHUNK}\" - COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-postgres ${TEST_FILE}\" + COMMAND=\"vendor/phpunit/phpunit/phpunit -c Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-postgres ${TEST_FILE}\" else COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-postgres ${TEST_FILE}\" fi @@ -1070,7 +1070,7 @@ services: php -v | grep '^PHP' if [ ${CHUNKS} -gt 0 ]; then echo \"Running chunk ${THISCHUNK}\" - COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-sqlite ${TEST_FILE}\" + COMMAND=\"vendor/phpunit/phpunit/phpunit -c Build/FunctionalTests-Job-${THISCHUNK}.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-sqlite ${TEST_FILE}\" else COMMAND=\"vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-sqlite ${TEST_FILE}\" fi