From 22d6d4f16291fd765071d5d027c675210f66416e Mon Sep 17 00:00:00 2001
From: Andreas Fernandez <a.fernandez@scripting-base.de>
Date: Mon, 6 Apr 2020 14:56:57 +0200
Subject: [PATCH] [TASK] Restructure and fine-tune builds
This patch updates the Docker configuration to use mssql 2019 as mssql
2017 images don't get updates anymore.
Additionally to the mssql update, the shuffled chunking to 50 jobs is
reverted as it didn't bring any benefit but even more hassle as the
stages failed earlier and more often. Instead, the jobs are now grouped
by PHP version which allows smaller chunks and reduced the probability to
clog the build queue.
To workaround errors thrown by Gerrit on heavy load, the cherry-pick
command is now executed up to five times, with a short sleep between each
execution.
Last but not least we check whether there is actually a dangling Docker
container instead of hard killing non-existent containers which pollutes
our log.
Resolves: #90965
Releases: master, 9.5
Change-Id: Ied60297127b86c3375d310759bf441ad9e10aeed
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/64087
Tested-by: TYPO3com <noreply@typo3.com>
Tested-by: Susanne Moog <look@susi.dev>
Tested-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Tested-by: Andreas Fernandez <a.fernandez@scripting-base.de>
Reviewed-by: Susanne Moog <look@susi.dev>
Reviewed-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Reviewed-by: Andreas Fernandez <a.fernandez@scripting-base.de>
---
Build/Scripts/runTests.sh | 11 +-
.../src/main/java/core/AbstractCoreSpec.java | 59 +++----
.../src/main/java/core/NightlySpec.java | 149 ++++++++----------
.../java/core/utilities/LimitedChunker.java | 63 --------
.../src/test/java/utilities/JobFixture.java | 13 --
.../java/utilities/LimitedChunkerTest.java | 115 --------------
.../testing-docker/bamboo/docker-compose.yml | 10 +-
Build/testing-docker/local/docker-compose.yml | 77 +--------
8 files changed, 108 insertions(+), 389 deletions(-)
delete mode 100644 Build/bamboo/src/main/java/core/utilities/LimitedChunker.java
delete mode 100644 Build/bamboo/src/test/java/utilities/JobFixture.java
delete mode 100644 Build/bamboo/src/test/java/utilities/LimitedChunkerTest.java
diff --git a/Build/Scripts/runTests.sh b/Build/Scripts/runTests.sh
index a6c6c4d8e32b..084b3bbf4387 100755
--- a/Build/Scripts/runTests.sh
+++ b/Build/Scripts/runTests.sh
@@ -98,7 +98,6 @@ Options:
- mariadb (default): use mariadb
- mysql55: use MySQL 5.5 server
- mssql: use mssql microsoft sql server
- - mssql2017latest: use latest version of microsoft sql server 2017
- postgres: use postgres
- sqlite: use sqlite
@@ -413,14 +412,8 @@ case ${TEST_SUITE} in
;;
mssql)
[[ ! -z "$DATABASE_DRIVER" ]] && echo "Using driver: ${DATABASE_DRIVER}"
- docker-compose run prepare_functional_mssql2017cu17
- docker-compose run functional_mssql2017cu17
- SUITE_EXIT_CODE=$?
- ;;
- mssql2017latest)
- [[ ! -z "$DATABASE_DRIVER" ]] && echo "Using driver: ${DATABASE_DRIVER}"
- docker-compose run prepare_functional_mssql2017latest
- docker-compose run functional_mssql2017latest
+ docker-compose run prepare_functional_mssql2019latest
+ docker-compose run functional_mssql2019latest
SUITE_EXIT_CODE=$?
;;
postgres)
diff --git a/Build/bamboo/src/main/java/core/AbstractCoreSpec.java b/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
index 585a6a10ae23..d38e78b1131e 100644
--- a/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
+++ b/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
@@ -734,7 +734,7 @@ abstract class AbstractCoreSpec {
" -e typo3DatabaseHost=localhost \\\n" +
" -e typo3DatabasePort=1433 \\\n" +
" -e typo3DatabaseCharset=utf-8 \\\n" +
- " -e typo3DatabaseHost=mssql2017cu17 \\\n" +
+ " -e typo3DatabaseHost=mssql2019latest \\\n" +
" -e typo3TestingRedisHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_redis4_1 \\\n" +
" -e typo3TestingMemcachedHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_memcached1-5_1 \\\n" +
" --name ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc \\\n" +
@@ -801,7 +801,7 @@ abstract class AbstractCoreSpec {
" -e typo3DatabaseHost=localhost \\\n" +
" -e typo3DatabasePort=1433 \\\n" +
" -e typo3DatabaseCharset=utf-8 \\\n" +
- " -e typo3DatabaseHost=mssql2017cu17 \\\n" +
+ " -e typo3DatabaseHost=mssql2019latest \\\n" +
" -e typo3TestingRedisHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_redis4_1 \\\n" +
" -e typo3TestingMemcachedHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_memcached1-5_1 \\\n" +
" --name ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc \\\n" +
@@ -1603,35 +1603,25 @@ abstract class AbstractCoreSpec {
* Task definition to cherry pick a patch set from gerrit on top of cloned core
*/
private Task getTaskGitCherryPick(Boolean isSecurity) {
- if (isSecurity) {
- return new ScriptTask()
- .description("Gerrit cherry pick")
- .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
- .inlineBody(
- this.getScriptTaskBashInlineBody() +
- "CHANGEURL=${bamboo.changeUrl}\n" +
- "CHANGEURLID=${CHANGEURL#https://review.typo3.org/}\n" +
- "PATCHSET=${bamboo.patchset}\n" +
- "\n" +
- "if [[ $CHANGEURL ]]; then\n" +
- " gerrit-cherry-pick https://review.typo3.org/Teams/Security/TYPO3v4-Core $CHANGEURLID/$PATCHSET || exit 1\n" +
- "fi\n"
- );
- } else {
- return new ScriptTask()
- .description("Gerrit cherry pick")
- .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
- .inlineBody(
- this.getScriptTaskBashInlineBody() +
- "CHANGEURL=${bamboo.changeUrl}\n" +
- "CHANGEURLID=${CHANGEURL#https://review.typo3.org/}\n" +
- "PATCHSET=${bamboo.patchset}\n" +
- "\n" +
- "if [[ $CHANGEURL ]]; then\n" +
- " gerrit-cherry-pick https://review.typo3.org/Packages/TYPO3.CMS $CHANGEURLID/$PATCHSET || exit 1\n" +
- "fi\n"
- );
- }
+ String cherryPickRepository = isSecurity ? "Teams/Security/TYPO3v4-Core" : "Packages/TYPO3.CMS";
+
+ return new ScriptTask()
+ .description("Gerrit cherry pick")
+ .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+ .inlineBody(
+ this.getScriptTaskBashInlineBody() +
+ "CHANGEURL=${bamboo.changeUrl}\n" +
+ "CHANGEURLID=${CHANGEURL#https://review.typo3.org/}\n" +
+ "PATCHSET=${bamboo.patchset}\n" +
+ "\n" +
+ "if [[ $CHANGEURL ]]; then\n" +
+ " NEXT_WAIT_TIME=0\n" +
+ " until gerrit-cherry-pick https://review.typo3.org/" + cherryPickRepository + " $CHANGEURLID/$PATCHSET; do\n" +
+ " [[ $NEXT_WAIT_TIME -eq 5 ]] && exit 1\n" +
+ " sleep $(( NEXT_WAIT_TIME++ ))\n" +
+ " done\n" +
+ "fi\n"
+ );
}
/**
@@ -1645,8 +1635,11 @@ abstract class AbstractCoreSpec {
.inlineBody(
this.getScriptTaskBashInlineBody() +
"cd Build/testing-docker/bamboo\n" +
- "docker-compose down -v\n" +
- "docker rm -f ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc\n" +
+ "docker inspect -f '{{.State.Running}}' ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc > /dev/null 2>&1\n" +
+ "if [[ $? -eq 0 ]]; then\n" +
+ " docker-compose down -v\n" +
+ " docker rm -f ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc\n" +
+ "fi\n" +
"exit 0\n"
);
}
diff --git a/Build/bamboo/src/main/java/core/NightlySpec.java b/Build/bamboo/src/main/java/core/NightlySpec.java
index 7a261ac4ed00..cdf81ad644d1 100644
--- a/Build/bamboo/src/main/java/core/NightlySpec.java
+++ b/Build/bamboo/src/main/java/core/NightlySpec.java
@@ -28,11 +28,8 @@ import com.atlassian.bamboo.specs.api.builders.task.Task;
import com.atlassian.bamboo.specs.builders.notification.PlanCompletedNotification;
import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
import com.atlassian.bamboo.specs.util.BambooServer;
-import core.utilities.LimitedChunker;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
/**
* Core master nightly test plan.
@@ -60,9 +57,6 @@ public class NightlySpec extends AbstractCoreSpec {
private String[] sqLiteVersions = {"3.15", "3.20", "3.25", "3.30"};
private String[] postGreSqlVersions = {"9.3", "9.4", "9.5", "9.6", "10.11", "11.6", "12.1"};
- private int totalJobsPerStage = 50;
- private int mssqlJobsPerStage = 25;
-
/**
* Run main to publish plan on Bamboo
*/
@@ -85,51 +79,24 @@ public class NightlySpec extends AbstractCoreSpec {
* Returns full Plan definition
*/
Plan createPlan() {
- Stage stagePreparation = getPreparationStage();
-
- Stage stageIntegrity = getIntegrityStage();
- ArrayList<Job> jobs = new ArrayList<Job>();
- ArrayList<Job> mssqlJobs = new ArrayList<Job>();
- jobs.addAll(getUnitTestJobs());
- jobs.addAll(getCodeceptionMySqlJobs());
- jobs.addAll(getCodeceptionSqLiteJobs());
- jobs.addAll(getCodeceptionPgSqlJobs());
- jobs.addAll(getFunctionalMySqlJobs());
- jobs.addAll(getFunctionalMySqlPdoJobs());
- jobs.addAll(getFunctionalPGSqlJobs());
- jobs.addAll(getFunctionalSqliteJobs());
- mssqlJobs.addAll(getFunctionalMsSqlJobs());
- mssqlJobs.addAll(getFunctionalMsSqlPdoJobs());
-
- Collections.shuffle(jobs);
- Collections.shuffle(mssqlJobs);
-
- ArrayList<Stage> stages = new ArrayList<Stage>();
- stages.add(stagePreparation);
- stages.add(stageIntegrity);
-
- LimitedChunker<Job> chunker = new LimitedChunker<>(mssqlJobsPerStage, totalJobsPerStage);
- List<List<Job>> chunks = chunker.chunk(mssqlJobs, jobs);
-
- int totalNumberOfJobs = jobs.size() + mssqlJobs.size();
- int stageIndex = 0;
- for (List<Job> chunk : chunks) {
- int firstJobIndex = totalJobsPerStage * stageIndex + 1;
- int lastJobIndex = totalJobsPerStage * (stageIndex + 1);
- lastJobIndex = Math.min(lastJobIndex, totalNumberOfJobs);
-
- Collections.shuffle(chunk);
- Stage stage = new Stage("Stage " + (stageIndex + 1) + ", Jobs " + firstJobIndex + " - " + lastJobIndex);
- stage.jobs(chunk.toArray(new Job[chunk.size()]));
- stages.add(stage);
- System.out.println("Stage " + (stageIndex + 1) + " got " + chunk.size() + " Jobs");
- stageIndex++;
- }
+ ArrayList<Stage> stages = new ArrayList<>();
+ stages.add(getPreparationStage());
+ stages.add(getIntegrityStage());
+ stages.addAll(getUnitTestStages());
+ stages.addAll(getCodeceptionMySqlStages());
+ stages.addAll(getCodeceptionSqLiteStages());
+ stages.addAll(getCodeceptionPgSqlStages());
+ stages.addAll(getFunctionalMySqlStages());
+ stages.addAll(getFunctionalMySqlPdoStages());
+ stages.addAll(getFunctionalPGSqlStages());
+ stages.addAll(getFunctionalSqliteStages());
+ stages.addAll(getFunctionalMsSqlStages());
+ stages.addAll(getFunctionalMsSqlPdoStages());
// Compile plan
return new Plan(project(), planName, planKey).description("Execute TYPO3 core master nightly tests. Auto generated! See Build/bamboo of core git repository.")
.pluginConfigurations(this.getDefaultPlanPluginConfiguration())
- .stages(stages.toArray(new Stage[stages.size()]))
+ .stages(stages.toArray(new Stage[0]))
.linkedRepositories("github TYPO3 TYPO3.CMS")
.triggers(new ScheduledTrigger().name("Scheduled")
.description("once a day")
@@ -144,145 +111,165 @@ public class NightlySpec extends AbstractCoreSpec {
/**
* functional tests in all composer install stages, executed with DBMS Sqlite
*/
- private ArrayList<Job> getFunctionalSqliteJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalSqliteStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsSqlite(stageNumber, numberOfFunctionalSqliteJobs, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Functionals sqlite " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* functional tests in all composer install stages, executed with DBMS PostgreSql
*/
- private ArrayList<Job> getFunctionalPGSqlJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalPGSqlStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsPgsql(stageNumber, numberOfFunctionalPgsqlJobs, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Functionals pgsql " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* functional tests in all composer install stages, executed with DBMS MsSQL, driver is pdo_sqlsrv
*/
- private ArrayList<Job> getFunctionalMsSqlPdoJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalMsSqlPdoStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsMssqlWithDriverPdoSqlSrv(stageNumber, numberOfFunctionalMssqlJobs, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Functionals mssql pdo " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* functional tests in all composer install stages, executed with DBMS MsSQL, driver is sqlsrv
*/
- private ArrayList<Job> getFunctionalMsSqlJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalMsSqlStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsMssqlWithDriverSqlSrv(stageNumber, numberOfFunctionalMssqlJobs, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Functionals mssql " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* functional tests in all composer install stages, executed with DBMS MySQL, driver is mysqli
*/
- private ArrayList<Job> getFunctionalMySqlJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalMySqlStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerJob = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsMysqlWithDriverMySqli(stageNumber, numberOfFunctionalMysqlJobs, phpVersion, composerJob, false));
}
+ stages.add(new Stage("Functionals mysql " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* functional tests in all composer install stages, executed with DBMS MySQL, driver is pdo_mysql
*/
- private ArrayList<Job> getFunctionalMySqlPdoJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalMySqlPdoStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerJob = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsMysqlWithDriverPdoMysql(stageNumber, numberOfFunctionalMysqlJobs, phpVersion, composerJob, false));
}
+ stages.add(new Stage("Functionals mysql pdo " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* all tests run via codeception framework on MySql, for all php versions and each with composer max and min install
*/
- private ArrayList<Job> getCodeceptionMySqlJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getCodeceptionMySqlStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.add(this.getJobAcceptanceTestInstallMysql(stageNumber, phpVersion, composerTask, false));
jobs.addAll(this.getJobsAcceptanceTestsBackendMysql(stageNumber, numberOfAcceptanceTestJobs, phpVersion, composerTask, false));
jobs.addAll(this.getJobsAcceptanceTestsPageTreeMysql(stageNumber, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Acceptance mysql " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* all tests run via codeception framework on SqLite, for all php versions and each with composer max and min install
*/
- private ArrayList<Job> getCodeceptionSqLiteJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getCodeceptionSqLiteStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.add(this.getJobAcceptanceTestInstallSqlite(stageNumber, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Acceptance sqlite " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* all tests run via codeception framework on PostGreSql, for all php versions and each with composer max and min install
*/
- private ArrayList<Job> getCodeceptionPgSqlJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getCodeceptionPgSqlStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.add(this.getJobAcceptanceTestInstallPgsql(stageNumber, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Acceptance pgsql " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* all unit tests, for all php versions and each with composer max and min install
*/
- private ArrayList<Job> getUnitTestJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getUnitTestStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobUnitPhpRandom(stageNumber, numberOfUnitRandomOrderJobs, phpVersion, composerTask, false));
jobs.add(this.getJobUnitDeprecatedPhp(stageNumber, phpVersion, composerTask, false));
jobs.add(this.getJobUnitPhp(stageNumber, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Unit Tests " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
@@ -292,7 +279,7 @@ public class NightlySpec extends AbstractCoreSpec {
*/
private Stage getIntegrityStage() {
String phpVersionForIntegrityStage = phpVersions[0]; // the version is not very important, just use one (except for linting!)
- ArrayList<Job> jobs = new ArrayList<Job>();
+ ArrayList<Job> jobs = new ArrayList<>();
jobs.add(this.getJobIntegrationAnnotations(phpVersionForIntegrityStage, this.getTaskComposerInstall(phpVersionForIntegrityStage), false));
jobs.add(this.getJobCglCheckFullCore(phpVersionForIntegrityStage, this.getTaskComposerInstall(phpVersionForIntegrityStage), false));
jobs.add(this.getJobIntegrationPhpStan(phpVersionForIntegrityStage, this.getTaskComposerInstall(phpVersionForIntegrityStage), false));
@@ -305,15 +292,15 @@ public class NightlySpec extends AbstractCoreSpec {
for (String phpVersion : phpVersions) {
jobs.add(this.getJobLintPhp(phpVersion, false));
}
- return new Stage("Integrity").jobs(jobs.toArray(new Job[jobs.size()]));
+ return new Stage("Integrity").jobs(jobs.toArray(new Job[0]));
}
/**
* preparation stage - this will only define labels for later communication of test results
*/
private Stage getPreparationStage() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ ArrayList<Job> jobs = new ArrayList<>();
jobs.add(this.getJobBuildLabels());
- return new Stage("Preparation").jobs(jobs.toArray(new Job[jobs.size()]));
+ return new Stage("Preparation").jobs(jobs.toArray(new Job[0]));
}
}
diff --git a/Build/bamboo/src/main/java/core/utilities/LimitedChunker.java b/Build/bamboo/src/main/java/core/utilities/LimitedChunker.java
deleted file mode 100644
index 6ba171dc1000..000000000000
--- a/Build/bamboo/src/main/java/core/utilities/LimitedChunker.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package core.utilities;
-
-import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-/**
- * Chunk two lists into chunks of a given size
- * where only `numLimitedJobsPerChunk` elements from one list are in each chunk
- *
- * @param <T>
- */
-public class LimitedChunker<T> {
-
- private final int numLimitedElementsPerChunk;
- private final int totalChunkSize;
-
- public LimitedChunker(int numLimitedElementsPerChunk, int totalChunkSize) {
- this.numLimitedElementsPerChunk = numLimitedElementsPerChunk;
- this.totalChunkSize = totalChunkSize;
-
- if (numLimitedElementsPerChunk < 1) {
- throw new IllegalArgumentException("Number of limited Elements per Chunk must be greater than one");
- }
- if (totalChunkSize < 1) {
- throw new IllegalArgumentException("Total chunk size must be greater than one");
- }
- if (totalChunkSize < numLimitedElementsPerChunk) {
- throw new IllegalArgumentException("Number of limited elements must not be greater than total chunk size");
- }
- }
-
- public List<List<T>> chunk(List<T> limitedElements, List<T> normalElements) {
- // chunk the limited elements in sets of limitedChunkSize
- LinkedList<List<T>> chunks = new LinkedList<>(prepareChunks(limitedElements, numLimitedElementsPerChunk));
-
- // fill up these chunks with normal elements
- Iterator<T> normalJobIterator = normalElements.iterator();
- for (List<T> chunk : chunks) {
- fillChunkToSize(chunk, totalChunkSize, normalJobIterator);
- }
- // add chunks for all remaining normal elements, if any
- while (normalJobIterator.hasNext()) {
- List<T> chunk = new ArrayList<>(totalChunkSize);
- fillChunkToSize(chunk, totalChunkSize, normalJobIterator);
- chunks.add(chunk);
- }
-
- return chunks;
- }
-
- private void fillChunkToSize(List<T> chunk, int totalChunkSize, Iterator<T> remainingElementsIterator) {
- while (remainingElementsIterator.hasNext() && chunk.size() < totalChunkSize) {
- chunk.add(remainingElementsIterator.next());
- }
- }
-
- private Collection<List<T>> prepareChunks(List<T> inputList, int chunkSize) {
- AtomicInteger counter = new AtomicInteger();
- return inputList.stream().collect(Collectors.groupingBy(it -> counter.getAndIncrement() / chunkSize)).values();
- }
-
-}
diff --git a/Build/bamboo/src/test/java/utilities/JobFixture.java b/Build/bamboo/src/test/java/utilities/JobFixture.java
deleted file mode 100644
index 8f03fffd88a6..000000000000
--- a/Build/bamboo/src/test/java/utilities/JobFixture.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package utilities;
-
-public class JobFixture {
- private final boolean isLimited;
-
- public JobFixture(boolean isLimited) {
- this.isLimited = isLimited;
- }
-
- public boolean isLimited(){
- return isLimited;
- }
-}
diff --git a/Build/bamboo/src/test/java/utilities/LimitedChunkerTest.java b/Build/bamboo/src/test/java/utilities/LimitedChunkerTest.java
deleted file mode 100644
index b3af82f6464c..000000000000
--- a/Build/bamboo/src/test/java/utilities/LimitedChunkerTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-package utilities;
-
-/*
- * This file is part of the TYPO3 CMS project.
- *
- * It is free software; you can redistribute it and/or modify it under
- * the terms of the GNU General Public License, either version 2
- * of the License, or any later version.
- *
- * For the full copyright and license information, please read the
- * LICENSE.txt file that was distributed with this source code.
- *
- * The TYPO3 project - inspiring people to share!
- */
-
-import core.utilities.LimitedChunker;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-@RunWith(Parameterized.class)
-public class LimitedChunkerTest {
- private final int numLimitedJobs;
- private final int numNormalJobs;
- private final int limitedChunkSize;
- private final int totalChunkSize;
- private int expectedNumberOfChunks;
-
- @Test(expected = IllegalArgumentException.class)
- public void throwsOnZeroLimitedElementsPerChunk() {
- LimitedChunker<JobFixture> chunker = new LimitedChunker<>(0, 25);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void throwsOnZeroTotalChunkSize() {
- LimitedChunker<JobFixture> chunker = new LimitedChunker<>(25, 0);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void throwsOnMoreLimitedThanChunkSize() {
- LimitedChunker<JobFixture> chunker = new LimitedChunker<>(25, 12);
- }
-
- @Test
- public void returnsEmptyChunkListWithEmptyInput() {
- LimitedChunker<JobFixture> chunker = new LimitedChunker<>(12, 25);
-
- ArrayList<JobFixture> jobs = new ArrayList<>(0);
- ArrayList<JobFixture> limitedJobs = new ArrayList<>(0);
-
- List<List<JobFixture>> chunks = chunker.chunk(limitedJobs, jobs);
-
- assert chunks.size() == 0;
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> data() {
- return Arrays.asList(
- new Integer[]{25, 25, 25, 50, 1},
- new Integer[]{25, 0, 25, 25, 1},
- new Integer[]{100, 100, 25, 50, 4},
- new Integer[]{110, 100, 25, 50, 5},
- new Integer[]{100, 110, 25, 50, 5}
- );
- }
-
- public LimitedChunkerTest(int numLimitedJobs, int numNormalJobs, int limitedChunkSize, int totalChunkSize, int expectedNumberOfChunks) {
- this.numLimitedJobs = numLimitedJobs;
- this.numNormalJobs = numNormalJobs;
- this.limitedChunkSize = limitedChunkSize;
- this.totalChunkSize = totalChunkSize;
- this.expectedNumberOfChunks = expectedNumberOfChunks;
- }
-
- @Test
- public void chunksCorrectly() {
- ArrayList<JobFixture> limitedJobs = getJobs(numLimitedJobs, true);
- ArrayList<JobFixture> normalJobs = getJobs(numNormalJobs, false);
-
- List<List<JobFixture>> chunks = new LimitedChunker<JobFixture>(limitedChunkSize, totalChunkSize).chunk(limitedJobs, normalJobs);
-
- assert chunks.size() == expectedNumberOfChunks;
-
- int actualNumberOfJobs = chunks.stream().map(List::size).reduce(0, Integer::sum);
- assert actualNumberOfJobs == numLimitedJobs + numNormalJobs;
-
-
- int chunkIndex = 0;
- for (List<JobFixture> chunk : chunks) {
- long numLimited = chunk.stream().filter(JobFixture::isLimited).count();
- assert(numLimited <= limitedChunkSize);
- if (chunkIndex < chunks.size() - 1) {
- // all chunks must be full
- assert(chunk.size() == totalChunkSize);
- } else {
- //except the last one
- assert(chunk.size() <= totalChunkSize);
- }
- chunkIndex++;
- }
- }
-
- private ArrayList<JobFixture> getJobs(int numLimitedJobs, boolean isLimited) {
- ArrayList<JobFixture> jobs = new ArrayList<>(numLimitedJobs);
- for (int i = 0; i < numLimitedJobs; i++) {
- jobs.add(new JobFixture(isLimited));
- }
- return jobs;
- }
-}
diff --git a/Build/testing-docker/bamboo/docker-compose.yml b/Build/testing-docker/bamboo/docker-compose.yml
index fdcc88d5fa2a..9744ef28b231 100644
--- a/Build/testing-docker/bamboo/docker-compose.yml
+++ b/Build/testing-docker/bamboo/docker-compose.yml
@@ -23,13 +23,13 @@ services:
- /var/lib/postgresql/data:rw,noexec,nosuid
networks:
- test
- mssql2017cu17:
- image: mcr.microsoft.com/mssql/server:2017-CU17-ubuntu
+
+ mssql2019latest:
+ image: mcr.microsoft.com/mssql/server:2019-latest
environment:
ACCEPT_EULA: Y
SA_PASSWORD: Test1234!
MSSQL_PID: Developer
- MSSQL_MEMORY_LIMIT_MB: 3584
tmpfs:
- /var/lib/mssql:rw,noexec,nosuid
volumes:
@@ -159,7 +159,7 @@ services:
start_dependencies_functional_mssql:
image: alpine:3.8
links:
- - mssql2017cu17
+ - mssql2019latest
- redis4
- memcached1-5
networks:
@@ -168,7 +168,7 @@ services:
/bin/sh -c "
echo Waiting for database start
COUNT=0
- while ! nc -z mssql2017cu17 1433; do
+ while ! nc -z mssql2019latest 1433; do
if [ "$${COUNT}" -ge "120" ]; then
echo Database did not come up
exit 1
diff --git a/Build/testing-docker/local/docker-compose.yml b/Build/testing-docker/local/docker-compose.yml
index a401c67a81ff..27dfc995d083 100644
--- a/Build/testing-docker/local/docker-compose.yml
+++ b/Build/testing-docker/local/docker-compose.yml
@@ -17,23 +17,12 @@ services:
tmpfs:
- /var/lib/mysql/:rw,noexec,nosuid
- mssql2017cu17:
- image: mcr.microsoft.com/mssql/server:2017-CU17-ubuntu
+ mssql2019latest:
+ image: mcr.microsoft.com/mssql/server:2019-latest
environment:
ACCEPT_EULA: Y
SA_PASSWORD: "Test1234!"
MSSQL_PID: Developer
- MSSQL_MEMORY_LIMIT_MB: 3584
- tmpfs:
- - /var/lib/mssql:rw,noexec,nosuid
-
- mssql2017latest:
- image: mcr.microsoft.com/mssql/server:2017-latest-ubuntu
- environment:
- ACCEPT_EULA: Y
- SA_PASSWORD: "Test1234!"
- MSSQL_PID: Developer
- MSSQL_MEMORY_LIMIT_MB: 3584
tmpfs:
- /var/lib/mssql:rw,noexec,nosuid
@@ -642,62 +631,10 @@ services:
fi
"
- prepare_functional_mssql2017cu17:
- image: alpine:3.8
- links:
- - mssql2017cu17
- - redis4
- - memcached1-5
- command: >
- /bin/sh -c "
- if [ ${SCRIPT_VERBOSE} -eq 1 ]; then
- set -x
- fi
- echo Waiting for database start...;
- while ! nc -z mssql2017cu17 1433; do
- sleep 1;
- done;
- echo Database is up;
- "
- functional_mssql2017cu17:
- image: typo3gmbh/${DOCKER_PHP_IMAGE}:latest
- user: ${HOST_UID}
- volumes:
- - ${CORE_ROOT}:${CORE_ROOT}
- - ${HOST_HOME}:${HOST_HOME}
- - /etc/passwd:/etc/passwd:ro
- - /etc/group:/etc/group:ro
- environment:
- typo3DatabaseDriver: "${DATABASE_DRIVER:-sqlsrv}"
- typo3DatabaseName: func
- typo3DatabasePassword: "Test1234!"
- typo3DatabaseUsername: SA
- typo3DatabasePort: 1433
- typo3DatabaseCharset: utf-8
- typo3DatabaseHost: mssql2017cu17
- typo3TestingRedisHost: redis4
- typo3TestingMemcachedHost: memcached1-5
- working_dir: ${CORE_ROOT}
- command: >
- /bin/sh -c "
- if [ ${SCRIPT_VERBOSE} -eq 1 ]; then
- set -x
- fi
- php -v | grep '^PHP'
- if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
- php -n -c /etc/php/cli-no-xdebug/php.ini \
- vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE};
- else
- DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
- XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
- vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE};
- fi
- "
-
- prepare_functional_mssql2017latest:
+ prepare_functional_mssql2019latest:
image: alpine:3.8
links:
- - mssql2017latest
+ - mssql2019latest
- redis4
- memcached1-5
command: >
@@ -706,13 +643,13 @@ services:
set -x
fi
echo Waiting for database start...;
- while ! nc -z mssql2017latest 1433; do
+ while ! nc -z mssql2019latest 1433; do
sleep 1;
done;
echo Database is up;
"
- functional_mssql2017latest:
+ functional_mssql2019latest:
image: typo3gmbh/${DOCKER_PHP_IMAGE}:latest
user: ${HOST_UID}
volumes:
@@ -727,7 +664,7 @@ services:
typo3DatabaseUsername: SA
typo3DatabasePort: 1433
typo3DatabaseCharset: utf-8
- typo3DatabaseHost: mssql2017latest
+ typo3DatabaseHost: mssql2019latest
typo3TestingRedisHost: redis4
typo3TestingMemcachedHost: memcached1-5
working_dir: ${CORE_ROOT}
--
GitLab