diff --git a/Build/Scripts/runTests.sh b/Build/Scripts/runTests.sh
index a6c6c4d8e32b877251de6bc0e28b2a9bd2c1c7e0..084b3bbf438790f08f5b2b687171e3c99f875b25 100755
--- a/Build/Scripts/runTests.sh
+++ b/Build/Scripts/runTests.sh
@@ -98,7 +98,6 @@ Options:
- mariadb (default): use mariadb
- mysql55: use MySQL 5.5 server
- mssql: use mssql microsoft sql server
- - mssql2017latest: use latest version of microsoft sql server 2017
- postgres: use postgres
- sqlite: use sqlite
@@ -413,14 +412,8 @@ case ${TEST_SUITE} in
;;
mssql)
[[ ! -z "$DATABASE_DRIVER" ]] && echo "Using driver: ${DATABASE_DRIVER}"
- docker-compose run prepare_functional_mssql2017cu17
- docker-compose run functional_mssql2017cu17
- SUITE_EXIT_CODE=$?
- ;;
- mssql2017latest)
- [[ ! -z "$DATABASE_DRIVER" ]] && echo "Using driver: ${DATABASE_DRIVER}"
- docker-compose run prepare_functional_mssql2017latest
- docker-compose run functional_mssql2017latest
+ docker-compose run prepare_functional_mssql2019latest
+ docker-compose run functional_mssql2019latest
SUITE_EXIT_CODE=$?
;;
postgres)
diff --git a/Build/bamboo/src/main/java/core/AbstractCoreSpec.java b/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
index 585a6a10ae2394df87d6ceccca759ed787ea75e8..d38e78b1131ec4e06d74044f7f0d39f1f68f4718 100644
--- a/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
+++ b/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
@@ -734,7 +734,7 @@ abstract class AbstractCoreSpec {
" -e typo3DatabaseHost=localhost \\\n" +
" -e typo3DatabasePort=1433 \\\n" +
" -e typo3DatabaseCharset=utf-8 \\\n" +
- " -e typo3DatabaseHost=mssql2017cu17 \\\n" +
+ " -e typo3DatabaseHost=mssql2019latest \\\n" +
" -e typo3TestingRedisHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_redis4_1 \\\n" +
" -e typo3TestingMemcachedHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_memcached1-5_1 \\\n" +
" --name ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc \\\n" +
@@ -801,7 +801,7 @@ abstract class AbstractCoreSpec {
" -e typo3DatabaseHost=localhost \\\n" +
" -e typo3DatabasePort=1433 \\\n" +
" -e typo3DatabaseCharset=utf-8 \\\n" +
- " -e typo3DatabaseHost=mssql2017cu17 \\\n" +
+ " -e typo3DatabaseHost=mssql2019latest \\\n" +
" -e typo3TestingRedisHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_redis4_1 \\\n" +
" -e typo3TestingMemcachedHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_memcached1-5_1 \\\n" +
" --name ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc \\\n" +
@@ -1603,35 +1603,25 @@ abstract class AbstractCoreSpec {
* Task definition to cherry pick a patch set from gerrit on top of cloned core
*/
private Task getTaskGitCherryPick(Boolean isSecurity) {
- if (isSecurity) {
- return new ScriptTask()
- .description("Gerrit cherry pick")
- .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
- .inlineBody(
- this.getScriptTaskBashInlineBody() +
- "CHANGEURL=${bamboo.changeUrl}\n" +
- "CHANGEURLID=${CHANGEURL#https://review.typo3.org/}\n" +
- "PATCHSET=${bamboo.patchset}\n" +
- "\n" +
- "if [[ $CHANGEURL ]]; then\n" +
- " gerrit-cherry-pick https://review.typo3.org/Teams/Security/TYPO3v4-Core $CHANGEURLID/$PATCHSET || exit 1\n" +
- "fi\n"
- );
- } else {
- return new ScriptTask()
- .description("Gerrit cherry pick")
- .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
- .inlineBody(
- this.getScriptTaskBashInlineBody() +
- "CHANGEURL=${bamboo.changeUrl}\n" +
- "CHANGEURLID=${CHANGEURL#https://review.typo3.org/}\n" +
- "PATCHSET=${bamboo.patchset}\n" +
- "\n" +
- "if [[ $CHANGEURL ]]; then\n" +
- " gerrit-cherry-pick https://review.typo3.org/Packages/TYPO3.CMS $CHANGEURLID/$PATCHSET || exit 1\n" +
- "fi\n"
- );
- }
+ String cherryPickRepository = isSecurity ? "Teams/Security/TYPO3v4-Core" : "Packages/TYPO3.CMS";
+
+ return new ScriptTask()
+ .description("Gerrit cherry pick")
+ .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+ .inlineBody(
+ this.getScriptTaskBashInlineBody() +
+ "CHANGEURL=${bamboo.changeUrl}\n" +
+ "CHANGEURLID=${CHANGEURL#https://review.typo3.org/}\n" +
+ "PATCHSET=${bamboo.patchset}\n" +
+ "\n" +
+ "if [[ $CHANGEURL ]]; then\n" +
+ " NEXT_WAIT_TIME=0\n" +
+ " until gerrit-cherry-pick https://review.typo3.org/" + cherryPickRepository + " $CHANGEURLID/$PATCHSET; do\n" +
+ " [[ $NEXT_WAIT_TIME -eq 5 ]] && exit 1\n" +
+ " sleep $(( NEXT_WAIT_TIME++ ))\n" +
+ " done\n" +
+ "fi\n"
+ );
}
/**
@@ -1645,8 +1635,11 @@ abstract class AbstractCoreSpec {
.inlineBody(
this.getScriptTaskBashInlineBody() +
"cd Build/testing-docker/bamboo\n" +
- "docker-compose down -v\n" +
- "docker rm -f ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc\n" +
+ "docker inspect -f '{{.State.Running}}' ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc > /dev/null 2>&1\n" +
+ "if [[ $? -eq 0 ]]; then\n" +
+ " docker-compose down -v\n" +
+ " docker rm -f ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc\n" +
+ "fi\n" +
"exit 0\n"
);
}
diff --git a/Build/bamboo/src/main/java/core/NightlySpec.java b/Build/bamboo/src/main/java/core/NightlySpec.java
index 7a261ac4ed00957ffd20d981b7ccd380a7198645..cdf81ad644d1433b8e056b78f429d483a2f32788 100644
--- a/Build/bamboo/src/main/java/core/NightlySpec.java
+++ b/Build/bamboo/src/main/java/core/NightlySpec.java
@@ -28,11 +28,8 @@ import com.atlassian.bamboo.specs.api.builders.task.Task;
import com.atlassian.bamboo.specs.builders.notification.PlanCompletedNotification;
import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
import com.atlassian.bamboo.specs.util.BambooServer;
-import core.utilities.LimitedChunker;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
/**
* Core master nightly test plan.
@@ -60,9 +57,6 @@ public class NightlySpec extends AbstractCoreSpec {
private String[] sqLiteVersions = {"3.15", "3.20", "3.25", "3.30"};
private String[] postGreSqlVersions = {"9.3", "9.4", "9.5", "9.6", "10.11", "11.6", "12.1"};
- private int totalJobsPerStage = 50;
- private int mssqlJobsPerStage = 25;
-
/**
* Run main to publish plan on Bamboo
*/
@@ -85,51 +79,24 @@ public class NightlySpec extends AbstractCoreSpec {
* Returns full Plan definition
*/
Plan createPlan() {
- Stage stagePreparation = getPreparationStage();
-
- Stage stageIntegrity = getIntegrityStage();
- ArrayList<Job> jobs = new ArrayList<Job>();
- ArrayList<Job> mssqlJobs = new ArrayList<Job>();
- jobs.addAll(getUnitTestJobs());
- jobs.addAll(getCodeceptionMySqlJobs());
- jobs.addAll(getCodeceptionSqLiteJobs());
- jobs.addAll(getCodeceptionPgSqlJobs());
- jobs.addAll(getFunctionalMySqlJobs());
- jobs.addAll(getFunctionalMySqlPdoJobs());
- jobs.addAll(getFunctionalPGSqlJobs());
- jobs.addAll(getFunctionalSqliteJobs());
- mssqlJobs.addAll(getFunctionalMsSqlJobs());
- mssqlJobs.addAll(getFunctionalMsSqlPdoJobs());
-
- Collections.shuffle(jobs);
- Collections.shuffle(mssqlJobs);
-
- ArrayList<Stage> stages = new ArrayList<Stage>();
- stages.add(stagePreparation);
- stages.add(stageIntegrity);
-
- LimitedChunker<Job> chunker = new LimitedChunker<>(mssqlJobsPerStage, totalJobsPerStage);
- List<List<Job>> chunks = chunker.chunk(mssqlJobs, jobs);
-
- int totalNumberOfJobs = jobs.size() + mssqlJobs.size();
- int stageIndex = 0;
- for (List<Job> chunk : chunks) {
- int firstJobIndex = totalJobsPerStage * stageIndex + 1;
- int lastJobIndex = totalJobsPerStage * (stageIndex + 1);
- lastJobIndex = Math.min(lastJobIndex, totalNumberOfJobs);
-
- Collections.shuffle(chunk);
- Stage stage = new Stage("Stage " + (stageIndex + 1) + ", Jobs " + firstJobIndex + " - " + lastJobIndex);
- stage.jobs(chunk.toArray(new Job[chunk.size()]));
- stages.add(stage);
- System.out.println("Stage " + (stageIndex + 1) + " got " + chunk.size() + " Jobs");
- stageIndex++;
- }
+ ArrayList<Stage> stages = new ArrayList<>();
+ stages.add(getPreparationStage());
+ stages.add(getIntegrityStage());
+ stages.addAll(getUnitTestStages());
+ stages.addAll(getCodeceptionMySqlStages());
+ stages.addAll(getCodeceptionSqLiteStages());
+ stages.addAll(getCodeceptionPgSqlStages());
+ stages.addAll(getFunctionalMySqlStages());
+ stages.addAll(getFunctionalMySqlPdoStages());
+ stages.addAll(getFunctionalPGSqlStages());
+ stages.addAll(getFunctionalSqliteStages());
+ stages.addAll(getFunctionalMsSqlStages());
+ stages.addAll(getFunctionalMsSqlPdoStages());
// Compile plan
return new Plan(project(), planName, planKey).description("Execute TYPO3 core master nightly tests. Auto generated! See Build/bamboo of core git repository.")
.pluginConfigurations(this.getDefaultPlanPluginConfiguration())
- .stages(stages.toArray(new Stage[stages.size()]))
+ .stages(stages.toArray(new Stage[0]))
.linkedRepositories("github TYPO3 TYPO3.CMS")
.triggers(new ScheduledTrigger().name("Scheduled")
.description("once a day")
@@ -144,145 +111,165 @@ public class NightlySpec extends AbstractCoreSpec {
/**
* functional tests in all composer install stages, executed with DBMS Sqlite
*/
- private ArrayList<Job> getFunctionalSqliteJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalSqliteStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsSqlite(stageNumber, numberOfFunctionalSqliteJobs, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Functionals sqlite " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* functional tests in all composer install stages, executed with DBMS PostgreSql
*/
- private ArrayList<Job> getFunctionalPGSqlJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalPGSqlStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsPgsql(stageNumber, numberOfFunctionalPgsqlJobs, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Functionals pgsql " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* functional tests in all composer install stages, executed with DBMS MsSQL, driver is pdo_sqlsrv
*/
- private ArrayList<Job> getFunctionalMsSqlPdoJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalMsSqlPdoStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsMssqlWithDriverPdoSqlSrv(stageNumber, numberOfFunctionalMssqlJobs, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Functionals mssql pdo " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* functional tests in all composer install stages, executed with DBMS MsSQL, driver is sqlsrv
*/
- private ArrayList<Job> getFunctionalMsSqlJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalMsSqlStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsMssqlWithDriverSqlSrv(stageNumber, numberOfFunctionalMssqlJobs, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Functionals mssql " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* functional tests in all composer install stages, executed with DBMS MySQL, driver is mysqli
*/
- private ArrayList<Job> getFunctionalMySqlJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalMySqlStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerJob = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsMysqlWithDriverMySqli(stageNumber, numberOfFunctionalMysqlJobs, phpVersion, composerJob, false));
}
+ stages.add(new Stage("Functionals mysql " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* functional tests in all composer install stages, executed with DBMS MySQL, driver is pdo_mysql
*/
- private ArrayList<Job> getFunctionalMySqlPdoJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getFunctionalMySqlPdoStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerJob = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobsFunctionalTestsMysqlWithDriverPdoMysql(stageNumber, numberOfFunctionalMysqlJobs, phpVersion, composerJob, false));
}
+ stages.add(new Stage("Functionals mysql pdo " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* all tests run via codeception framework on MySql, for all php versions and each with composer max and min install
*/
- private ArrayList<Job> getCodeceptionMySqlJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getCodeceptionMySqlStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.add(this.getJobAcceptanceTestInstallMysql(stageNumber, phpVersion, composerTask, false));
jobs.addAll(this.getJobsAcceptanceTestsBackendMysql(stageNumber, numberOfAcceptanceTestJobs, phpVersion, composerTask, false));
jobs.addAll(this.getJobsAcceptanceTestsPageTreeMysql(stageNumber, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Acceptance mysql " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* all tests run via codeception framework on SqLite, for all php versions and each with composer max and min install
*/
- private ArrayList<Job> getCodeceptionSqLiteJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getCodeceptionSqLiteStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.add(this.getJobAcceptanceTestInstallSqlite(stageNumber, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Acceptance sqlite " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* all tests run via codeception framework on PostGreSql, for all php versions and each with composer max and min install
*/
- private ArrayList<Job> getCodeceptionPgSqlJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getCodeceptionPgSqlStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.add(this.getJobAcceptanceTestInstallPgsql(stageNumber, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Acceptance pgsql " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
* all unit tests, for all php versions and each with composer max and min install
*/
- private ArrayList<Job> getUnitTestJobs() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ private ArrayList<Stage> getUnitTestStages() {
+ ArrayList<Stage> stages = new ArrayList<>();
for (String phpVersion : phpVersions) {
+ ArrayList<Job> jobs = new ArrayList<>();
for (int stageNumber = 0; stageNumber <= 2; stageNumber++) {
Task composerTask = getComposerTaskByStageNumber(phpVersion, stageNumber);
jobs.addAll(this.getJobUnitPhpRandom(stageNumber, numberOfUnitRandomOrderJobs, phpVersion, composerTask, false));
jobs.add(this.getJobUnitDeprecatedPhp(stageNumber, phpVersion, composerTask, false));
jobs.add(this.getJobUnitPhp(stageNumber, phpVersion, composerTask, false));
}
+ stages.add(new Stage("Unit Tests " + phpVersion).jobs(jobs.toArray(new Job[0])));
}
- return jobs;
+ return stages;
}
/**
@@ -292,7 +279,7 @@ public class NightlySpec extends AbstractCoreSpec {
*/
private Stage getIntegrityStage() {
String phpVersionForIntegrityStage = phpVersions[0]; // the version is not very important, just use one (except for linting!)
- ArrayList<Job> jobs = new ArrayList<Job>();
+ ArrayList<Job> jobs = new ArrayList<>();
jobs.add(this.getJobIntegrationAnnotations(phpVersionForIntegrityStage, this.getTaskComposerInstall(phpVersionForIntegrityStage), false));
jobs.add(this.getJobCglCheckFullCore(phpVersionForIntegrityStage, this.getTaskComposerInstall(phpVersionForIntegrityStage), false));
jobs.add(this.getJobIntegrationPhpStan(phpVersionForIntegrityStage, this.getTaskComposerInstall(phpVersionForIntegrityStage), false));
@@ -305,15 +292,15 @@ public class NightlySpec extends AbstractCoreSpec {
for (String phpVersion : phpVersions) {
jobs.add(this.getJobLintPhp(phpVersion, false));
}
- return new Stage("Integrity").jobs(jobs.toArray(new Job[jobs.size()]));
+ return new Stage("Integrity").jobs(jobs.toArray(new Job[0]));
}
/**
* preparation stage - this will only define labels for later communication of test results
*/
private Stage getPreparationStage() {
- ArrayList<Job> jobs = new ArrayList<Job>();
+ ArrayList<Job> jobs = new ArrayList<>();
jobs.add(this.getJobBuildLabels());
- return new Stage("Preparation").jobs(jobs.toArray(new Job[jobs.size()]));
+ return new Stage("Preparation").jobs(jobs.toArray(new Job[0]));
}
}
diff --git a/Build/bamboo/src/main/java/core/utilities/LimitedChunker.java b/Build/bamboo/src/main/java/core/utilities/LimitedChunker.java
deleted file mode 100644
index 6ba171dc100099ebd1266a5d9c18f2668e71a8a4..0000000000000000000000000000000000000000
--- a/Build/bamboo/src/main/java/core/utilities/LimitedChunker.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package core.utilities;
-
-import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-/**
- * Chunk two lists into chunks of a given size
- * where only `numLimitedJobsPerChunk` elements from one list are in each chunk
- *
- * @param <T>
- */
-public class LimitedChunker<T> {
-
- private final int numLimitedElementsPerChunk;
- private final int totalChunkSize;
-
- public LimitedChunker(int numLimitedElementsPerChunk, int totalChunkSize) {
- this.numLimitedElementsPerChunk = numLimitedElementsPerChunk;
- this.totalChunkSize = totalChunkSize;
-
- if (numLimitedElementsPerChunk < 1) {
- throw new IllegalArgumentException("Number of limited Elements per Chunk must be greater than one");
- }
- if (totalChunkSize < 1) {
- throw new IllegalArgumentException("Total chunk size must be greater than one");
- }
- if (totalChunkSize < numLimitedElementsPerChunk) {
- throw new IllegalArgumentException("Number of limited elements must not be greater than total chunk size");
- }
- }
-
- public List<List<T>> chunk(List<T> limitedElements, List<T> normalElements) {
- // chunk the limited elements in sets of limitedChunkSize
- LinkedList<List<T>> chunks = new LinkedList<>(prepareChunks(limitedElements, numLimitedElementsPerChunk));
-
- // fill up these chunks with normal elements
- Iterator<T> normalJobIterator = normalElements.iterator();
- for (List<T> chunk : chunks) {
- fillChunkToSize(chunk, totalChunkSize, normalJobIterator);
- }
- // add chunks for all remaining normal elements, if any
- while (normalJobIterator.hasNext()) {
- List<T> chunk = new ArrayList<>(totalChunkSize);
- fillChunkToSize(chunk, totalChunkSize, normalJobIterator);
- chunks.add(chunk);
- }
-
- return chunks;
- }
-
- private void fillChunkToSize(List<T> chunk, int totalChunkSize, Iterator<T> remainingElementsIterator) {
- while (remainingElementsIterator.hasNext() && chunk.size() < totalChunkSize) {
- chunk.add(remainingElementsIterator.next());
- }
- }
-
- private Collection<List<T>> prepareChunks(List<T> inputList, int chunkSize) {
- AtomicInteger counter = new AtomicInteger();
- return inputList.stream().collect(Collectors.groupingBy(it -> counter.getAndIncrement() / chunkSize)).values();
- }
-
-}
diff --git a/Build/bamboo/src/test/java/utilities/JobFixture.java b/Build/bamboo/src/test/java/utilities/JobFixture.java
deleted file mode 100644
index 8f03fffd88a68514b51d9ce73b53e6b9cd8ddd7a..0000000000000000000000000000000000000000
--- a/Build/bamboo/src/test/java/utilities/JobFixture.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package utilities;
-
-public class JobFixture {
- private final boolean isLimited;
-
- public JobFixture(boolean isLimited) {
- this.isLimited = isLimited;
- }
-
- public boolean isLimited(){
- return isLimited;
- }
-}
diff --git a/Build/bamboo/src/test/java/utilities/LimitedChunkerTest.java b/Build/bamboo/src/test/java/utilities/LimitedChunkerTest.java
deleted file mode 100644
index b3af82f6464c8f3b53e4800e4bd42fc51202cdd0..0000000000000000000000000000000000000000
--- a/Build/bamboo/src/test/java/utilities/LimitedChunkerTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-package utilities;
-
-/*
- * This file is part of the TYPO3 CMS project.
- *
- * It is free software; you can redistribute it and/or modify it under
- * the terms of the GNU General Public License, either version 2
- * of the License, or any later version.
- *
- * For the full copyright and license information, please read the
- * LICENSE.txt file that was distributed with this source code.
- *
- * The TYPO3 project - inspiring people to share!
- */
-
-import core.utilities.LimitedChunker;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-@RunWith(Parameterized.class)
-public class LimitedChunkerTest {
- private final int numLimitedJobs;
- private final int numNormalJobs;
- private final int limitedChunkSize;
- private final int totalChunkSize;
- private int expectedNumberOfChunks;
-
- @Test(expected = IllegalArgumentException.class)
- public void throwsOnZeroLimitedElementsPerChunk() {
- LimitedChunker<JobFixture> chunker = new LimitedChunker<>(0, 25);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void throwsOnZeroTotalChunkSize() {
- LimitedChunker<JobFixture> chunker = new LimitedChunker<>(25, 0);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void throwsOnMoreLimitedThanChunkSize() {
- LimitedChunker<JobFixture> chunker = new LimitedChunker<>(25, 12);
- }
-
- @Test
- public void returnsEmptyChunkListWithEmptyInput() {
- LimitedChunker<JobFixture> chunker = new LimitedChunker<>(12, 25);
-
- ArrayList<JobFixture> jobs = new ArrayList<>(0);
- ArrayList<JobFixture> limitedJobs = new ArrayList<>(0);
-
- List<List<JobFixture>> chunks = chunker.chunk(limitedJobs, jobs);
-
- assert chunks.size() == 0;
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> data() {
- return Arrays.asList(
- new Integer[]{25, 25, 25, 50, 1},
- new Integer[]{25, 0, 25, 25, 1},
- new Integer[]{100, 100, 25, 50, 4},
- new Integer[]{110, 100, 25, 50, 5},
- new Integer[]{100, 110, 25, 50, 5}
- );
- }
-
- public LimitedChunkerTest(int numLimitedJobs, int numNormalJobs, int limitedChunkSize, int totalChunkSize, int expectedNumberOfChunks) {
- this.numLimitedJobs = numLimitedJobs;
- this.numNormalJobs = numNormalJobs;
- this.limitedChunkSize = limitedChunkSize;
- this.totalChunkSize = totalChunkSize;
- this.expectedNumberOfChunks = expectedNumberOfChunks;
- }
-
- @Test
- public void chunksCorrectly() {
- ArrayList<JobFixture> limitedJobs = getJobs(numLimitedJobs, true);
- ArrayList<JobFixture> normalJobs = getJobs(numNormalJobs, false);
-
- List<List<JobFixture>> chunks = new LimitedChunker<JobFixture>(limitedChunkSize, totalChunkSize).chunk(limitedJobs, normalJobs);
-
- assert chunks.size() == expectedNumberOfChunks;
-
- int actualNumberOfJobs = chunks.stream().map(List::size).reduce(0, Integer::sum);
- assert actualNumberOfJobs == numLimitedJobs + numNormalJobs;
-
-
- int chunkIndex = 0;
- for (List<JobFixture> chunk : chunks) {
- long numLimited = chunk.stream().filter(JobFixture::isLimited).count();
- assert(numLimited <= limitedChunkSize);
- if (chunkIndex < chunks.size() - 1) {
- // all chunks must be full
- assert(chunk.size() == totalChunkSize);
- } else {
- //except the last one
- assert(chunk.size() <= totalChunkSize);
- }
- chunkIndex++;
- }
- }
-
- private ArrayList<JobFixture> getJobs(int numLimitedJobs, boolean isLimited) {
- ArrayList<JobFixture> jobs = new ArrayList<>(numLimitedJobs);
- for (int i = 0; i < numLimitedJobs; i++) {
- jobs.add(new JobFixture(isLimited));
- }
- return jobs;
- }
-}
diff --git a/Build/testing-docker/bamboo/docker-compose.yml b/Build/testing-docker/bamboo/docker-compose.yml
index fdcc88d5fa2a448f461f159d3500ebfdb0422a52..9744ef28b231ad981d0f01634631d6176cc50005 100644
--- a/Build/testing-docker/bamboo/docker-compose.yml
+++ b/Build/testing-docker/bamboo/docker-compose.yml
@@ -23,13 +23,13 @@ services:
- /var/lib/postgresql/data:rw,noexec,nosuid
networks:
- test
- mssql2017cu17:
- image: mcr.microsoft.com/mssql/server:2017-CU17-ubuntu
+
+ mssql2019latest:
+ image: mcr.microsoft.com/mssql/server:2019-latest
environment:
ACCEPT_EULA: Y
SA_PASSWORD: Test1234!
MSSQL_PID: Developer
- MSSQL_MEMORY_LIMIT_MB: 3584
tmpfs:
- /var/lib/mssql:rw,noexec,nosuid
volumes:
@@ -159,7 +159,7 @@ services:
start_dependencies_functional_mssql:
image: alpine:3.8
links:
- - mssql2017cu17
+ - mssql2019latest
- redis4
- memcached1-5
networks:
@@ -168,7 +168,7 @@ services:
/bin/sh -c "
echo Waiting for database start
COUNT=0
- while ! nc -z mssql2017cu17 1433; do
+ while ! nc -z mssql2019latest 1433; do
if [ "$${COUNT}" -ge "120" ]; then
echo Database did not come up
exit 1
diff --git a/Build/testing-docker/local/docker-compose.yml b/Build/testing-docker/local/docker-compose.yml
index a401c67a81ff570f127275f8bc09990201141be6..27dfc995d083f61cbc061aad628c5f550f9c1bd2 100644
--- a/Build/testing-docker/local/docker-compose.yml
+++ b/Build/testing-docker/local/docker-compose.yml
@@ -17,23 +17,12 @@ services:
tmpfs:
- /var/lib/mysql/:rw,noexec,nosuid
- mssql2017cu17:
- image: mcr.microsoft.com/mssql/server:2017-CU17-ubuntu
+ mssql2019latest:
+ image: mcr.microsoft.com/mssql/server:2019-latest
environment:
ACCEPT_EULA: Y
SA_PASSWORD: "Test1234!"
MSSQL_PID: Developer
- MSSQL_MEMORY_LIMIT_MB: 3584
- tmpfs:
- - /var/lib/mssql:rw,noexec,nosuid
-
- mssql2017latest:
- image: mcr.microsoft.com/mssql/server:2017-latest-ubuntu
- environment:
- ACCEPT_EULA: Y
- SA_PASSWORD: "Test1234!"
- MSSQL_PID: Developer
- MSSQL_MEMORY_LIMIT_MB: 3584
tmpfs:
- /var/lib/mssql:rw,noexec,nosuid
@@ -642,62 +631,10 @@ services:
fi
"
- prepare_functional_mssql2017cu17:
- image: alpine:3.8
- links:
- - mssql2017cu17
- - redis4
- - memcached1-5
- command: >
- /bin/sh -c "
- if [ ${SCRIPT_VERBOSE} -eq 1 ]; then
- set -x
- fi
- echo Waiting for database start...;
- while ! nc -z mssql2017cu17 1433; do
- sleep 1;
- done;
- echo Database is up;
- "
- functional_mssql2017cu17:
- image: typo3gmbh/${DOCKER_PHP_IMAGE}:latest
- user: ${HOST_UID}
- volumes:
- - ${CORE_ROOT}:${CORE_ROOT}
- - ${HOST_HOME}:${HOST_HOME}
- - /etc/passwd:/etc/passwd:ro
- - /etc/group:/etc/group:ro
- environment:
- typo3DatabaseDriver: "${DATABASE_DRIVER:-sqlsrv}"
- typo3DatabaseName: func
- typo3DatabasePassword: "Test1234!"
- typo3DatabaseUsername: SA
- typo3DatabasePort: 1433
- typo3DatabaseCharset: utf-8
- typo3DatabaseHost: mssql2017cu17
- typo3TestingRedisHost: redis4
- typo3TestingMemcachedHost: memcached1-5
- working_dir: ${CORE_ROOT}
- command: >
- /bin/sh -c "
- if [ ${SCRIPT_VERBOSE} -eq 1 ]; then
- set -x
- fi
- php -v | grep '^PHP'
- if [ ${PHP_XDEBUG_ON} -eq 0 ]; then
- php -n -c /etc/php/cli-no-xdebug/php.ini \
- vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE};
- else
- DOCKER_HOST=`route -n | awk '/^0.0.0.0/ { print $$2 }'`
- XDEBUG_CONFIG=\"remote_port=${PHP_XDEBUG_PORT} remote_enable=1 remote_host=$${DOCKER_HOST}\" \
- vendor/phpunit/phpunit/phpunit -c vendor/typo3/testing-framework/Resources/Core/Build/FunctionalTests.xml ${EXTRA_TEST_OPTIONS} --exclude-group not-mssql ${TEST_FILE};
- fi
- "
-
- prepare_functional_mssql2017latest:
+ prepare_functional_mssql2019latest:
image: alpine:3.8
links:
- - mssql2017latest
+ - mssql2019latest
- redis4
- memcached1-5
command: >
@@ -706,13 +643,13 @@ services:
set -x
fi
echo Waiting for database start...;
- while ! nc -z mssql2017latest 1433; do
+ while ! nc -z mssql2019latest 1433; do
sleep 1;
done;
echo Database is up;
"
- functional_mssql2017latest:
+ functional_mssql2019latest:
image: typo3gmbh/${DOCKER_PHP_IMAGE}:latest
user: ${HOST_UID}
volumes:
@@ -727,7 +664,7 @@ services:
typo3DatabaseUsername: SA
typo3DatabasePort: 1433
typo3DatabaseCharset: utf-8
- typo3DatabaseHost: mssql2017latest
+ typo3DatabaseHost: mssql2019latest
typo3TestingRedisHost: redis4
typo3TestingMemcachedHost: memcached1-5
working_dir: ${CORE_ROOT}