diff --git a/Build/bamboo/src/main/java/core/NightlySpec.java b/Build/bamboo/src/main/java/core/NightlySpec.java
index a4e0c8f19358db24bcd72de1efda8fcc757398ce..5170b18dca17cb37b92e956105c7921f31c52ba2 100644
--- a/Build/bamboo/src/main/java/core/NightlySpec.java
+++ b/Build/bamboo/src/main/java/core/NightlySpec.java
@@ -28,6 +28,7 @@ import com.atlassian.bamboo.specs.api.builders.task.Task;
import com.atlassian.bamboo.specs.builders.notification.PlanCompletedNotification;
import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
import com.atlassian.bamboo.specs.util.BambooServer;
+import core.utilities.LimitedChunker;
import java.util.ArrayList;
import java.util.Collections;
@@ -59,7 +60,7 @@ public class NightlySpec extends AbstractCoreSpec {
private String[] sqLiteVersions = {"3.15", "3.20", "3.25", "3.30"};
private String[] postGreSqlVersions = {"9.3", "9.4", "9.5", "9.6", "10.11", "11.6", "12.1"};
- private int jobListSize = 50;
+ private int totalJobsPerStage = 50;
private int mssqlJobsPerStage = 25;
/**
@@ -107,50 +108,23 @@ public class NightlySpec extends AbstractCoreSpec {
stages.add(stagePreparation);
stages.add(stageIntegrity);
- int otherJobsStages = jobs.size() / (jobListSize - mssqlJobsPerStage);
- int mssqlJobsStages = mssqlJobs.size() / mssqlJobsPerStage;
- int handledJobs = 0;
- int jobCount = 0;
- int mssqlCount = 0;
- int otherCount = 0;
- for (int i = 0; i < Math.max(otherJobsStages, mssqlJobsStages); i++) {
- List<Job> mssqlJobsChunk = new ArrayList<Job>();
- int chunkMinIndex = i * mssqlJobsPerStage;
- int chunkMaxIndex = (i + 1) * mssqlJobsPerStage;
-
- if (mssqlJobs.size() >= chunkMaxIndex) {
- mssqlJobsChunk = mssqlJobs.subList(chunkMinIndex, chunkMaxIndex);
- } else {
- if (mssqlJobs.size() >= chunkMinIndex) {
- mssqlJobsChunk = mssqlJobs.subList(chunkMinIndex, mssqlJobs.size());
- }
- }
-
- List<Job> otherJobsChunk;
- chunkMinIndex = handledJobs;
- chunkMaxIndex = (jobListSize - mssqlJobsChunk.size() + handledJobs);
- if (jobs.size() >= chunkMaxIndex) {
- otherJobsChunk = jobs.subList(chunkMinIndex, chunkMaxIndex);
- } else {
- otherJobsChunk = jobs.subList(chunkMinIndex, jobs.size());
- }
- handledJobs = handledJobs + otherJobsChunk.size();
-
- ArrayList<Job> stagingJobs = new ArrayList<Job>();
- stagingJobs.addAll(mssqlJobsChunk);
- stagingJobs.addAll(otherJobsChunk);
- otherCount = otherCount + otherJobsChunk.size();
- mssqlCount = mssqlCount + mssqlJobsChunk.size();
- jobCount = jobCount + otherJobsChunk.size() + mssqlJobsChunk.size();
- if (stagingJobs.size() > 0) {
- Collections.shuffle(stagingJobs);
- Stage stage = new Stage("Stage " + (i + 1) + ", Jobs " + (i * jobListSize) + " - " + (((i + 1) * jobListSize) - 1));
- System.out.println("Stage " + (i + 1) + " got " + stagingJobs.size() + " Jobs, " + otherJobsChunk.size() + " jobs and " + mssqlJobsChunk.size() + " mssql jobs");
- stage.jobs(stagingJobs.toArray(new Job[stagingJobs.size()]));
- stages.add(stage);
- }
+ LimitedChunker<Job> chunker = new LimitedChunker<>(mssqlJobsPerStage, totalJobsPerStage);
+ List<List<Job>> chunks = chunker.chunk(mssqlJobs, jobs);
+
+ int totalNumberOfJobs = jobs.size() + mssqlJobs.size();
+ int stageIndex = 0;
+ for (List<Job> chunk : chunks) {
+ int firstJobIndex = totalJobsPerStage * stageIndex + 1;
+ int lastJobIndex = totalJobsPerStage * (stageIndex + 1);
+ lastJobIndex = Math.min(lastJobIndex, totalNumberOfJobs);
+
+ Collections.shuffle(chunk);
+ Stage stage = new Stage("Stage " + (stageIndex + 1) + ", Jobs " + firstJobIndex + " - " + lastJobIndex);
+ stage.jobs(chunk.toArray(new Job[chunk.size()]));
+ stages.add(stage);
+ System.out.println("Stage " + (stageIndex + 1) + " got " + chunk.size() + " Jobs");
+ stageIndex++;
}
- System.out.println("deployed " + jobCount + " (" + mssqlCount + " mssql and " + otherCount + " other) " + " out of " + (jobs.size() + mssqlJobs.size()) + " Jobs (" + mssqlJobs.size() + " mssql and " + jobs.size() + " other) in " + (stages.size() - 2) + " Stages");
// Compile plan
return new Plan(project(), planName, planKey).description("Execute TYPO3 core master nightly tests. Auto generated! See Build/bamboo of core git repository.")
diff --git a/Build/bamboo/src/main/java/core/utilities/LimitedChunker.java b/Build/bamboo/src/main/java/core/utilities/LimitedChunker.java
new file mode 100644
index 0000000000000000000000000000000000000000..6ba171dc100099ebd1266a5d9c18f2668e71a8a4
--- /dev/null
+++ b/Build/bamboo/src/main/java/core/utilities/LimitedChunker.java
@@ -0,0 +1,63 @@
+package core.utilities;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+/**
+ * Chunk two lists into chunks of a given size
+ * where only `numLimitedJobsPerChunk` elements from one list are in each chunk
+ *
+ * @param <T>
+ */
+public class LimitedChunker<T> {
+
+ private final int numLimitedElementsPerChunk;
+ private final int totalChunkSize;
+
+ public LimitedChunker(int numLimitedElementsPerChunk, int totalChunkSize) {
+ this.numLimitedElementsPerChunk = numLimitedElementsPerChunk;
+ this.totalChunkSize = totalChunkSize;
+
+ if (numLimitedElementsPerChunk < 1) {
+ throw new IllegalArgumentException("Number of limited Elements per Chunk must be greater than one");
+ }
+ if (totalChunkSize < 1) {
+ throw new IllegalArgumentException("Total chunk size must be greater than one");
+ }
+ if (totalChunkSize < numLimitedElementsPerChunk) {
+ throw new IllegalArgumentException("Number of limited elements must not be greater than total chunk size");
+ }
+ }
+
+ public List<List<T>> chunk(List<T> limitedElements, List<T> normalElements) {
+ // chunk the limited elements in sets of limitedChunkSize
+ LinkedList<List<T>> chunks = new LinkedList<>(prepareChunks(limitedElements, numLimitedElementsPerChunk));
+
+ // fill up these chunks with normal elements
+ Iterator<T> normalJobIterator = normalElements.iterator();
+ for (List<T> chunk : chunks) {
+ fillChunkToSize(chunk, totalChunkSize, normalJobIterator);
+ }
+ // add chunks for all remaining normal elements, if any
+ while (normalJobIterator.hasNext()) {
+ List<T> chunk = new ArrayList<>(totalChunkSize);
+ fillChunkToSize(chunk, totalChunkSize, normalJobIterator);
+ chunks.add(chunk);
+ }
+
+ return chunks;
+ }
+
+ private void fillChunkToSize(List<T> chunk, int totalChunkSize, Iterator<T> remainingElementsIterator) {
+ while (remainingElementsIterator.hasNext() && chunk.size() < totalChunkSize) {
+ chunk.add(remainingElementsIterator.next());
+ }
+ }
+
+ private Collection<List<T>> prepareChunks(List<T> inputList, int chunkSize) {
+ AtomicInteger counter = new AtomicInteger();
+ return inputList.stream().collect(Collectors.groupingBy(it -> counter.getAndIncrement() / chunkSize)).values();
+ }
+
+}
diff --git a/Build/bamboo/src/test/java/utilities/JobFixture.java b/Build/bamboo/src/test/java/utilities/JobFixture.java
new file mode 100644
index 0000000000000000000000000000000000000000..8f03fffd88a68514b51d9ce73b53e6b9cd8ddd7a
--- /dev/null
+++ b/Build/bamboo/src/test/java/utilities/JobFixture.java
@@ -0,0 +1,13 @@
+package utilities;
+
+public class JobFixture {
+ private final boolean isLimited;
+
+ public JobFixture(boolean isLimited) {
+ this.isLimited = isLimited;
+ }
+
+ public boolean isLimited(){
+ return isLimited;
+ }
+}
diff --git a/Build/bamboo/src/test/java/utilities/LimitedChunkerTest.java b/Build/bamboo/src/test/java/utilities/LimitedChunkerTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..b3af82f6464c8f3b53e4800e4bd42fc51202cdd0
--- /dev/null
+++ b/Build/bamboo/src/test/java/utilities/LimitedChunkerTest.java
@@ -0,0 +1,115 @@
+package utilities;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+import core.utilities.LimitedChunker;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+@RunWith(Parameterized.class)
+public class LimitedChunkerTest {
+ private final int numLimitedJobs;
+ private final int numNormalJobs;
+ private final int limitedChunkSize;
+ private final int totalChunkSize;
+ private int expectedNumberOfChunks;
+
+ @Test(expected = IllegalArgumentException.class)
+ public void throwsOnZeroLimitedElementsPerChunk() {
+ LimitedChunker<JobFixture> chunker = new LimitedChunker<>(0, 25);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void throwsOnZeroTotalChunkSize() {
+ LimitedChunker<JobFixture> chunker = new LimitedChunker<>(25, 0);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void throwsOnMoreLimitedThanChunkSize() {
+ LimitedChunker<JobFixture> chunker = new LimitedChunker<>(25, 12);
+ }
+
+ @Test
+ public void returnsEmptyChunkListWithEmptyInput() {
+ LimitedChunker<JobFixture> chunker = new LimitedChunker<>(12, 25);
+
+ ArrayList<JobFixture> jobs = new ArrayList<>(0);
+ ArrayList<JobFixture> limitedJobs = new ArrayList<>(0);
+
+ List<List<JobFixture>> chunks = chunker.chunk(limitedJobs, jobs);
+
+ assert chunks.size() == 0;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(
+ new Integer[]{25, 25, 25, 50, 1},
+ new Integer[]{25, 0, 25, 25, 1},
+ new Integer[]{100, 100, 25, 50, 4},
+ new Integer[]{110, 100, 25, 50, 5},
+ new Integer[]{100, 110, 25, 50, 5}
+ );
+ }
+
+ public LimitedChunkerTest(int numLimitedJobs, int numNormalJobs, int limitedChunkSize, int totalChunkSize, int expectedNumberOfChunks) {
+ this.numLimitedJobs = numLimitedJobs;
+ this.numNormalJobs = numNormalJobs;
+ this.limitedChunkSize = limitedChunkSize;
+ this.totalChunkSize = totalChunkSize;
+ this.expectedNumberOfChunks = expectedNumberOfChunks;
+ }
+
+ @Test
+ public void chunksCorrectly() {
+ ArrayList<JobFixture> limitedJobs = getJobs(numLimitedJobs, true);
+ ArrayList<JobFixture> normalJobs = getJobs(numNormalJobs, false);
+
+ List<List<JobFixture>> chunks = new LimitedChunker<JobFixture>(limitedChunkSize, totalChunkSize).chunk(limitedJobs, normalJobs);
+
+ assert chunks.size() == expectedNumberOfChunks;
+
+ int actualNumberOfJobs = chunks.stream().map(List::size).reduce(0, Integer::sum);
+ assert actualNumberOfJobs == numLimitedJobs + numNormalJobs;
+
+
+ int chunkIndex = 0;
+ for (List<JobFixture> chunk : chunks) {
+ long numLimited = chunk.stream().filter(JobFixture::isLimited).count();
+ assert(numLimited <= limitedChunkSize);
+ if (chunkIndex < chunks.size() - 1) {
+ // all chunks must be full
+ assert(chunk.size() == totalChunkSize);
+ } else {
+ //except the last one
+ assert(chunk.size() <= totalChunkSize);
+ }
+ chunkIndex++;
+ }
+ }
+
+ private ArrayList<JobFixture> getJobs(int numLimitedJobs, boolean isLimited) {
+ ArrayList<JobFixture> jobs = new ArrayList<>(numLimitedJobs);
+ for (int i = 0; i < numLimitedJobs; i++) {
+ jobs.add(new JobFixture(isLimited));
+ }
+ return jobs;
+ }
+}