From 6e11355a1eea5d17c79eb6c13bb180754fb3a8b6 Mon Sep 17 00:00:00 2001
From: Christian Kuhn <lolli@schwarzbu.ch>
Date: Thu, 19 Jul 2018 20:46:33 +0200
Subject: [PATCH] [TASK] Enable acceptance tests again
Bring back backend acceptance test execution as pre-merge
and nightly tests. They are hopefully more stable now and
we'll test if they survive the hard pre-merge load without
being too flaky..
The patch changes the bamboo plan specs accordingly but disables
a couple for FormEngine related detail tests that need further
investigation.
The patch also fetches a new patch level of testing-framework
to have proper ac path definition of the ac splitter script:
composer update typo3/testing-framework
Change-Id: Iffcfd60391ba60d5ab2196a00a5bd0a4a93fff7f
Resolves: #85599
Releases: master
Reviewed-on: https://review.typo3.org/57635
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
Tested-by: Christian Kuhn <lolli@schwarzbu.ch>
---
.../src/main/java/core/AbstractCoreSpec.java | 62 +++++++++++++------
.../src/main/java/core/NightlySpec.java | 2 +-
.../src/main/java/core/PreMergeSpec.java | 2 +-
.../testing-docker/bamboo/docker-compose.yml | 18 ++++++
composer.lock | 10 +--
.../core/Tests/Acceptance/Backend.suite.yml | 5 +-
.../FormEngine/ElementsBasicInputCest.php | 8 ++-
.../FormEngine/ElementsBasicInputDateCest.php | 12 ++--
8 files changed, 85 insertions(+), 34 deletions(-)
diff --git a/Build/bamboo/src/main/java/core/AbstractCoreSpec.java b/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
index 119dc0b0dbc1..305e2a7d8e5e 100644
--- a/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
+++ b/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
@@ -49,15 +49,6 @@ abstract public class AbstractCoreSpec {
protected String testingFrameworkBuildPath = "vendor/typo3/testing-framework/Resources/Core/Build/";
- /**
- * @todo This can be removed if acceptance mysql tests are rewritten and active again
- */
- protected String credentialsMysql =
- "typo3DatabaseName=\"func_test\"" +
- " typo3DatabaseUsername=\"root\"" +
- " typo3DatabasePassword=\"funcp\"" +
- " typo3DatabaseHost=\"mariadb10\"";
-
/**
* @todo This can be removed if acceptance mssql functional tests work again
*/
@@ -333,12 +324,10 @@ abstract public class AbstractCoreSpec {
/**
* Jobs for mysql based acceptance tests
*
- * @todo Currently disabled and broken
- *
* @param int numberOfChunks
* @param String requirementIdentifier
*/
- protected ArrayList<Job> getJobsAcceptanceTestsMysql(int numberOfChunks, String requirementIdentifier) {
+ protected ArrayList<Job> getJobsAcceptanceTestsBackendMysql(int numberOfChunks, String requirementIdentifier) {
ArrayList<Job> jobs = new ArrayList<Job>();
for (int i=1; i<=numberOfChunks; i++) {
@@ -354,6 +343,7 @@ abstract public class AbstractCoreSpec {
this.getTaskGitCherryPick(),
this.getTaskComposerInstall(requirementIdentifier),
this.getTaskPrepareAcceptanceTest(),
+ this.getTaskDockerDependenciesAcceptanceBackendMariadb10(),
new ScriptTask()
.description("Split acceptance tests")
.interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
@@ -361,11 +351,29 @@ abstract public class AbstractCoreSpec {
this.getScriptTaskBashInlineBody() +
"./" + this.testingFrameworkBuildPath + "Scripts/splitAcceptanceTests.sh " + numberOfChunks + "\n"
),
- new CommandTask()
+ new ScriptTask()
.description("Execute codeception acceptance suite group " + formattedI)
- .executable("codecept")
- .argument("run Acceptance -d -g AcceptanceTests-Job-" + i + " -c " + this.testingFrameworkBuildPath + "AcceptanceTests.yml --xml reports.xml --html reports.html")
- .environmentVariables(this.credentialsMysql)
+ .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+ .inlineBody(
+ this.getScriptTaskBashInlineBody() +
+ "function codecept() {\n" +
+ " docker run \\\n" +
+ " -u ${HOST_UID} \\\n" +
+ " -v /bamboo-data/${BAMBOO_COMPOSE_PROJECT_NAME}/passwd:/etc/passwd \\\n" +
+ " -v ${BAMBOO_COMPOSE_PROJECT_NAME}_bamboo-data:/srv/bamboo/xml-data/build-dir/ \\\n" +
+ " -e typo3DatabaseName=func_test \\\n" +
+ " -e typo3DatabaseUsername=root \\\n" +
+ " -e typo3DatabasePassword=funcp \\\n" +
+ " -e typo3DatabaseHost=mariadb10 \\\n" +
+ " --name ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc \\\n" +
+ " --network ${BAMBOO_COMPOSE_PROJECT_NAME}_test \\\n" +
+ " --rm \\\n" +
+ " typo3gmbh/" + requirementIdentifier.toLowerCase() + ":latest \\\n" +
+ " bin/bash -c \"cd ${PWD}; ./bin/codecept $*\"\n" +
+ "}\n" +
+ "\n" +
+ "codecept run Backend -d -g AcceptanceTests-Job-" + i + " -c typo3/sysext/core/Tests/codeception.yml --xml reports.xml --html reports.html\n"
+ )
)
.finalTasks(
this.getTaskStopDockerDependencies(),
@@ -381,7 +389,6 @@ abstract public class AbstractCoreSpec {
this.getRequirementDocker10()
)
.cleanWorkingDirectory(true)
- .enabled(false)
);
}
@@ -1182,7 +1189,7 @@ abstract public class AbstractCoreSpec {
}
/**
- * Start docker sibling containers to execute acceptance tests on mariadb
+ * Start docker sibling containers to execute acceptance install tests on mariadb
*/
protected Task getTaskDockerDependenciesAcceptanceInstallMariadb10() {
return new ScriptTask()
@@ -1197,7 +1204,7 @@ abstract public class AbstractCoreSpec {
}
/**
- * Start docker sibling containers to execute acceptance tests on postgres
+ * Start docker sibling containers to execute acceptance install tests on postgres
*/
protected Task getTaskDockerDependenciesAcceptanceInstallPostgres10() {
return new ScriptTask()
@@ -1212,7 +1219,7 @@ abstract public class AbstractCoreSpec {
}
/**
- * Start docker sibling containers to execute acceptance tests on sqlite
+ * Start docker sibling containers to execute acceptance install tests on sqlite
*/
protected Task getTaskDockerDependenciesAcceptanceInstallSqlite() {
return new ScriptTask()
@@ -1226,6 +1233,21 @@ abstract public class AbstractCoreSpec {
);
}
+ /**
+ * Start docker sibling containers to execute acceptance backend tests on mariadb
+ */
+ protected Task getTaskDockerDependenciesAcceptanceBackendMariadb10() {
+ return new ScriptTask()
+ .description("Start docker siblings for acceptance test backend mariadb")
+ .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+ .inlineBody(
+ this.getScriptTaskBashInlineBody() +
+ "cd Build/testing-docker/bamboo\n" +
+ "echo COMPOSE_PROJECT_NAME=${BAMBOO_COMPOSE_PROJECT_NAME}sib > .env\n" +
+ "docker-compose run start_dependencies_acceptance_backend_mariadb10"
+ );
+ }
+
/**
* Start docker sibling containers to execute functional tests on mariadb
*/
diff --git a/Build/bamboo/src/main/java/core/NightlySpec.java b/Build/bamboo/src/main/java/core/NightlySpec.java
index e00f5ec6f79f..2acc6e8bddb2 100644
--- a/Build/bamboo/src/main/java/core/NightlySpec.java
+++ b/Build/bamboo/src/main/java/core/NightlySpec.java
@@ -75,7 +75,7 @@ public class NightlySpec extends AbstractCoreSpec {
jobsMainStage.add(this.getJobAcceptanceTestInstallPgsql("PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallSqlite("PHP72"));
- jobsMainStage.addAll(this.getJobsAcceptanceTestsMysql(this.numberOfAcceptanceTestJobs, "PHP72"));
+ jobsMainStage.addAll(this.getJobsAcceptanceTestsBackendMysql(this.numberOfAcceptanceTestJobs, "PHP72"));
jobsMainStage.add(this.getJobCglCheckFullCore("PHP72"));
diff --git a/Build/bamboo/src/main/java/core/PreMergeSpec.java b/Build/bamboo/src/main/java/core/PreMergeSpec.java
index 2d38da72654b..f49691fef065 100644
--- a/Build/bamboo/src/main/java/core/PreMergeSpec.java
+++ b/Build/bamboo/src/main/java/core/PreMergeSpec.java
@@ -94,7 +94,7 @@ public class PreMergeSpec extends AbstractCoreSpec {
jobsMainStage.add(this.getJobAcceptanceTestInstallPgsql("PHP72"));
jobsMainStage.add(this.getJobAcceptanceTestInstallSqlite("PHP72"));
- jobsMainStage.addAll(this.getJobsAcceptanceTestsMysql(this.numberOfAcceptanceTestJobs, "PHP72"));
+ jobsMainStage.addAll(this.getJobsAcceptanceTestsBackendMysql(this.numberOfAcceptanceTestJobs, "PHP72"));
jobsMainStage.add(this.getJobIntegrationAnnotations("PHP72"));
diff --git a/Build/testing-docker/bamboo/docker-compose.yml b/Build/testing-docker/bamboo/docker-compose.yml
index 5df4af234bc9..346bf7142e5c 100644
--- a/Build/testing-docker/bamboo/docker-compose.yml
+++ b/Build/testing-docker/bamboo/docker-compose.yml
@@ -87,6 +87,24 @@ services:
sleep 1;
"
+ start_dependencies_acceptance_backend_mariadb10:
+ image: alpine:3.8
+ links:
+ - mariadb10
+ - chrome
+ - web
+ networks:
+ - test
+ command: >
+ /bin/sh -c "
+ echo Waiting for db start...;
+ while ! nc -z mariadb10 3306;
+ do
+ sleep 1;
+ done;
+ echo Connected!;
+ "
+
start_dependencies_functional_mariadb10:
image: alpine:3.8
links:
diff --git a/composer.lock b/composer.lock
index de4c5bc6fdae..6aa7c2e6443f 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4617,16 +4617,16 @@
},
{
"name": "typo3/testing-framework",
- "version": "4.0.0",
+ "version": "4.0.1",
"source": {
"type": "git",
"url": "https://github.com/TYPO3/testing-framework.git",
- "reference": "0155dc11572dd7722506ea0005a0e1c24ce89bdf"
+ "reference": "e24b2075122e7fa31d1e91b3024a796e5c725f17"
},
"dist": {
"type": "zip",
- "url": "https://api.github.com/repos/TYPO3/testing-framework/zipball/0155dc11572dd7722506ea0005a0e1c24ce89bdf",
- "reference": "0155dc11572dd7722506ea0005a0e1c24ce89bdf",
+ "url": "https://api.github.com/repos/TYPO3/testing-framework/zipball/e24b2075122e7fa31d1e91b3024a796e5c725f17",
+ "reference": "e24b2075122e7fa31d1e91b3024a796e5c725f17",
"shasum": ""
},
"require": {
@@ -4673,7 +4673,7 @@
"tests",
"typo3"
],
- "time": "2018-07-07T08:51:19+00:00"
+ "time": "2018-07-19T20:14:59+00:00"
},
{
"name": "webmozart/assert",
diff --git a/typo3/sysext/core/Tests/Acceptance/Backend.suite.yml b/typo3/sysext/core/Tests/Acceptance/Backend.suite.yml
index 3d0b9a542e44..c2c3923e0825 100644
--- a/typo3/sysext/core/Tests/Acceptance/Backend.suite.yml
+++ b/typo3/sysext/core/Tests/Acceptance/Backend.suite.yml
@@ -2,7 +2,7 @@ class_name: BackendTester
modules:
enabled:
- WebDriver:
- url: http://master:8000/typo3temp/var/tests/acceptance
+ url: http://web:8000/typo3temp/var/tests/acceptance
browser: chrome
wait: 1
host: chrome
@@ -18,6 +18,9 @@ extensions:
enabled:
- TYPO3\TestingFramework\Core\Acceptance\Extension\BackendCoreEnvironment
+groups:
+ AcceptanceTests-Job-*: AcceptanceTests-Job-*
+
env:
headless:
modules:
diff --git a/typo3/sysext/core/Tests/Acceptance/Backend/FormEngine/ElementsBasicInputCest.php b/typo3/sysext/core/Tests/Acceptance/Backend/FormEngine/ElementsBasicInputCest.php
index 3d3040c2b889..40af284397ac 100644
--- a/typo3/sysext/core/Tests/Acceptance/Backend/FormEngine/ElementsBasicInputCest.php
+++ b/typo3/sysext/core/Tests/Acceptance/Backend/FormEngine/ElementsBasicInputCest.php
@@ -262,8 +262,9 @@ class ElementsBasicInputCest extends AbstractElementsBasicCest
'expectedValueAfterSave' => '129',
'comment' => '',
],
+ /**
[
- // this one probably broke with the type="number" patch
+ // @todo this one probably broke with the html type="number" patch
'label' => 'input_9 eval=int',
'inputValue' => 'TYPO3',
'expectedValue' => '0',
@@ -271,6 +272,7 @@ class ElementsBasicInputCest extends AbstractElementsBasicCest
'expectedValueAfterSave' => '0',
'comment' => '',
],
+ */
[
'label' => 'input_9 eval=int',
'inputValue' => '3TYPO',
@@ -330,8 +332,9 @@ class ElementsBasicInputCest extends AbstractElementsBasicCest
protected function simpleRangeAndMd5FieldsDataProvider()
{
return [
+ /**
[
- // this one probably broke with the type="number" patch
+ // @todo this one probably broke with the type="number" patch
'label' => 'input_25 eval=int, default=0, range lower=-2, range upper=2',
'inputValue' => 'Kasper TYPO3',
'expectedValue' => '0',
@@ -339,6 +342,7 @@ class ElementsBasicInputCest extends AbstractElementsBasicCest
'expectedValueAfterSave' => '0',
'comment' => '',
],
+ */
[
'label' => 'input_25 eval=int, default=0, range lower=-2, range upper=2',
'inputValue' => '2',
diff --git a/typo3/sysext/core/Tests/Acceptance/Backend/FormEngine/ElementsBasicInputDateCest.php b/typo3/sysext/core/Tests/Acceptance/Backend/FormEngine/ElementsBasicInputDateCest.php
index 98d1197a45ab..6cd081601ac5 100644
--- a/typo3/sysext/core/Tests/Acceptance/Backend/FormEngine/ElementsBasicInputDateCest.php
+++ b/typo3/sysext/core/Tests/Acceptance/Backend/FormEngine/ElementsBasicInputDateCest.php
@@ -97,11 +97,13 @@ class ElementsBasicInputDateCest extends AbstractElementsBasicCest
* @dataProvider dbTypeDateEvalDateDataProvider
* @param BackendTester $I
* @param Example $testData
- */
+ * @todo fix these unstable tests
public function dbTypeDateEvalDate(BackendTester $I, Example $testData)
{
$this->runInputFieldTest($I, $testData);
}
+ */
+
/**
* dbType date eval datetime data provider
*/
@@ -151,11 +153,12 @@ class ElementsBasicInputDateCest extends AbstractElementsBasicCest
* @dataProvider dbTypeDateEvalDatetimeDataProvider
* @param BackendTester $I
* @param Example $testData
- */
+ * @todo fix these unstable tests
public function dbTypeDateEvalDatetime(BackendTester $I, Example $testData)
{
$this->runInputFieldTest($I, $testData);
}
+ */
/**
* db type date eval time data provider
@@ -215,11 +218,12 @@ class ElementsBasicInputDateCest extends AbstractElementsBasicCest
* @dataProvider dbTypeDateEvalTimeDataProvider
* @param BackendTester $I
* @param Example $testData
- */
+ * @todo fix these unstable tests
public function dbTypeDateEvalTime(BackendTester $I, Example $testData)
{
$this->runInputFieldTest($I, $testData);
}
+ */
/**
* @param BackendTester $I
@@ -227,7 +231,7 @@ class ElementsBasicInputDateCest extends AbstractElementsBasicCest
/*
public function checkThatValidationWorks_EvalDateTime_DbTypeDateTime(AcceptanceTester $I)
{
- // @todo fix these unstable test
+ // @todo fix these unstable tests
$dataSets = [
'input_37 dbType=datetime eval=datetime' => [
[
--
GitLab