From 7594d8949aafeb44cc4cd84cc51c1fdd3b950ee7 Mon Sep 17 00:00:00 2001
From: Christian Kuhn <lolli@schwarzbu.ch>
Date: Fri, 20 Jul 2018 14:11:26 +0200
Subject: [PATCH] [TASK] Enable functional tests on microsoft sql server again
Executing functional tests on mssql has been disabled a while
ago since the daemon crashed frequently: The server can not
run on tmpfs volumes that we usually use to speed up database
operations. sql server uses O_DIRECT i/o system calls which
tmpfs does not support. Find details at
https://github.com/Microsoft/mssql-docker/issues/12
We now run mssql tests on hard disks again, but enable them
only as nightly since they are too slow as pre-merge tests.
Change-Id: Iefc61670dcf74012fcdaada8e5b771538e7cf8c3
Resolves: #85614
Releases: master
Reviewed-on: https://review.typo3.org/57641
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
Tested-by: Christian Kuhn <lolli@schwarzbu.ch>
---
.../src/main/java/core/AbstractCoreSpec.java | 95 +++++++++++++++----
.../src/main/java/core/NightlySpec.java | 11 ++-
.../src/main/java/core/PreMergeSpec.java | 44 +--------
.../testing-docker/bamboo/docker-compose.yml | 28 ++++++
.../core/Classes/Database/Connection.php | 10 +-
.../Functional/Error/ErrorHandlerTest.php | 5 +-
6 files changed, 129 insertions(+), 64 deletions(-)
diff --git a/Build/bamboo/src/main/java/core/AbstractCoreSpec.java b/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
index 305e2a7d8e5e..76a4a1d9c817 100644
--- a/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
+++ b/Build/bamboo/src/main/java/core/AbstractCoreSpec.java
@@ -49,18 +49,6 @@ abstract public class AbstractCoreSpec {
protected String testingFrameworkBuildPath = "vendor/typo3/testing-framework/Resources/Core/Build/";
- /**
- * @todo This can be removed if acceptance mssql functional tests work again
- */
- protected String credentialsMssql =
- "typo3DatabaseDriver=\"sqlsrv\"" +
- " typo3DatabaseName=\"func\"" +
- " typo3DatabasePassword='Test1234!'" +
- " typo3DatabaseUsername=\"SA\"" +
- " typo3DatabaseHost=\"localhost\"" +
- " typo3DatabasePort=\"1433\"" +
- " typo3DatabaseCharset=\"utf-8\"";
-
/**
* Default permissions on core plans
*
@@ -132,6 +120,45 @@ abstract public class AbstractCoreSpec {
);
}
+ /**
+ * Job creating labels needed for intercept communication
+ */
+ protected Job getJobBuildLabels() {
+ return new Job("Create build labels", new BambooKey("CLFB"))
+ .description("Create changeId and patch set labels from variable access and parsing result of a dummy task")
+ .pluginConfigurations(new AllOtherPluginsConfiguration()
+ .configuration(new MapBuilder()
+ .put("repositoryDefiningWorkingDirectory", -1)
+ .put("custom", new MapBuilder()
+ .put("auto", new MapBuilder()
+ .put("regex", "https:\\/\\/review\\.typo3\\.org\\/(#\\/c\\/)?(\\d+)")
+ .put("label", "change-\\2\\, patchset-${bamboo.patchset}")
+ .build()
+ )
+ .put("buildHangingConfig.enabled", "false")
+ .put("ncover.path", "")
+ .put("clover", new MapBuilder()
+ .put("path", "")
+ .put("license", "")
+ .put("useLocalLicenseKey", "true")
+ .build()
+ )
+ .build()
+ )
+ .build()
+ )
+ )
+ .tasks(
+ new ScriptTask()
+ .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+ .inlineBody("echo \"I'm just here for the labels!\"")
+ )
+ .requirements(
+ this.getRequirementDocker10()
+ )
+ .cleanWorkingDirectory(true);
+ }
+
/**
* Job composer validate
*
@@ -462,8 +489,6 @@ abstract public class AbstractCoreSpec {
/**
* Jobs for mssql based functional tests
*
- * @todo Currently disabled and broken
- *
* @param int numberOfChunks
* @param String requirementIdentifier
*/
@@ -482,15 +507,37 @@ abstract public class AbstractCoreSpec {
this.getTaskGitCloneRepository(),
this.getTaskGitCherryPick(),
this.getTaskComposerInstall(requirementIdentifier),
+ this.getTaskDockerDependenciesFunctionalMssql(),
this.getTaskSplitFunctionalJobs(numberOfChunks, requirementIdentifier),
new ScriptTask()
.description("Run phpunit with functional chunk " + formattedI)
.interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
.inlineBody(
this.getScriptTaskBashInlineBody() +
- "./bin/phpunit --exclude-group not-mssql --log-junit test-reports/phpunit.xml -c " + this.testingFrameworkBuildPath + "FunctionalTests-Job-" + i + ".xml"
+ "function phpunit() {\n" +
+ " docker run \\\n" +
+ " -u ${HOST_UID} \\\n" +
+ " -v /bamboo-data/${BAMBOO_COMPOSE_PROJECT_NAME}/passwd:/etc/passwd \\\n" +
+ " -v ${BAMBOO_COMPOSE_PROJECT_NAME}_bamboo-data:/srv/bamboo/xml-data/build-dir/ \\\n" +
+ " -e typo3DatabaseDriver=sqlsrv \\\n" +
+ " -e typo3DatabaseName=func \\\n" +
+ " -e typo3DatabasePassword=Test1234! \\\n" +
+ " -e typo3DatabaseUsername=SA \\\n" +
+ " -e typo3DatabaseHost=localhost \\\n" +
+ " -e typo3DatabasePort=1433 \\\n" +
+ " -e typo3DatabaseCharset=utf-8 \\\n" +
+ " -e typo3DatabaseHost=mssql2017cu9 \\\n" +
+ " -e typo3TestingRedisHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_redis4_1 \\\n" +
+ " -e typo3TestingMemcachedHost=${BAMBOO_COMPOSE_PROJECT_NAME}sib_memcached1-5_1 \\\n" +
+ " --name ${BAMBOO_COMPOSE_PROJECT_NAME}sib_adhoc \\\n" +
+ " --network ${BAMBOO_COMPOSE_PROJECT_NAME}_test \\\n" +
+ " --rm \\\n" +
+ " typo3gmbh/" + requirementIdentifier.toLowerCase() + ":latest \\\n" +
+ " bin/bash -c \"cd ${PWD}; ./bin/phpunit $*\"\n" +
+ "}\n" +
+ "\n" +
+ "phpunit --exclude-group not-mssql --log-junit test-reports/phpunit.xml -c " + this.testingFrameworkBuildPath + "FunctionalTests-Job-" + i + ".xml"
)
- .environmentVariables(this.credentialsMssql)
)
.finalTasks(
this.getTaskStopDockerDependencies(),
@@ -501,7 +548,6 @@ abstract public class AbstractCoreSpec {
this.getRequirementDocker10()
)
.cleanWorkingDirectory(true)
- .enabled(false)
);
}
@@ -1263,6 +1309,21 @@ abstract public class AbstractCoreSpec {
);
}
+ /**
+ * Start docker sibling containers to execute functional tests on mariadb
+ */
+ protected Task getTaskDockerDependenciesFunctionalMssql() {
+ return new ScriptTask()
+ .description("Start docker siblings for functional tests on mssql")
+ .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
+ .inlineBody(
+ this.getScriptTaskBashInlineBody() +
+ "cd Build/testing-docker/bamboo\n" +
+ "echo COMPOSE_PROJECT_NAME=${BAMBOO_COMPOSE_PROJECT_NAME}sib > .env\n" +
+ "docker-compose run start_dependencies_functional_mssql"
+ );
+ }
+
/**
* Start docker sibling containers to execute functional tests on postgres
*/
diff --git a/Build/bamboo/src/main/java/core/NightlySpec.java b/Build/bamboo/src/main/java/core/NightlySpec.java
index b67a67d7fe90..383613761368 100644
--- a/Build/bamboo/src/main/java/core/NightlySpec.java
+++ b/Build/bamboo/src/main/java/core/NightlySpec.java
@@ -40,9 +40,9 @@ public class NightlySpec extends AbstractCoreSpec {
protected int numberOfAcceptanceTestJobs = 8;
protected int numberOfFunctionalMysqlJobs = 6;
- protected int numberOfFunctionalMssqlJobs = 6;
+ protected int numberOfFunctionalMssqlJobs = 16;
protected int numberOfFunctionalPgsqlJobs = 6;
- protected int numberOfFunctionalSqliteJobs = 10;
+ protected int numberOfFunctionalSqliteJobs = 6;
protected int numberOfUnitRandomOrderJobs = 4;
/**
@@ -66,6 +66,12 @@ public class NightlySpec extends AbstractCoreSpec {
* Returns full Plan definition
*/
Plan createPlan() {
+ // PREPARATION stage
+ ArrayList<Job> jobsPreparationStage = new ArrayList<Job>();
+ jobsPreparationStage.add(this.getJobBuildLabels());
+ Stage stagePreparation = new Stage("Preparation")
+ .jobs(jobsPreparationStage.toArray(new Job[jobsPreparationStage.size()]));
+
// MAIN stage
ArrayList<Job> jobsMainStage = new ArrayList<Job>();
@@ -107,6 +113,7 @@ public class NightlySpec extends AbstractCoreSpec {
.description("Execute TYPO3 core master nightly tests. Auto generated! See Build/bamboo of core git repository.")
.pluginConfigurations(this.getDefaultPlanPluginConfiguration())
.stages(
+ stagePreparation,
stageMainStage
)
.linkedRepositories("github TYPO3 TYPO3.CMS")
diff --git a/Build/bamboo/src/main/java/core/PreMergeSpec.java b/Build/bamboo/src/main/java/core/PreMergeSpec.java
index 9f01fb7cb06e..eb507a90fd3b 100644
--- a/Build/bamboo/src/main/java/core/PreMergeSpec.java
+++ b/Build/bamboo/src/main/java/core/PreMergeSpec.java
@@ -26,7 +26,6 @@ import com.atlassian.bamboo.specs.api.builders.plan.Plan;
import com.atlassian.bamboo.specs.api.builders.plan.Stage;
import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
-import com.atlassian.bamboo.specs.api.builders.plan.configuration.AllOtherPluginsConfiguration;
import com.atlassian.bamboo.specs.api.builders.project.Project;
import com.atlassian.bamboo.specs.api.builders.requirement.Requirement;
import com.atlassian.bamboo.specs.builders.notification.PlanCompletedNotification;
@@ -35,7 +34,6 @@ import com.atlassian.bamboo.specs.builders.trigger.RemoteTrigger;
import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties;
import com.atlassian.bamboo.specs.util.BambooServer;
-import com.atlassian.bamboo.specs.util.MapBuilder;
/**
* Core master pre-merge test plan.
@@ -101,7 +99,8 @@ public class PreMergeSpec extends AbstractCoreSpec {
jobsMainStage.add(this.getJobIntegrationVarious("PHP72"));
jobsMainStage.addAll(this.getJobsFunctionalTestsMysql(this.numberOfFunctionalMysqlJobs, "PHP72"));
- jobsMainStage.addAll(this.getJobsFunctionalTestsMssql(this.numberOfFunctionalMssqlJobs, "PHP72"));
+ // mssql functionals are not executed as pre-merge
+ // jobsMainStage.addAll(this.getJobsFunctionalTestsMssql(this.numberOfFunctionalMssqlJobs, "PHP72"));
jobsMainStage.addAll(this.getJobsFunctionalTestsPgsql(this.numberOfFunctionalPgsqlJobs, "PHP72"));
jobsMainStage.addAll(this.getJobsFunctionalTestsSqlite(this.numberOfFunctionalSqliteJobs, "PHP72"));
@@ -152,45 +151,6 @@ public class PreMergeSpec extends AbstractCoreSpec {
);
}
- /**
- * Job creating labels needed for intercept communication
- */
- protected Job getJobBuildLabels() {
- return new Job("Create build labels", new BambooKey("CLFB"))
- .description("Create changeId and patch set labels from variable access and parsing result of a dummy task")
- .pluginConfigurations(new AllOtherPluginsConfiguration()
- .configuration(new MapBuilder()
- .put("repositoryDefiningWorkingDirectory", -1)
- .put("custom", new MapBuilder()
- .put("auto", new MapBuilder()
- .put("regex", "https:\\/\\/review\\.typo3\\.org\\/(#\\/c\\/)?(\\d+)")
- .put("label", "change-\\2\\, patchset-${bamboo.patchset}")
- .build()
- )
- .put("buildHangingConfig.enabled", "false")
- .put("ncover.path", "")
- .put("clover", new MapBuilder()
- .put("path", "")
- .put("license", "")
- .put("useLocalLicenseKey", "true")
- .build()
- )
- .build()
- )
- .build()
- )
- )
- .tasks(
- new ScriptTask()
- .interpreter(ScriptTaskProperties.Interpreter.BINSH_OR_CMDEXE)
- .inlineBody("echo \"I'm just here for the labels!\"")
- )
- .requirements(
- this.getRequirementDocker10()
- )
- .cleanWorkingDirectory(true);
- }
-
/**
* Job checking CGL of last git commit
*
diff --git a/Build/testing-docker/bamboo/docker-compose.yml b/Build/testing-docker/bamboo/docker-compose.yml
index 346bf7142e5c..b54052ad7612 100644
--- a/Build/testing-docker/bamboo/docker-compose.yml
+++ b/Build/testing-docker/bamboo/docker-compose.yml
@@ -23,6 +23,16 @@ services:
- /var/lib/postgresql/data:rw,noexec,nosuid
networks:
- test
+ mssql2017cu9:
+ image: microsoft/mssql-server-linux:2017-CU9
+ environment:
+ ACCEPT_EULA: Y
+ SA_PASSWORD: Test1234!
+ MSSQL_PID: Developer
+ volumes:
+ - /etc/passwd:/etc/passwd,ro
+ networks:
+ - test
redis4:
image: redis:4-alpine
networks:
@@ -123,6 +133,24 @@ services:
echo Connected!;
"
+ start_dependencies_functional_mssql:
+ image: alpine:3.8
+ links:
+ - mssql2017cu9
+ - redis4
+ - memcached1-5
+ networks:
+ - test
+ command: >
+ /bin/sh -c "
+ echo Waiting for db start...;
+ while ! nc -z mssql2017cu9 1433;
+ do
+ sleep 1;
+ done;
+ echo Connected!;
+ "
+
start_dependencies_functional_postgres10:
image: alpine:3.8
links:
diff --git a/typo3/sysext/core/Classes/Database/Connection.php b/typo3/sysext/core/Classes/Database/Connection.php
index f39d7683674d..2c239f9f2ee5 100644
--- a/typo3/sysext/core/Classes/Database/Connection.php
+++ b/typo3/sysext/core/Classes/Database/Connection.php
@@ -20,6 +20,7 @@ use Doctrine\DBAL\Configuration;
use Doctrine\DBAL\Driver;
use Doctrine\DBAL\Driver\Statement;
use Doctrine\DBAL\Platforms\PostgreSqlPlatform;
+use Doctrine\DBAL\Platforms\SQLServer2012Platform;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;
use TYPO3\CMS\Core\Database\Query\Expression\ExpressionBuilder;
@@ -448,9 +449,16 @@ class Connection extends \Doctrine\DBAL\Connection implements LoggerAwareInterfa
*/
public function lastInsertId($tableName = null, string $fieldName = 'uid'): string
{
- if ($this->getDatabasePlatform() instanceof PostgreSqlPlatform) {
+ $databasePlatform = $this->getDatabasePlatform();
+ if ($databasePlatform instanceof PostgreSqlPlatform) {
return parent::lastInsertId(trim(implode('_', [$tableName, $fieldName, 'seq']), '_'));
}
+ if ($databasePlatform instanceof SQLServer2012Platform) {
+ // lastInsertId() in mssql >2012 takes a sequence name and not the table name as
+ // argument. If no argument is given, last insert id of latest table is returned.
+ // https://docs.microsoft.com/de-de/sql/connect/php/pdo-lastinsertid?view=sql-server-2017
+ return (string)parent::lastInsertId();
+ }
return (string)parent::lastInsertId($tableName);
}
diff --git a/typo3/sysext/core/Tests/Functional/Error/ErrorHandlerTest.php b/typo3/sysext/core/Tests/Functional/Error/ErrorHandlerTest.php
index 1833e47dbada..f7d31312b1f8 100644
--- a/typo3/sysext/core/Tests/Functional/Error/ErrorHandlerTest.php
+++ b/typo3/sysext/core/Tests/Functional/Error/ErrorHandlerTest.php
@@ -36,11 +36,12 @@ class ErrorHandlerTest extends FunctionalTestCase
];
/**
- * Disabled on sqlite: It does not support init command "SET NAMES 'UTF8'". That's ok
- * since this test is not about db platform support but error handling in core.
+ * Disabled on sqlite and mssql: They don't support init command "SET NAMES 'UTF8'". That's
+ * ok since this test is not about db platform support but error handling in core.
*
* @test
* @group not-sqlite
+ * @group not-mssql
*/
public function handleErrorFetchesDeprecations()
{
--
GitLab