From f80f5fd040796ef12c241b3b81990a5d0a0548a5 Mon Sep 17 00:00:00 2001
From: Susanne Moog <susanne.moog@typo3.org>
Date: Thu, 31 May 2018 13:53:56 +0200
Subject: [PATCH] [BUGFIX] Fix extension data import on sqlite

Sqlite limits the maximum number of variables
allowed in a query. When importing the extension
list the chunks are too big (> 999 variables).

This has been chunked to enable import on sqlite.

Resolves: #85129
Releases: master
Change-Id: If65476707946f4078a1b1e55128239aed07f6ed8
Reviewed-on: https://review.typo3.org/57103
Reviewed-by: Oliver Hader <oliver.hader@typo3.org>
Tested-by: Oliver Hader <oliver.hader@typo3.org>
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Tested-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Reviewed-by: Mathias Brodala <mbrodala@pagemachine.de>
Reviewed-by: Andreas Fernandez <a.fernandez@scripting-base.de>
Tested-by: Andreas Fernandez <a.fernandez@scripting-base.de>
---
 .../Domain/Repository/ExtensionRepository.php | 26 +++++++++-----
 .../Utility/Importer/ExtensionListUtility.php | 36 ++++++++++++++++---
 2 files changed, 49 insertions(+), 13 deletions(-)

diff --git a/typo3/sysext/extensionmanager/Classes/Domain/Repository/ExtensionRepository.php b/typo3/sysext/extensionmanager/Classes/Domain/Repository/ExtensionRepository.php
index 54d0c71cfe1e..276eadc14c0e 100644
--- a/typo3/sysext/extensionmanager/Classes/Domain/Repository/ExtensionRepository.php
+++ b/typo3/sysext/extensionmanager/Classes/Domain/Repository/ExtensionRepository.php
@@ -16,6 +16,7 @@ namespace TYPO3\CMS\Extensionmanager\Domain\Repository;
 
 use TYPO3\CMS\Core\Database\Connection;
 use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Platform\PlatformInformation;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
@@ -338,17 +339,24 @@ class ExtensionRepository extends \TYPO3\CMS\Extbase\Persistence\Repository
         $uidsOfCurrentVersion = $this->fetchMaximalVersionsForAllExtensions($repositoryUid);
         $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
             ->getQueryBuilderForTable(self::TABLE_NAME);
+        $connection = GeneralUtility::makeInstance(ConnectionPool::class)
+            ->getConnectionForTable(self::TABLE_NAME);
+        $maxBindParameters = PlatformInformation::getMaxBindParameters(
+            $connection->getDatabasePlatform()
+        );
 
-        $queryBuilder
-            ->update(self::TABLE_NAME)
-            ->where(
-                $queryBuilder->expr()->in(
-                    'uid',
-                    $queryBuilder->createNamedParameter($uidsOfCurrentVersion, Connection::PARAM_INT_ARRAY)
+        foreach (array_chunk($uidsOfCurrentVersion, $maxBindParameters - 10) as $chunk) {
+            $queryBuilder
+                ->update(self::TABLE_NAME)
+                ->where(
+                    $queryBuilder->expr()->in(
+                        'uid',
+                        $queryBuilder->createNamedParameter($chunk, Connection::PARAM_INT_ARRAY)
+                    )
                 )
-            )
-            ->set('current_version', 1)
-            ->execute();
+                ->set('current_version', 1)
+                ->execute();
+        }
     }
 
     /**
diff --git a/typo3/sysext/extensionmanager/Classes/Utility/Importer/ExtensionListUtility.php b/typo3/sysext/extensionmanager/Classes/Utility/Importer/ExtensionListUtility.php
index 8242e11ac537..e8b23c113410 100644
--- a/typo3/sysext/extensionmanager/Classes/Utility/Importer/ExtensionListUtility.php
+++ b/typo3/sysext/extensionmanager/Classes/Utility/Importer/ExtensionListUtility.php
@@ -14,6 +14,7 @@ namespace TYPO3\CMS\Extensionmanager\Utility\Importer;
  * The TYPO3 project - inspiring people to share!
  */
 use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Platform\PlatformInformation;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\VersionNumberUtility;
 use TYPO3\CMS\Extensionmanager\Utility\Parser\AbstractExtensionXmlParser;
@@ -72,6 +73,21 @@ class ExtensionListUtility implements \SplObserver
         'update_comment'
     ];
 
+    /**
+     * Table name to be used to store extension models.
+     *
+     * @var string
+     */
+    protected static $tableName = 'tx_extensionmanager_domain_model_extension';
+
+    /**
+     * Maximum of rows that can be used in a bulk insert for the current
+     * database platform.
+     *
+     * @var int
+     */
+    protected $maxRowsPerChunk = 50;
+
     /**
      * Keeps indexes of fields that should not be quoted.
      *
@@ -132,6 +148,19 @@ class ExtensionListUtility implements \SplObserver
                 1476108717
             );
         }
+
+        $connection = GeneralUtility::makeInstance(ConnectionPool::class)
+            ->getConnectionForTable(self::$tableName);
+        $maxBindParameters = PlatformInformation::getMaxBindParameters(
+            $connection->getDatabasePlatform()
+        );
+        $countOfBindParamsPerRow = count(self::$fieldNames);
+        // flush at least chunks of 50 elements - in case the currently used
+        // database platform does not support that, the threshold is lowered
+        $this->maxRowsPerChunk = min(
+            $this->maxRowsPerChunk,
+            floor($maxBindParameters / $countOfBindParamsPerRow)
+        );
     }
 
     /**
@@ -171,12 +200,11 @@ class ExtensionListUtility implements \SplObserver
      */
     protected function loadIntoDatabase(AbstractExtensionXmlParser &$subject)
     {
-        // flush every 50 rows to database
-        if ($this->sumRecords !== 0 && $this->sumRecords % 50 === 0) {
+        if ($this->sumRecords !== 0 && $this->sumRecords % $this->maxRowsPerChunk === 0) {
             GeneralUtility::makeInstance(ConnectionPool::class)
-                ->getConnectionForTable('tx_extensionmanager_domain_model_extension')
+                ->getConnectionForTable(self::$tableName)
                 ->bulkInsert(
-                    'tx_extensionmanager_domain_model_extension',
+                    self::$tableName,
                     $this->arrRows,
                     self::$fieldNames
                 );
-- 
GitLab