[BUGFIX] Fix extension installation process

The download queue is build recursively, but if an
extension is marked for download, it is added
to the queue before its dependencies have been
resolved, which leads to a wrong download and
installation order of extensions.

We also need to add dependency resolving when
marking an extension for installation to fix
the exact same problem when extensions already
reside in the system.

Lastly we must take care of flushed class loader
caches and trigger a rebuild to avoid fatals.
This is done by introducing a signal and registering
a method in package manager as slot that set the
packages for the class loader to trigger a rebuild
of the caches.

The parts of this patch that fix dependency handling
should be backported to older 6.x branches.

Resolves: #57199
Releases: 6.2
Change-Id: Iab343c544bfe2e3e19cbf4c05090eb4994df57b1
Reviewed-on: https://review.typo3.org/28660
Reviewed-by: Sebastian Fischer
Reviewed-by: Philipp Gampe
Tested-by: Philipp Gampe
Reviewed-by: Christian Kuhn
Reviewed-by: Xavier Perseguers
Tested-by: Xavier Perseguers
Reviewed-by: Markus Klein
Tested-by: Markus Klein

typo3/sysext/core/Classes/Package/PackageManager.php

@@ -156,6 +156,19 @@ class PackageManager extends \TYPO3\Flow\Package\PackageManager implements \TYPO
 		}
 	}
 
+	/**
+	 * Updates the class loader with currently active packages.
+	 * This method is currently a slot that monitors the after
+	 * extension is installed signal to make the class loader
+	 * populate its caches again.
+	 * Maybe we find a better solution in the future, but as of now
+	 * we have to do this as all caches are flushed after an extension
+	 * is installed and the current request might fail otherwise.
+	 */
+	public function updatePackagesForClassLoader() {
+		$this->classLoader->setPackages($this->activePackages);
+	}
+
 	/**
 	 * @return PackageFactory
 	 */

typo3/sysext/extensionmanager/Classes/Service/ExtensionManagementService.php

@@ -51,6 +51,12 @@ class ExtensionManagementService implements \TYPO3\CMS\Core\SingletonInterface {
 	 */
 	protected $installUtility;
 
+	/**
+	 * @var \TYPO3\CMS\Extensionmanager\Utility\ExtensionModelUtility
+	 * @inject
+	 */
+	protected $extensionModelUtility;
+
 	/**
 	 * @var \TYPO3\CMS\Extensionmanager\Utility\ListUtility
 	 * @inject
@@ -63,11 +69,24 @@ class ExtensionManagementService implements \TYPO3\CMS\Core\SingletonInterface {
 	 */
 	protected $downloadUtility;
 
+	/**
+	 * @var \TYPO3\CMS\Core\Package\PackageManager
+	 * @inject
+	 */
+	protected $packageManager;
+
 	/**
 	 * @param string $extensionKey
 	 * @return void
 	 */
 	public function markExtensionForInstallation($extensionKey) {
+		// We have to check for dependencies of the extension first, before marking it for installation
+		// because this extension might have dependencies, which need to be installed first
+		$this->dependencyUtility->buildExtensionDependenciesTree(
+			$this->extensionModelUtility->mapExtensionArrayToModel(
+				$this->installUtility->enrichExtensionWithDetails($extensionKey)
+			)
+		);
 		$this->downloadQueue->addExtensionToInstallQueue($extensionKey);
 	}
 
@@ -89,8 +108,10 @@ class ExtensionManagementService implements \TYPO3\CMS\Core\SingletonInterface {
 	 * @return void
 	 */
 	public function markExtensionForDownload(\TYPO3\CMS\Extensionmanager\Domain\Model\Extension $extension) {
-		$this->downloadQueue->addExtensionToQueue($extension);
+		// We have to check for dependencies of the extension first, before marking it for download
+		// because this extension might have dependencies, which need to be downloaded and installed first
 		$this->dependencyUtility->buildExtensionDependenciesTree($extension);
+		$this->downloadQueue->addExtensionToQueue($extension);
 	}
 
 	/**
@@ -98,8 +119,10 @@ class ExtensionManagementService implements \TYPO3\CMS\Core\SingletonInterface {
 	 * @return void
 	 */
 	public function markExtensionForUpdate(\TYPO3\CMS\Extensionmanager\Domain\Model\Extension $extension) {
-		$this->downloadQueue->addExtensionToQueue($extension, 'update');
+		// We have to check for dependencies of the extension first, before marking it for download
+		// because this extension might have dependencies, which need to be downloaded and installed first
 		$this->dependencyUtility->buildExtensionDependenciesTree($extension);
+		$this->downloadQueue->addExtensionToQueue($extension, 'update');
 	}
 
 	/**
@@ -110,7 +133,7 @@ class ExtensionManagementService implements \TYPO3\CMS\Core\SingletonInterface {
 	 * @return array
 	 */
 	public function resolveDependenciesAndInstall(\TYPO3\CMS\Extensionmanager\Domain\Model\Extension $extension) {
-		$downloadedDependencies = $this->downloadMainExtension($extension);
+		$this->downloadMainExtension($extension);
 		$extensionKey = $extension->getExtensionKey();
 		$this->setInExtensionRepository($extensionKey);
 		$this->dependencyUtility->buildExtensionDependenciesTree($extension);
@@ -123,9 +146,9 @@ class ExtensionManagementService implements \TYPO3\CMS\Core\SingletonInterface {
 		if (count($copyQueue) > 0) {
 			$this->copyDependencies($copyQueue);
 		}
-
+		$downloadedDependencies = array();
 		if (array_key_exists('download', $queue)) {
-			$downloadedDependencies = array_merge($downloadedDependencies, $this->downloadDependencies($queue['download']));
+			$downloadedDependencies = $this->downloadDependencies($queue['download']);
 		}
 		if (array_key_exists('update', $queue)) {
 			$this->downloadDependencies($queue['update']);
@@ -200,6 +223,7 @@ class ExtensionManagementService implements \TYPO3\CMS\Core\SingletonInterface {
 		$resolvedDependencies = array();
 		foreach ($installQueue as $extensionKey => $extensionDetails) {
 			$this->installUtility->install($extensionDetails);
+			$this->emitHasInstalledExtension($extensionDetails);
 			if (!is_array($resolvedDependencies['installed'])) {
 				$resolvedDependencies['installed'] = array();
 			}
@@ -247,16 +271,12 @@ class ExtensionManagementService implements \TYPO3\CMS\Core\SingletonInterface {
 	 * as an extension is able to provide it's own dependencies
 	 *
 	 * @param \TYPO3\CMS\Extensionmanager\Domain\Model\Extension $extension
-	 * @return array
+	 * @return void
 	 */
 	public function downloadMainExtension(\TYPO3\CMS\Extensionmanager\Domain\Model\Extension $extension) {
-		$downloadedDependencies = array();
-		if ($extension->getUid()) {
-			$this->downloadQueue->addExtensionToQueue($extension);
-			$queue = $this->downloadQueue->getExtensionQueue();
-			$downloadedDependencies = $this->downloadDependencies($queue['download']);
+		if (!$this->packageManager->isPackageAvailable($extension->getExtensionKey())) {
+			$this->downloadUtility->download($extension);
 		}
-		return $downloadedDependencies;
 	}
 
 	/**
@@ -266,6 +286,13 @@ class ExtensionManagementService implements \TYPO3\CMS\Core\SingletonInterface {
 		$this->getSignalSlotDispatcher()->dispatch(__CLASS__, 'willInstallExtensions', array($installQueue));
 	}
 
+	/**
+	 * @param string $extensionKey
+	 */
+	protected function emitHasInstalledExtension($extensionKey) {
+		$this->getSignalSlotDispatcher()->dispatch(__CLASS__, 'hasInstalledExtensions', array($extensionKey));
+	}
+
 	/**
 	 * Get the SignalSlot dispatcher
 	 *

typo3/sysext/extensionmanager/ext_localconf.php

@@ -21,6 +21,12 @@ if (TYPO3_MODE === 'BE') {
 			'TYPO3\\CMS\\Core\\Package\\PackageManager',
 			'scanAvailablePackages'
 		);
+		$signalSlotDispatcher->connect(
+			'TYPO3\\CMS\\Extensionmanager\\Service\\ExtensionManagementService',
+			'hasInstalledExtensions',
+			'TYPO3\\CMS\\Core\\Package\\PackageManager',
+			'updatePackagesForClassLoader'
+		);
 		$signalSlotDispatcher->connect(
 			'TYPO3\\CMS\\Extensionmanager\\Utility\\InstallUtility',
 			'tablesDefinitionIsBeingBuilt',