From 1450b10f6e4a96b5fb76158387de8f7178a9e163 Mon Sep 17 00:00:00 2001
From: Alina Fleser <afleser@arxia.com>
Date: Sat, 31 Jan 2015 17:57:55 +0200
Subject: [PATCH] [BUGFIX] Processing folder has now nested subfolders
In case of big installations the amount of processed files in
one folder is too much and causes performance issues and other
problems.
To prevent this kind of issues, nested subfolders are now
created in the processing folder in order to split the processed
files into more than one folder.
Resolves: #56557
Releases: master, 7.6
Change-Id: Id9a4fc3b4bb4b28ca26ff96221097171c3835eb7
Reviewed-on: https://review.typo3.org/36523
Reviewed-by: Frank Naegler <frank.naegler@typo3.org>
Tested-by: Frank Naegler <frank.naegler@typo3.org>
Reviewed-by: Philipp Gampe <philipp.gampe@typo3.org>
Tested-by: Philipp Gampe <philipp.gampe@typo3.org>
---
.../core/Classes/Resource/ProcessedFile.php | 4 +-
.../Processing/LocalImageProcessor.php | 2 +-
.../core/Classes/Resource/ResourceStorage.php | 68 +++++++++++++++++--
.../Unit/Resource/ResourceStorageTest.php | 32 +++++++++
4 files changed, 99 insertions(+), 7 deletions(-)
diff --git a/typo3/sysext/core/Classes/Resource/ProcessedFile.php b/typo3/sysext/core/Classes/Resource/ProcessedFile.php
index 10f4ed9cf46d..0412dd4b1c62 100644
--- a/typo3/sysext/core/Classes/Resource/ProcessedFile.php
+++ b/typo3/sysext/core/Classes/Resource/ProcessedFile.php
@@ -184,7 +184,7 @@ class ProcessedFile extends AbstractFile
if ($this->identifier === null) {
throw new \RuntimeException('Cannot update original file!', 1350582054);
}
- $processingFolder = $this->originalFile->getStorage()->getProcessingFolder();
+ $processingFolder = $this->originalFile->getStorage()->getProcessingFolder($this->originalFile);
$addedFile = $this->storage->updateProcessedFile($filePath, $this, $processingFolder);
// Update some related properties
@@ -256,7 +256,7 @@ class ProcessedFile extends AbstractFile
$this->name = $name;
// @todo this is a *weird* hack that will fail if the storage is non-hierarchical!
- $this->identifier = $this->storage->getProcessingFolder()->getIdentifier() . $this->name;
+ $this->identifier = $this->storage->getProcessingFolder($this->originalFile)->getIdentifier() . $this->name;
$this->updated = true;
}
diff --git a/typo3/sysext/core/Classes/Resource/Processing/LocalImageProcessor.php b/typo3/sysext/core/Classes/Resource/Processing/LocalImageProcessor.php
index 10c559bfad3e..b6b222ed2575 100644
--- a/typo3/sysext/core/Classes/Resource/Processing/LocalImageProcessor.php
+++ b/typo3/sysext/core/Classes/Resource/Processing/LocalImageProcessor.php
@@ -108,7 +108,7 @@ class LocalImageProcessor implements ProcessorInterface
{
// the storage of the processed file, not of the original file!
$storage = $task->getTargetFile()->getStorage();
- $processingFolder = $storage->getProcessingFolder();
+ $processingFolder = $storage->getProcessingFolder($task->getSourceFile());
// explicitly check for the raw filename here, as we check for files that existed before we even started
// processing, i.e. that were processed earlier
diff --git a/typo3/sysext/core/Classes/Resource/ResourceStorage.php b/typo3/sysext/core/Classes/Resource/ResourceStorage.php
index bf210e77f987..b8bb97357ddc 100644
--- a/typo3/sysext/core/Classes/Resource/ResourceStorage.php
+++ b/typo3/sysext/core/Classes/Resource/ResourceStorage.php
@@ -154,6 +154,11 @@ class ResourceStorage implements ResourceStorageInterface
*/
protected $fileAndFolderNameFilters = array();
+ /**
+ * Levels numbers used to generate hashed subfolders in the processing folder
+ */
+ const PROCESSING_FOLDER_LEVELS = 2;
+
/**
* Constructor for a storage object.
*
@@ -1204,10 +1209,9 @@ class ResourceStorage implements ResourceStorageInterface
throw new \InvalidArgumentException('File "' . $localFilePath . '" does not exist.', 1319552746);
}
if ($processingFolder === null) {
- $processingFolder = $this->getProcessingFolder();
+ $processingFolder = $this->getProcessingFolder($processedFile->getOriginalFile());
}
$fileIdentifier = $this->driver->addFile($localFilePath, $processingFolder->getIdentifier(), $processedFile->getName());
-
// @todo check if we have to update the processed file other then the identifier
$processedFile->setIdentifier($fileIdentifier);
return $processedFile;
@@ -2856,9 +2860,10 @@ class ResourceStorage implements ResourceStorageInterface
* Getter function to return the folder where the files can
* be processed. Does not check for access rights here.
*
+ * @param File $file Specific file you want to have the processing folder for
* @return Folder
*/
- public function getProcessingFolder()
+ public function getProcessingFolder(File $file = null)
{
if (!isset($this->processingFolder)) {
$processingFolder = self::DEFAULT_ProcessingFolder;
@@ -2892,7 +2897,62 @@ class ResourceStorage implements ResourceStorageInterface
);
}
}
- return $this->processingFolder;
+
+ $processingFolder = $this->processingFolder;
+ if (!empty($file)) {
+ $processingFolder = $this->getNestedProcessingFolder($file, $processingFolder);
+ }
+ return $processingFolder;
+ }
+
+ /**
+ * Getter function to return the the file's corresponding hashed subfolder
+ * of the processed folder
+ *
+ * @param File $file
+ * @param Folder $rootProcessingFolder
+ * @return Folder
+ * @throws Exception\InsufficientFolderWritePermissionsException
+ */
+ protected function getNestedProcessingFolder(File $file, Folder $rootProcessingFolder)
+ {
+ $processingFolder = $rootProcessingFolder;
+ $nestedFolderNames = $this->getNamesForNestedProcessingFolder(
+ $file->getIdentifier(),
+ self::PROCESSING_FOLDER_LEVELS
+ );
+
+ try {
+ foreach ($nestedFolderNames as $folderName) {
+ if ($processingFolder->hasFolder($folderName)) {
+ $processingFolder = $processingFolder->getSubfolder($folderName);
+ } else {
+ $processingFolder = $processingFolder->createFolder($folderName);
+ }
+ }
+ } catch (Exception\FolderDoesNotExistException $e) {}
+
+ return $processingFolder;
+ }
+
+ /**
+ * Generates appropriate hashed sub-folder path for a given file identifier
+ *
+ * @param string $fileIdentifier
+ * @param int $levels
+ * @return []
+ */
+ protected function getNamesForNestedProcessingFolder($fileIdentifier, $levels)
+ {
+ $names = [];
+ if ($levels === 0) {
+ return $names;
+ }
+ $hash = md5($fileIdentifier);
+ for ($i = 1; $i <= $levels; $i++) {
+ $names[] = substr($hash, $i, 1);
+ }
+ return $names;
}
/**
diff --git a/typo3/sysext/core/Tests/Unit/Resource/ResourceStorageTest.php b/typo3/sysext/core/Tests/Unit/Resource/ResourceStorageTest.php
index 335747306847..c8bdeac0f605 100644
--- a/typo3/sysext/core/Tests/Unit/Resource/ResourceStorageTest.php
+++ b/typo3/sysext/core/Tests/Unit/Resource/ResourceStorageTest.php
@@ -752,4 +752,36 @@ class ResourceStorageTest extends BaseTestCase
$this->assertSame(FolderInterface::ROLE_DEFAULT, $role);
}
+
+ /**
+ * @test
+ */
+ public function getProcessingRootFolderTest()
+ {
+ $this->prepareSubject(array());
+ $processingFolder = $this->subject->getProcessingFolder();
+
+ $this->assertInstanceOf(Folder::class, $processingFolder);
+ }
+
+ /**
+ * @test
+ */
+ public function getNestedProcessingFolderTest()
+ {
+ $mockedDriver = $this->createDriverMock(array('basePath' => $this->getMountRootUrl()), null, null);
+ $this->prepareSubject(array(), true, $mockedDriver);
+ $mockedFile = $this->getSimpleFileMock('/someFile');
+
+ $rootProcessingFolder = $this->subject->getProcessingFolder();
+ $processingFolder = $this->subject->getProcessingFolder($mockedFile);
+
+ $this->assertInstanceOf(Folder::class, $processingFolder);
+ $this->assertNotEquals($rootProcessingFolder, $processingFolder);
+
+ for ($i = ResourceStorage::PROCESSING_FOLDER_LEVELS; $i>0; $i--) {
+ $processingFolder = $processingFolder->getParentFolder();
+ }
+ $this->assertEquals($rootProcessingFolder, $processingFolder);
+ }
}
--
GitLab