From 52dd9357e4d851a4e8418100bc1cf8cd711e4943 Mon Sep 17 00:00:00 2001
From: Benni Mack <benni@typo3.org>
Date: Mon, 14 Nov 2016 08:21:25 +0100
Subject: [PATCH] [!!!][TASK] Migrate lowlevel rte_images command to Symfony
Console
The CLI command to detect RTE magic images (images added
within the RTE) and (as far as possible) remove unused RTE images
as well as copy RTE images referenced multiple times, is migrated to
Symfony Console to better structure the CLI command and get rid
of dependencies of the old CLI command line tools (non-extbase).
The command was previously available under
./typo3/cli_dispatch lowlevel_cleaner rte_images
and is now called via
./typo3/sysext/core/bin/typo3 cleanup:rteimages
and allows the following options:
--dry-run - do not update the images and delete lost files, but just print them
--update-refindex - update the reference index, do not ask the user
Resolves: #78895
Releases: master
Change-Id: I6e30b7a32fe14bc89cebd85167961ec8fa996c54
Reviewed-on: https://review.typo3.org/50650
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Susanne Moog <susanne.moog@typo3.org>
Tested-by: Susanne Moog <susanne.moog@typo3.org>
Reviewed-by: Wouter Wolters <typo3@wouterwolters.nl>
Tested-by: Wouter Wolters <typo3@wouterwolters.nl>
---
...levelRteImagesCommandParametersChanged.rst | 42 ++
.../Classes/Command/RteImagesCommand.php | 381 ++++++++++++++++++
.../lowlevel/Classes/RteImagesCommand.php | 277 -------------
.../lowlevel/Configuration/Commands.php | 5 +
typo3/sysext/lowlevel/ext_localconf.php | 1 -
5 files changed, 428 insertions(+), 278 deletions(-)
create mode 100644 typo3/sysext/core/Documentation/Changelog/master/Breaking-78895-LowlevelRteImagesCommandParametersChanged.rst
create mode 100644 typo3/sysext/lowlevel/Classes/Command/RteImagesCommand.php
delete mode 100644 typo3/sysext/lowlevel/Classes/RteImagesCommand.php
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Breaking-78895-LowlevelRteImagesCommandParametersChanged.rst b/typo3/sysext/core/Documentation/Changelog/master/Breaking-78895-LowlevelRteImagesCommandParametersChanged.rst
new file mode 100644
index 000000000000..8cf8a4157660
--- /dev/null
+++ b/typo3/sysext/core/Documentation/Changelog/master/Breaking-78895-LowlevelRteImagesCommandParametersChanged.rst
@@ -0,0 +1,42 @@
+.. include:: ../../Includes.txt
+
+===============================================================
+Breaking: #78895 - Lowlevel RteImagesCommand parameters changed
+===============================================================
+
+See :issue:`78895`
+
+Description
+===========
+
+The existing CLI command within EXT:lowlevel for detecting and removing RTE files within uploads/ which are not referenced by TYPO3
+has been migrated to a Symfony Console command. The same command is also used to copy RTE images which are used multiple
+times on multiple references, to be only used once.
+
+The previously command available via `./typo3/cli_dispatch.phpsh lowlevel_cleaner rte_images` is now available via
+`./typo3/sysext/core/bin/typo3 cleanup:rteimages` and allows the following CLI options to be set:
+
+`--update-refindex` - updates the reference index before scanning for lost files. If not set, the user is asked if the task should be run
+`--dry-run` - do not copy / delete the files but only list the files that are not wrongly connected or not connected at all.
+
+The PHP class of the old CLI command `TYPO3\CMS\Lowlevel\RteImagesCommand` has been removed.
+
+
+Impact
+======
+
+Calling the old CLI command `./typo3/cli_dispatch.phpsh lowlevel_cleaner rte_images` will result in an error message.
+
+
+Affected Installations
+======================
+
+Any TYPO3 instances using the lowlevel cleaner for finding RTE image files.
+
+
+Migration
+=========
+
+Update the CLI call on your servers to the new command line and available options as shown above.
+
+.. index:: CLI
\ No newline at end of file
diff --git a/typo3/sysext/lowlevel/Classes/Command/RteImagesCommand.php b/typo3/sysext/lowlevel/Classes/Command/RteImagesCommand.php
new file mode 100644
index 000000000000..85213a381531
--- /dev/null
+++ b/typo3/sysext/lowlevel/Classes/Command/RteImagesCommand.php
@@ -0,0 +1,381 @@
+<?php
+declare(strict_types=1);
+namespace TYPO3\CMS\Lowlevel\Command;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\ReferenceIndex;
+use TYPO3\CMS\Core\Utility\File\BasicFileUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\PathUtility;
+
+/**
+ * Looking up all occurencies of RTEmagic images in the database and check existence of parent and
+ * copy files on the file system plus report possibly lost files of this type
+ */
+class RteImagesCommand extends Command
+{
+
+ /**
+ * Configure the command by defining the name, options and arguments
+ */
+ public function configure()
+ {
+ $this
+ ->setDescription('Looking up all occurrences of RTEmagic images in the database and check existence of parent and copy files on the file system plus report possibly lost RTE files.')
+ ->setHelp('
+Assumptions:
+- a perfect integrity of the reference index table (always update the reference index table before using this tool!)
+- that all RTEmagic image files in the database are registered with the soft reference parser "images"
+- images found in deleted records are included (means that you might find lost RTEmagic images after flushing deleted records)
+
+The assumptions are not requirements by the TYPO3 API but reflects the de facto implementation of most TYPO3 installations.
+However, many custom fields using an RTE will probably not have the "images" soft reference parser registered and so the index will be incomplete and not listing all RTEmagic image files.
+The consequence of this limitation is that you should be careful if you wish to delete lost RTEmagic images - they could be referenced from a field not parsed by the "images" soft reference parser!
+
+Automatic Repair of Errors:
+- Will search for double-usages of RTEmagic images and make copies as required.
+- Lost files can be deleted automatically, but it is recommended to delete them manually if you do not recognize them as used somewhere the system does not know about.
+
+Manual repair suggestions:
+- Missing files: Re-insert missing files or edit record where the reference is found.
+
+If the option "--dry-run" is not set, the files are then deleted automatically.
+Warning: First, make sure those files are not used somewhere TYPO3 does not know about! See the assumptions above.
+
+If you want to get more detailed information, use the --verbose option.')
+ ->addOption(
+ 'dry-run',
+ null,
+ InputOption::VALUE_NONE,
+ 'If this option is set, the files will not actually be deleted, but just the output which files would be deleted are shown'
+ )
+ ->addOption(
+ 'update-refindex',
+ null,
+ InputOption::VALUE_NONE,
+ 'Setting this option automatically updates the reference index and does not ask on command line. Alternatively, use -n to avoid the interactive mode'
+ );
+ }
+
+ /**
+ * Executes the command to
+ * - optionally update the reference index (to have clean data)
+ * - find files within uploads/* which are not connected to the reference index
+ * - remove these files if --dry-run is not set
+ *
+ * @param InputInterface $input
+ * @param OutputInterface $output
+ *
+ * @return void
+ */
+ protected function execute(InputInterface $input, OutputInterface $output)
+ {
+ $io = new SymfonyStyle($input, $output);
+ $io->title($this->getDescription());
+
+ $dryRun = $input->hasOption('dry-run') && $input->getOption('dry-run') != false ? true : false;
+
+ $this->updateReferenceIndex($input, $io);
+
+ // Find the RTE files
+ $allRteImagesInUse = $this->findAllReferencedRteImagesWithOriginals();
+
+ if (count($allRteImagesInUse)) {
+ $allRteImagesWithOriginals = [];
+ $multipleReferenced = [];
+ $missingFiles = [];
+ $lostFiles = [];
+
+ // Searching for duplicates, and missing files (also missing originals)
+ foreach ($allRteImagesInUse as $fileName => $fileInfo) {
+ $allRteImagesWithOriginals[$fileName]++;
+ $allRteImagesWithOriginals[$fileInfo['original']]++;
+ if ($fileInfo['count'] > 1 && $fileInfo['exists'] && $fileInfo['original_exists']) {
+ $multipleReferenced[$fileName] = $fileInfo['softReferences'];
+ }
+ // Missing files:
+ if (!$fileInfo['exists']) {
+ $missingFiles[$fileName] = $fileInfo['softReferences'];
+ }
+ if (!$fileInfo['original_exists']) {
+ $missingFiles[$fileInfo['original']] = $fileInfo['softReferences'];
+ }
+ }
+
+ // Now, ask for RTEmagic files inside uploads/ folder:
+ $magicFiles = $this->findAllRteFilesInDirectory();
+ foreach ($magicFiles as $fileName) {
+ if (!isset($allRteImagesWithOriginals[$fileName])) {
+ $lostFiles[$fileName] = $fileName;
+ }
+ }
+ ksort($missingFiles);
+ ksort($multipleReferenced);
+
+ // Output info about missing files
+ if (!$io->isQuiet()) {
+ $io->note('Found ' . count($missingFiles) . ' RTE images that are referenced, but missing.');
+ if ($io->isVerbose()) {
+ $io->listing($missingFiles);
+ }
+ }
+
+ // Duplicate RTEmagic image files
+ // These files are RTEmagic images found used in multiple records! RTEmagic images should be used by only
+ // one record at a time. A large amount of such images probably stems from previous versions of TYPO3 (before 4.2)
+ // which did not support making copies automatically of RTEmagic images in case of new copies / versions.
+ $this->copyMultipleReferencedRteImages($multipleReferenced, $dryRun, $io);
+
+ // Delete lost files
+ // Lost RTEmagic files from uploads/
+ // These files you might be able to delete but only if _all_ RTEmagic images are found by the soft reference parser.
+ // If you are using the RTE in third-party extensions it is likely that the soft reference parser is not applied
+ // correctly to their RTE and thus these "lost" files actually represent valid RTEmagic images,
+ // just not registered. Lost files can be auto-fixed but only if you specifically
+ // set "lostFiles" as parameter to the --AUTOFIX option.
+ if (count($lostFiles)) {
+ ksort($lostFiles);
+ $this->deleteLostFiles($lostFiles, $dryRun, $io);
+ $io->success('Deleted ' . count($lostFiles) . ' lost files.');
+ }
+ } else {
+ $io->success('Nothing to do, your system does not have any RTE images.');
+ }
+ }
+
+ /**
+ * Function to update the reference index
+ * - if the option --update-refindex is set, do it
+ * - otherwise, if in interactive mode (not having -n set), ask the user
+ * - otherwise assume everything is fine
+ *
+ * @param InputInterface $input holds information about entered parameters
+ * @param SymfonyStyle $io necessary for outputting information
+ * @return void
+ */
+ protected function updateReferenceIndex(InputInterface $input, SymfonyStyle $io)
+ {
+ // Check for reference index to update
+ $io->note('Finding RTE images used in TYPO3 requires a clean reference index (sys_refindex)');
+ $updateReferenceIndex = false;
+ if ($input->hasOption('update-refindex') && $input->getOption('update-refindex')) {
+ $updateReferenceIndex = true;
+ } elseif ($input->isInteractive()) {
+ $updateReferenceIndex = $io->confirm('Should the reference index be updated right now?', false);
+ }
+
+ // Update the reference index
+ if ($updateReferenceIndex) {
+ $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
+ $referenceIndex->updateIndex(false, !$io->isQuiet());
+ } else {
+ $io->writeln('Reference index is assumed to be up to date, continuing.');
+ }
+ }
+
+ /**
+ * Find lost files in uploads/ folder
+ *
+ * @return array an array of files (relative to PATH_site) that are not connected
+ */
+ protected function findAllReferencedRteImagesWithOriginals(): array
+ {
+ $allRteImagesInUse = [];
+
+ // Select all RTEmagic files in the reference table (only from soft references of course)
+ $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
+ ->getQueryBuilderForTable('sys_refindex');
+
+ $result = $queryBuilder
+ ->select('*')
+ ->from('sys_refindex')
+ ->where(
+ $queryBuilder->expr()->eq(
+ 'ref_table',
+ $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)
+ ),
+ $queryBuilder->expr()->like(
+ 'ref_string',
+ $queryBuilder->createNamedParameter('%/RTEmagic%', \PDO::PARAM_STR)
+ ),
+ $queryBuilder->expr()->eq(
+ 'softref_key',
+ $queryBuilder->createNamedParameter('images', \PDO::PARAM_STR)
+ )
+ )
+ ->execute();
+
+ // Traverse the files and put into a large table:
+ while ($rec = $result->fetch()) {
+ $file = $rec['ref_string'];
+ $filename = basename($file);
+ if (strpos($filename, 'RTEmagicC_') === 0) {
+ // First time the file is referenced => build index
+ if (!is_array($allRteImagesInUse[$file])) {
+ $original = 'RTEmagicP_' . preg_replace('/\\.[[:alnum:]]+$/', '', substr($filename, 10));
+ $original = substr($file, 0, -strlen($filename)) . $original;
+ $allRteImagesInUse[$file] = [
+ 'exists' => @is_file(PATH_site . $file),
+ 'original' => $original,
+ 'original_exists' => @is_file(PATH_site . $original),
+ 'count' => 0,
+ 'softReferences' => []
+ ];
+ }
+ $allRteImagesInUse[$file]['count']++;
+ $allRteImagesInUse[$file]['softReferences'][$rec['hash']] = $this->formatReferenceIndexEntryToString($rec);
+ }
+ }
+
+ ksort($allRteImagesInUse);
+ return $allRteImagesInUse;
+ }
+
+ /**
+ * Find all RTE files in uploads/ folder
+ *
+ * @param string $folder the name of the folder to start from
+ * @return array an array of files (relative to PATH_site) that are not connected
+ */
+ protected function findAllRteFilesInDirectory($folder = 'uploads/'): array
+ {
+ $filesFound = [];
+
+ // Get all files
+ $files = [];
+ $files = GeneralUtility::getAllFilesAndFoldersInPath($files, PATH_site . $folder);
+ $files = GeneralUtility::removePrefixPathFromList($files, PATH_site);
+
+ // Traverse files
+ foreach ($files as $key => $value) {
+ // If the file is a RTEmagic-image name
+ if (preg_match('/^RTEmagic[P|C]_/', basename($value))) {
+ $filesFound[] = $value;
+ continue;
+ }
+ }
+
+ return $filesFound;
+ }
+
+ /**
+ * Removes given files from the uploads/ folder
+ *
+ * @param array $lostFiles Contains the lost files found
+ * @param bool $dryRun if set, the files are just displayed, but not deleted
+ * @param SymfonyStyle $io the IO object for output
+ * @return void
+ */
+ protected function deleteLostFiles(array $lostFiles, bool $dryRun, SymfonyStyle $io)
+ {
+ foreach ($lostFiles as $lostFile) {
+ $absoluteFileName = GeneralUtility::getFileAbsFileName($lostFile);
+ if ($io->isVeryVerbose()) {
+ $io->writeln('Deleting file "' . $absoluteFileName . '"');
+ }
+ if (!$dryRun) {
+ if ($absoluteFileName && @is_file($absoluteFileName)) {
+ unlink($absoluteFileName);
+ if (!$io->isQuiet()) {
+ $io->writeln('Permanently deleted file "' . $absoluteFileName . '".');
+ }
+ } else {
+ $io->error('File "' . $absoluteFileName . '" was not found!');
+ }
+ }
+ }
+ }
+
+ /**
+ * Duplicate RTEmagic image files which are used on several records. RTEmagic images should be used by only
+ * one record at a time. A large amount of such images probably stems from previous versions of TYPO3 (before 4.2)
+ * which did not support making copies automatically of RTEmagic images in case of new copies / versions.
+ *
+ * @param array $multipleReferencedImages
+ * @param bool $dryRun
+ * @param SymfonyStyle $io
+ */
+ protected function copyMultipleReferencedRteImages(array $multipleReferencedImages, bool $dryRun, SymfonyStyle $io)
+ {
+ $fileProcObj = GeneralUtility::makeInstance(BasicFileUtility::class);
+ foreach ($multipleReferencedImages as $fileName => $fileInfo) {
+ // Traverse all records using the file
+ $c = 0;
+ foreach ($fileInfo['usedIn'] as $hash => $recordID) {
+ if ($c === 0) {
+ $io->writeln('Keeping file ' . $fileName . ' for record ' . $recordID);
+ } else {
+ $io->writeln('Copying file ' . basename($fileName) . ' for record ' . $recordID);
+ // Get directory prefix for file and set the original name
+ $dirPrefix = dirname($fileName) . '/';
+ $rteOrigName = basename($fileInfo['original']);
+ // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
+ if ($rteOrigName && strpos($dirPrefix, 'uploads/') === 0 && @is_dir((PATH_site . $dirPrefix))) {
+ // From the "original" RTE filename, produce a new "original" destination filename which is unused.
+ $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
+ // Create copy file name
+ $pI = pathinfo($fileName);
+ $copyDestName = dirname($origDestName) . '/RTEmagicC_' . substr(basename($origDestName), 10) . '.' . $pI['extension'];
+ if (!@is_file($copyDestName) && !@is_file($origDestName) && $origDestName === GeneralUtility::getFileAbsFileName($origDestName) && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)) {
+ $io->writeln('Copying file ' . basename($fileName) . ' for record ' . $recordID . ' to ' . basename($copyDestName));
+ if (!$dryRun) {
+ // Making copies
+ GeneralUtility::upload_copy_move(PATH_site . $fileInfo['original'], $origDestName);
+ GeneralUtility::upload_copy_move(PATH_site . $fileName, $copyDestName);
+ clearstatcache();
+ if (@is_file($copyDestName)) {
+ $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
+ $error = $referenceIndex->setReferenceValue($hash, PathUtility::stripPathSitePrefix($copyDestName));
+ if ($error) {
+ $io->error('ReferenceIndex::setReferenceValue() reported "' . $error . '"');
+ }
+ } else {
+ $io->error('File "' . $copyDestName . '" could not be created.');
+ }
+ }
+ } else {
+ $io->error('Could not construct new unique names for file.');
+ }
+ } else {
+ $io->error('Maybe directory of file was not within "uploads/"?');
+ }
+ }
+ $c++;
+ }
+ }
+ }
+
+ /**
+ * Formats a sys_refindex entry to something readable
+ *
+ * @param array $record
+ * @return string
+ */
+ protected function formatReferenceIndexEntryToString(array $record): string
+ {
+ return $record['tablename']
+ . ':' . $record['recuid']
+ . ':' . $record['field']
+ . ($record['flexpointer'] ? ':' . $record['flexpointer'] : '')
+ . ($record['softref_key'] ? ':' . $record['softref_key'] . ' (Soft Reference) ' : '')
+ . ($record['deleted'] ? ' (DELETED)' : '');
+ }
+}
diff --git a/typo3/sysext/lowlevel/Classes/RteImagesCommand.php b/typo3/sysext/lowlevel/Classes/RteImagesCommand.php
deleted file mode 100644
index 7b412d1adaab..000000000000
--- a/typo3/sysext/lowlevel/Classes/RteImagesCommand.php
+++ /dev/null
@@ -1,277 +0,0 @@
-<?php
-namespace TYPO3\CMS\Lowlevel;
-
-/*
- * This file is part of the TYPO3 CMS project.
- *
- * It is free software; you can redistribute it and/or modify it under
- * the terms of the GNU General Public License, either version 2
- * of the License, or any later version.
- *
- * For the full copyright and license information, please read the
- * LICENSE.txt file that was distributed with this source code.
- *
- * The TYPO3 project - inspiring people to share!
- */
-
-use TYPO3\CMS\Core\Database\ConnectionPool;
-use TYPO3\CMS\Core\Database\ReferenceIndex;
-use TYPO3\CMS\Core\Utility\File\ExtendedFileUtility;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
-use TYPO3\CMS\Core\Utility\PathUtility;
-
-/**
- * Looking for RTE images integrity
- */
-class RteImagesCommand extends CleanerCommand
-{
- /**
- * @var bool
- */
- public $checkRefIndex = true;
-
- /**
- * @var ExtendedFileUtility
- */
- protected $fileProcObj = null;
-
- /**
- * Constructor
- */
- public function __construct()
- {
- parent::__construct();
- // Setting up help:
- $this->cli_help['name'] = 'rte_images -- Looking up all occurencies of RTEmagic images in the database and check existence of parent and copy files on the file system plus report possibly lost files of this type.';
- $this->cli_help['description'] = trim('
-Assumptions:
-- a perfect integrity of the reference index table (always update the reference index table before using this tool!)
-- that all RTEmagic image files in the database are registered with the soft reference parser "images"
-- images found in deleted records are included (means that you might find lost RTEmagic images after flushing deleted records)
-
-The assumptions are not requirements by the TYPO3 API but reflects the de facto implementation of most TYPO3 installations.
-However, many custom fields using an RTE will probably not have the "images" soft reference parser registered and so the index will be incomplete and not listing all RTEmagic image files.
-The consequence of this limitation is that you should be careful if you wish to delete lost RTEmagic images - they could be referenced from a field not parsed by the "images" soft reference parser!
-
-Automatic Repair of Errors:
-- Will search for double-usages of RTEmagic images and make copies as required.
-- Lost files can be deleted automatically by setting the value "lostFiles" as an optional parameter to --AUTOFIX, but otherwise delete them manually if you do not recognize them as used somewhere the system does not know about.
-
-Manual repair suggestions:
-- Missing files: Re-insert missing files or edit record where the reference is found.
-');
- $this->cli_help['examples'] = '/.../cli_dispatch.phpsh lowlevel_cleaner rte_images -s -r
-Reports problems with RTE images';
- }
-
- /**
- * Analyse situation with RTE magic images. (still to define what the most useful output is).
- * Fix methods: API in \TYPO3\CMS\Core\Database\ReferenceIndex that allows to
- * change the value of a reference (we could copy the files) or remove reference
- *
- * @return array
- */
- public function main()
- {
- // Initialize result array:
- $resultArray = [
- 'message' => $this->cli_help['name'] . LF . LF . $this->cli_help['description'],
- 'headers' => [
- 'completeFileList' => ['Complete list of used RTEmagic files', 'Both parent and copy are listed here including usage count (which should in theory all be "1"). This list does not exclude files that might be missing.', 1],
- 'RTEmagicFilePairs' => ['Statistical info about RTEmagic files', '(copy used as index)', 0],
- 'doubleFiles' => ['Duplicate RTEmagic image files', 'These files are RTEmagic images found used in multiple records! RTEmagic images should be used by only one record at a time. A large amount of such images probably stems from previous versions of TYPO3 (before 4.2) which did not support making copies automatically of RTEmagic images in case of new copies / versions.', 3],
- 'missingFiles' => ['Missing RTEmagic image files', 'These files are not found in the file system! Should be corrected!', 3],
- 'lostFiles' => ['Lost RTEmagic files from uploads/', 'These files you might be able to delete but only if _all_ RTEmagic images are found by the soft reference parser. If you are using the RTE in third-party extensions it is likely that the soft reference parser is not applied correctly to their RTE and thus these "lost" files actually represent valid RTEmagic images, just not registered. Lost files can be auto-fixed but only if you specifically set "lostFiles" as parameter to the --AUTOFIX option.', 2]
- ],
- 'RTEmagicFilePairs' => [],
- 'doubleFiles' => [],
- 'completeFileList' => [],
- 'missingFiles' => [],
- 'lostFiles' => []
- ];
- // Select all RTEmagic files in the reference table (only from soft references of course)
- $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
- ->getQueryBuilderForTable('sys_refindex');
-
- $result = $queryBuilder
- ->select('*')
- ->from('sys_refindex')
- ->where(
- $queryBuilder->expr()->eq(
- 'ref_table',
- $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)
- ),
- $queryBuilder->expr()->like(
- 'ref_string',
- $queryBuilder->createNamedParameter('%/RTEmagic%', \PDO::PARAM_STR)
- ),
- $queryBuilder->expr()->eq(
- 'softref_key',
- $queryBuilder->createNamedParameter('images', \PDO::PARAM_STR)
- )
- )
- ->orderBy('sorting', 'DESC')
- ->execute();
-
- // Traverse the files and put into a large table:
-
- while ($rec = $result->fetch()) {
- $filename = basename($rec['ref_string']);
- if (GeneralUtility::isFirstPartOfStr($filename, 'RTEmagicC_')) {
- $original = 'RTEmagicP_' . preg_replace('/\\.[[:alnum:]]+$/', '', substr($filename, 10));
- $infoString = $this->infoStr($rec);
- // Build index:
- $resultArray['RTEmagicFilePairs'][$rec['ref_string']]['exists'] = @is_file((PATH_site . $rec['ref_string']));
- $resultArray['RTEmagicFilePairs'][$rec['ref_string']]['original'] = substr($rec['ref_string'], 0, -strlen($filename)) . $original;
- $resultArray['RTEmagicFilePairs'][$rec['ref_string']]['original_exists'] = @is_file((PATH_site . $resultArray['RTEmagicFilePairs'][$rec['ref_string']]['original']));
- $resultArray['RTEmagicFilePairs'][$rec['ref_string']]['count']++;
- $resultArray['RTEmagicFilePairs'][$rec['ref_string']]['usedIn'][$rec['hash']] = $infoString;
- $resultArray['completeFileList'][$resultArray['RTEmagicFilePairs'][$rec['ref_string']]['original']]++;
- $resultArray['completeFileList'][$rec['ref_string']]++;
- // Missing files:
- if (!$resultArray['RTEmagicFilePairs'][$rec['ref_string']]['exists']) {
- $resultArray['missingFiles'][$rec['ref_string']] = $resultArray['RTEmagicFilePairs'][$rec['ref_string']]['usedIn'];
- }
- if (!$resultArray['RTEmagicFilePairs'][$rec['ref_string']]['original_exists']) {
- $resultArray['missingFiles'][$resultArray['RTEmagicFilePairs'][$rec['ref_string']]['original']] = $resultArray['RTEmagicFilePairs'][$rec['ref_string']]['usedIn'];
- }
- }
- }
- // Searching for duplicates:
- foreach ($resultArray['RTEmagicFilePairs'] as $fileName => $fileInfo) {
- if ($fileInfo['count'] > 1 && $fileInfo['exists'] && $fileInfo['original_exists']) {
- $resultArray['doubleFiles'][$fileName] = $fileInfo['usedIn'];
- }
- }
-
- // Now, ask for RTEmagic files inside uploads/ folder:
- $cleanerModules = $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules'];
- $cleanerMode = GeneralUtility::getUserObj($cleanerModules['lost_files'][0]);
- $resLostFiles = $cleanerMode->main([], false, true);
- if (is_array($resLostFiles['RTEmagicFiles'])) {
- foreach ($resLostFiles['RTEmagicFiles'] as $fileName) {
- if (!isset($resultArray['completeFileList'][$fileName])) {
- $resultArray['lostFiles'][$fileName] = $fileName;
- }
- }
- }
- ksort($resultArray['RTEmagicFilePairs']);
- ksort($resultArray['completeFileList']);
- ksort($resultArray['missingFiles']);
- ksort($resultArray['doubleFiles']);
- ksort($resultArray['lostFiles']);
- return $resultArray;
- }
-
- /**
- * Mandatory autofix function
- * Will run auto-fix on the result array. Echos status during processing.
- *
- * @param array $resultArray Result array from main() function
- * @return void
- */
- public function main_autoFix($resultArray)
- {
- $limitTo = $this->cli_args['--AUTOFIX'][0];
- if (is_array($resultArray['doubleFiles'])) {
- if (!$limitTo || $limitTo === 'doubleFiles') {
- echo 'FIXING double-usages of RTE files in uploads/: ' . LF;
- foreach ($resultArray['RTEmagicFilePairs'] as $fileName => $fileInfo) {
- // Only fix something if there is a usage count of more than 1 plus if both original and copy exists:
- if ($fileInfo['count'] > 1 && $fileInfo['exists'] && $fileInfo['original_exists']) {
- // Traverse all records using the file:
- $c = 0;
- foreach ($fileInfo['usedIn'] as $hash => $recordID) {
- if ($c == 0) {
- echo ' Keeping file ' . $fileName . ' for record ' . $recordID . LF;
- } else {
- // CODE below is adapted from \TYPO3\CMS\Impexp\ImportExport where there is support for duplication of RTE images:
- echo ' Copying file ' . basename($fileName) . ' for record ' . $recordID . ' ';
- // Initialize; Get directory prefix for file and set the original name:
- $dirPrefix = dirname($fileName) . '/';
- $rteOrigName = basename($fileInfo['original']);
- // If filename looks like an RTE file, and the directory is in "uploads/", then process as a RTE file!
- if ($rteOrigName && GeneralUtility::isFirstPartOfStr($dirPrefix, 'uploads/') && @is_dir((PATH_site . $dirPrefix))) {
- // RTE:
- // From the "original" RTE filename, produce a new "original" destination filename which is unused.
- $fileProcObj = $this->getFileProcObj();
- $origDestName = $fileProcObj->getUniqueName($rteOrigName, PATH_site . $dirPrefix);
- // Create copy file name:
- $pI = pathinfo($fileName);
- $copyDestName = dirname($origDestName) . '/RTEmagicC_' . substr(basename($origDestName), 10) . '.' . $pI['extension'];
- if (!@is_file($copyDestName) && !@is_file($origDestName) && $origDestName === GeneralUtility::getFileAbsFileName($origDestName) && $copyDestName === GeneralUtility::getFileAbsFileName($copyDestName)) {
- echo ' to ' . basename($copyDestName);
- if ($bypass = $this->cli_noExecutionCheck($fileName)) {
- echo $bypass;
- } else {
- // Making copies:
- GeneralUtility::upload_copy_move(PATH_site . $fileInfo['original'], $origDestName);
- GeneralUtility::upload_copy_move(PATH_site . $fileName, $copyDestName);
- clearstatcache();
- if (@is_file($copyDestName)) {
- $sysRefObj = GeneralUtility::makeInstance(ReferenceIndex::class);
- $error = $sysRefObj->setReferenceValue($hash, PathUtility::stripPathSitePrefix($copyDestName));
- if ($error) {
- echo ' - ERROR: TYPO3\\CMS\\Core\\Database\\ReferenceIndex::setReferenceValue(): ' . $error . LF;
- die;
- } else {
- echo ' - DONE';
- }
- } else {
- echo ' - ERROR: File "' . $copyDestName . '" was not created!';
- }
- }
- } else {
- echo ' - ERROR: Could not construct new unique names for file!';
- }
- } else {
- echo ' - ERROR: Maybe directory of file was not within "uploads/"?';
- }
- echo LF;
- }
- $c++;
- }
- }
- }
- } else {
- echo 'Bypassing fixing of double-usages since --AUTOFIX was not "doubleFiles"' . LF;
- }
- }
- if (is_array($resultArray['lostFiles'])) {
- if ($limitTo === 'lostFiles') {
- echo 'Removing lost RTEmagic files from folders inside uploads/: ' . LF;
- foreach ($resultArray['lostFiles'] as $key => $value) {
- $absFileName = GeneralUtility::getFileAbsFileName($value);
- echo 'Deleting file: "' . $absFileName . '": ';
- if ($bypass = $this->cli_noExecutionCheck($absFileName)) {
- echo $bypass;
- } else {
- if ($absFileName && @is_file($absFileName)) {
- unlink($absFileName);
- echo 'DONE';
- } else {
- echo ' ERROR: File "' . $absFileName . '" was not found!';
- }
- }
- echo LF;
- }
- }
- } else {
- echo 'Bypassing fixing of double-usages since --AUTOFIX was not "lostFiles"' . LF;
- }
- }
-
- /**
- * Returns file processing object, initialized only once.
- *
- * @return ExtendedFileUtility File processor object
- */
- public function getFileProcObj()
- {
- if (!is_object($this->fileProcObj)) {
- $this->fileProcObj = GeneralUtility::makeInstance(ExtendedFileUtility::class);
- $this->fileProcObj->setActionPermissions();
- }
- return $this->fileProcObj;
- }
-}
diff --git a/typo3/sysext/lowlevel/Configuration/Commands.php b/typo3/sysext/lowlevel/Configuration/Commands.php
index 2496e084f807..7b669c264700 100644
--- a/typo3/sysext/lowlevel/Configuration/Commands.php
+++ b/typo3/sysext/lowlevel/Configuration/Commands.php
@@ -25,6 +25,11 @@ return [
// needed for updating the reference index (optional)
'user' => '_cli_lowlevel'
],
+ 'cleanup:rteimages' => [
+ 'class' => \TYPO3\CMS\Lowlevel\Command\RteImagesCommand::class,
+ // needed for updating the reference index (optional)
+ 'user' => '_cli_lowlevel'
+ ],
'cleanup:missingrelations' => [
'class' => \TYPO3\CMS\Lowlevel\Command\MissingRelationsCommand::class,
// needed for updating the reference index (optional)
diff --git a/typo3/sysext/lowlevel/ext_localconf.php b/typo3/sysext/lowlevel/ext_localconf.php
index 98412fe329e8..23b68b5d6888 100644
--- a/typo3/sysext/lowlevel/ext_localconf.php
+++ b/typo3/sysext/lowlevel/ext_localconf.php
@@ -10,6 +10,5 @@ if (TYPO3_MODE === 'BE') {
},
'_CLI_lowlevel'
];
- $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['rte_images'] = [\TYPO3\CMS\Lowlevel\RteImagesCommand::class];
$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['versions'] = [\TYPO3\CMS\Lowlevel\VersionsCommand::class];
}
--
GitLab