diff --git a/typo3/sysext/core/Documentation/Changelog/master/Breaking-78627-LowlevelDoubleFilesCommandParametersChanged.rst b/typo3/sysext/core/Documentation/Changelog/master/Breaking-78627-LowlevelDoubleFilesCommandParametersChanged.rst
new file mode 100644
index 0000000000000000000000000000000000000000..ce23de865748f8dd8dd644f249328bf9d6c9245f
--- /dev/null
+++ b/typo3/sysext/core/Documentation/Changelog/master/Breaking-78627-LowlevelDoubleFilesCommandParametersChanged.rst
@@ -0,0 +1,41 @@
+.. include:: ../../Includes.txt
+
+======================================================================
+Breaking: #78627 - Lowlevel MissingRelationsCommand parameters changed
+======================================================================
+
+See :issue:`78627`
+
+Description
+===========
+
+The existing CLI command within EXT:lowlevel for showing files within uploads/ that are used by records twice (non-FAL)
+has been migrated to a Symfony Console command.
+
+The previously command available via `./typo3/cli_dispatch.phpsh lowlevel_cleaner double_files` is now available
+via `./typo3/sysext/core/bin/typo3 cleanup:multiplereferencedfiles` and allows the following CLI options to be set:
+
+`--update-refindex` - updates the reference index before scanning for multiple-referenced files. If not set, the user is asked if the task should be run
+`--dry-run` - do not copy the files to single-reference them, but only list the references and files.
+
+The PHP class of the old CLI command `TYPO3\CMS\Lowlevel\DoubleFilesCommand` has been removed.
+
+
+Impact
+======
+
+Calling the old CLI command `./typo3/cli_dispatch.phpsh lowlevel_cleaner double_files` will result in an error message.
+
+
+Affected Installations
+======================
+
+Any TYPO3 instances using the lowlevel cleaner for finding files with two records pointing to them.
+
+
+Migration
+=========
+
+Update the CLI call on your servers to the new command line and available options as shown above.
+
+.. index:: CLI
diff --git a/typo3/sysext/lowlevel/Classes/Command/FilesWithMultipleReferencesCommand.php b/typo3/sysext/lowlevel/Classes/Command/FilesWithMultipleReferencesCommand.php
new file mode 100644
index 0000000000000000000000000000000000000000..28d8cc0cb8e81229c2f8a2155b73f32bfc1e10ad
--- /dev/null
+++ b/typo3/sysext/lowlevel/Classes/Command/FilesWithMultipleReferencesCommand.php
@@ -0,0 +1,249 @@
+<?php
+declare(strict_types=1);
+namespace TYPO3\CMS\Lowlevel\Command;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\ReferenceIndex;
+use TYPO3\CMS\Core\Utility\ArrayUtility;
+use TYPO3\CMS\Core\Utility\File\BasicFileUtility;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\PathUtility;
+
+/**
+ * Finds files within uploads/ which are used multiple times by relations within the database
+ */
+class FilesWithMultipleReferencesCommand extends Command
+{
+
+ /**
+ * Configure the command by defining the name, options and arguments
+ */
+ public function configure()
+ {
+ $this
+ ->setDescription('Looking for files from TYPO3 managed records which are referenced more than once')
+ ->setHelp('
+Assumptions:
+- a perfect integrity of the reference index table (always update the reference index table before using this tool!)
+- files found in deleted records are included (otherwise you would see a false list of lost files)
+
+Files attached to records in TYPO3 using a "group" type configuration in TCA or FlexForm DataStructure are managed exclusively by the system and there must always exist a 1-1 reference between the file and the reference in the record.
+This tool will expose when such files are referenced from multiple locations which is considered an integrity error.
+If a multi-reference is found it was typically created because the record was copied or modified outside of DataHandler which will otherwise maintain the relations correctly.
+Multi-references should be resolved to 1-1 references as soon as possible. The danger of keeping multi-references is that if the file is removed from one of the referring records it will actually be deleted in the file system, leaving missing files for the remaining referers!
+
+If the option "--dry-run" is not set, the files that are referenced multiple times are copied with a new name
+and the references are updated accordingly.
+Warning: First, make sure those files are not used somewhere TYPO3 does not know about!
+
+If you want to get more detailed information, use the --verbose option.')
+ ->addOption(
+ 'dry-run',
+ null,
+ InputOption::VALUE_NONE,
+ 'If this option is set, the files will not actually be deleted, but just the output which files would be deleted are shown'
+ )
+ ->addOption(
+ 'update-refindex',
+ null,
+ InputOption::VALUE_NONE,
+ 'Setting this option automatically updates the reference index and does not ask on command line. Alternatively, use -n to avoid the interactive mode'
+ );
+ }
+
+ /**
+ * Executes the command to
+ * - optionally update the reference index (to have clean data)
+ * - find files within the reference index which are referenced more than once
+ * - copy these files if --dry-run is not set and update the references accordingly
+ *
+ * @param InputInterface $input
+ * @param OutputInterface $output
+ *
+ * @return void
+ */
+ protected function execute(InputInterface $input, OutputInterface $output)
+ {
+ $io = new SymfonyStyle($input, $output);
+ $io->title($this->getDescription());
+
+ $dryRun = $input->hasOption('dry-run') && $input->getOption('dry-run') != false ? true : false;
+
+ $this->updateReferenceIndex($input, $io);
+
+ // Find files which are referenced multiple times
+ $doubleFiles = $this->findMultipleReferencedFiles();
+
+ if (count($doubleFiles)) {
+ if (!$io->isQuiet()) {
+ $io->note('Found ' . count($doubleFiles) . ' files that are referenced more than once.');
+ if ($io->isVerbose()) {
+ $io->listing($doubleFiles);
+ }
+ }
+
+ $this->copyMultipleReferencedFiles($doubleFiles, $dryRun, $io);
+ $io->success('Cleaned up ' . count($doubleFiles) . ' files which have been referenced multiple times.');
+ } else {
+ $io->success('Nothing to do, no files found which are referenced more than once.');
+ }
+ }
+
+ /**
+ * Function to update the reference index
+ * - if the option --update-refindex is set, do it
+ * - otherwise, if in interactive mode (not having -n set), ask the user
+ * - otherwise assume everything is fine
+ *
+ * @param InputInterface $input holds information about entered parameters
+ * @param SymfonyStyle $io necessary for outputting information
+ * @return void
+ */
+ protected function updateReferenceIndex(InputInterface $input, SymfonyStyle $io)
+ {
+ // Check for reference index to update
+ $io->note('Finding files referenced multiple times in records managed by TYPO3 requires a clean reference index (sys_refindex)');
+ $updateReferenceIndex = false;
+ if ($input->hasOption('update-refindex') && $input->getOption('update-refindex')) {
+ $updateReferenceIndex = true;
+ } elseif ($input->isInteractive()) {
+ $updateReferenceIndex = $io->confirm('Should the reference index be updated right now?', false);
+ }
+
+ // Update the reference index
+ if ($updateReferenceIndex) {
+ $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
+ $referenceIndex->updateIndex(false, !$io->isQuiet());
+ } else {
+ $io->writeln('Reference index is assumed to be up to date, continuing.');
+ }
+ }
+
+ /**
+ * Find files which are referenced multiple times in uploads/ folder
+ *
+ * @return array an array of files and their reference hashes that are referenced multiple times
+ */
+ protected function findMultipleReferencedFiles(): array
+ {
+ $multipleReferencesList = [];
+
+ // Select all files in the reference table not found by a soft reference parser (thus TCA configured)
+ $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
+ ->getQueryBuilderForTable('sys_refindex');
+
+ $result = $queryBuilder
+ ->select('*')
+ ->from('sys_refindex')
+ ->where(
+ $queryBuilder->expr()->eq('ref_table', $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)),
+ $queryBuilder->expr()->eq('softref_key', $queryBuilder->createNamedParameter('', \PDO::PARAM_STR))
+ )
+ ->execute();
+
+ // Traverse the files and put into a large table
+ $allReferencesToFiles = [];
+ while ($record = $result->fetch()) {
+ // Compile info string for location of reference
+ $infoString = $this->formatReferenceIndexEntryToString($record);
+ $hash = $record['hash'];
+ $fileName = $record['ref_string'];
+ // Add entry if file has multiple references pointing to it
+ if (isset($allReferencesToFiles[$fileName])) {
+ if (!is_array($multipleReferencesList[$fileName])) {
+ $multipleReferencesList[$fileName] = [];
+ $multipleReferencesList[$fileName][$allReferencesToFiles[$fileName]['hash']] = $allReferencesToFiles[$fileName]['infoString'];
+ }
+ $multipleReferencesList[$fileName][$hash] = $infoString;
+ } else {
+ $allReferencesToFiles[$fileName] = [
+ 'infoString' => $infoString,
+ 'hash' => $hash
+ ];
+ }
+ }
+
+ return ArrayUtility::sortByKeyRecursive($multipleReferencesList);
+ }
+
+ /**
+ * Copies files which are referenced multiple times and updates the reference index so they are only used once
+ *
+ * @param array $multipleReferencesToFiles Contains files which have been referenced multiple times
+ * @param bool $dryRun if set, the info is just displayed, but no files are copied nor reference index updated
+ * @param SymfonyStyle $io the IO object for output
+ * @return void
+ */
+ protected function copyMultipleReferencedFiles(array $multipleReferencesToFiles, bool $dryRun, SymfonyStyle $io)
+ {
+ $fileFunc = GeneralUtility::makeInstance(BasicFileUtility::class);
+ $referenceIndex = GeneralUtility::makeInstance(ReferenceIndex::class);
+
+ foreach ($multipleReferencesToFiles as $fileName => $usages) {
+ $absoluteFileName = GeneralUtility::getFileAbsFileName($fileName);
+ if ($absoluteFileName && @is_file($absoluteFileName)) {
+ if ($io->isVeryVerbose()) {
+ $io->writeln('Processing file "' . $absoluteFileName . '"');
+ }
+ $counter = 0;
+ foreach ($usages as $hash => $recReference) {
+ if ($counter++ === 0) {
+ $io->writeln('Keeping "' . $fileName . '" for record "' . $recReference . '"');
+ } else {
+ // Create unique name for file
+ $newName = $fileFunc->getUniqueName(basename($fileName), dirname($absoluteFileName));
+ $io->writeln('Copying "' . $fileName . '" to "' . PathUtility::stripPathSitePrefix($newName) . '" for record "' . $recReference . '"');
+ if (!$dryRun) {
+ GeneralUtility::upload_copy_move($absoluteFileName, $newName);
+ clearstatcache();
+ if (@is_file($newName)) {
+ $error = $referenceIndex->setReferenceValue($hash, basename($newName));
+ if ($error) {
+ $io->error('ReferenceIndex::setReferenceValue() reported "' . $error . '"');
+ }
+ } else {
+ $io->error('File "' . $newName . '" could not be created.');
+ }
+ }
+ }
+ }
+ } else {
+ $io->error('File "' . $absoluteFileName . '" was not found.');
+ }
+ }
+ }
+
+ /**
+ * Formats a sys_refindex entry to something readable
+ *
+ * @param array $record
+ * @return string
+ */
+ protected function formatReferenceIndexEntryToString(array $record): string
+ {
+ return $record['tablename']
+ . ':' . $record['recuid']
+ . ':' . $record['field']
+ . ($record['flexpointer'] ? ':' . $record['flexpointer'] : '')
+ . ($record['softref_key'] ? ':' . $record['softref_key'] . ' (Soft Reference) ' : '')
+ . ($record['deleted'] ? ' (DELETED)' : '');
+ }
+}
diff --git a/typo3/sysext/lowlevel/Classes/DoubleFilesCommand.php b/typo3/sysext/lowlevel/Classes/DoubleFilesCommand.php
deleted file mode 100644
index a583292cc3f367e363142a4a6a0e07ff2ec324ad..0000000000000000000000000000000000000000
--- a/typo3/sysext/lowlevel/Classes/DoubleFilesCommand.php
+++ /dev/null
@@ -1,191 +0,0 @@
-<?php
-namespace TYPO3\CMS\Lowlevel;
-
-/*
- * This file is part of the TYPO3 CMS project.
- *
- * It is free software; you can redistribute it and/or modify it under
- * the terms of the GNU General Public License, either version 2
- * of the License, or any later version.
- *
- * For the full copyright and license information, please read the
- * LICENSE.txt file that was distributed with this source code.
- *
- * The TYPO3 project - inspiring people to share!
- */
-
-use TYPO3\CMS\Core\Database\ConnectionPool;
-use TYPO3\CMS\Core\Database\ReferenceIndex;
-use TYPO3\CMS\Core\Utility\File\BasicFileUtility;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
-
-/**
- * Looking for double files
- */
-class DoubleFilesCommand extends CleanerCommand
-{
- /**
- * @var bool
- */
- public $checkRefIndex = true;
-
- /**
- * Constructor
- */
- public function __construct()
- {
- parent::__construct();
- // Setting up help:
- $this->cli_help['name'] = 'double_files -- Looking for files from TYPO3 managed records which are referenced more than one time (only one time allowed)';
- $this->cli_help['description'] = trim('
-Assumptions:
-- a perfect integrity of the reference index table (always update the reference index table before using this tool!)
-- files found in deleted records are included (otherwise you would see a false list of lost files)
-
-Files attached to records in TYPO3 using a "group" type configuration in TCA or FlexForm DataStructure are managed exclusively by the system and there must always exist a 1-1 reference between the file and the reference in the record.
-This tool will expose when such files are referenced from multiple locations which is considered an integrity error.
-If a multi-reference is found it was typically created because the record was copied or modified outside of DataHandler which will otherwise maintain the relations correctly.
-Multi-references should be resolved to 1-1 references as soon as possible. The danger of keeping multi-references is that if the file is removed from one of the refering records it will actually be deleted in the file system, leaving missing files for the remaining referers!
-
-Automatic Repair of Errors:
-- The multi-referenced file is copied under a new name and references updated.
-
-Manual repair suggestions:
-- None that can not be handled by the automatic repair.
-');
- $this->cli_help['examples'] = '/.../cli_dispatch.phpsh lowlevel_cleaner double_files -s -r
-This will check the system for double files relations.';
- }
-
- /**
- * Find managed files which are referred to more than one time
- * Fix methods: API in \TYPO3\CMS\Core\Database\ReferenceIndex that allows to
- * change the value of a reference (we could copy the file) or remove reference
- *
- * @return array
- */
- public function main()
- {
- // Initialize result array:
- $resultArray = [
- 'message' => $this->cli_help['name'] . LF . LF . $this->cli_help['description'],
- 'headers' => [
- 'multipleReferencesList_count' => ['Number of multi-reference files', '(See below)', 0],
- 'singleReferencesList_count' => ['Number of files correctly referenced', 'The amount of correct 1-1 references', 0],
- 'multipleReferencesList' => ['Entries with files having multiple references', 'These are serious problems that should be resolved ASAP to prevent data loss! ' . $this->label_infoString, 3],
- 'dirname_registry' => ['Registry of directories in which files are found.', 'Registry includes which table/field pairs store files in them plus how many files their store.', 0],
- 'missingFiles' => ['Tracking missing files', '(Extra feature, not related to tracking of double references. Further, the list may include more files than found in the missing_files()-test because this list includes missing files from deleted records.)', 0],
- 'warnings' => ['Warnings picked up', '', 2]
- ],
- 'multipleReferencesList_count' => ['count' => 0],
- 'singleReferencesList_count' => ['count' => 0],
- 'multipleReferencesList' => [],
- 'dirname_registry' => [],
- 'missingFiles' => [],
- 'warnings' => []
- ];
- // Select all files in the reference table not found by a soft reference parser (thus TCA configured)
- $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
- ->getQueryBuilderForTable('sys_refindex');
-
- $result = $queryBuilder
- ->select('*')
- ->from('sys_refindex')
- ->where(
- $queryBuilder->expr()->eq('ref_table', $queryBuilder->createNamedParameter('_FILE', \PDO::PARAM_STR)),
- $queryBuilder->expr()->eq('softref_key', $queryBuilder->createNamedParameter('', \PDO::PARAM_STR))
- )
- ->orderBy('sorting', 'DESC')
- ->execute();
-
- // Traverse the files and put into a large table:
- $tempCount = [];
- while ($rec = $result->fetch()) {
- // Compile info string for location of reference:
- $infoString = $this->infoStr($rec);
- // Registering occurencies in directories:
- $resultArray['dirname_registry'][dirname($rec['ref_string'])][$rec['tablename'] . ':' . $rec['field']]++;
- // Handle missing file:
- if (!@is_file((PATH_site . $rec['ref_string']))) {
- $resultArray['missingFiles'][$rec['ref_string']][$rec['hash']] = $infoString;
- ksort($resultArray['missingFiles'][$rec['ref_string']]);
- }
- // Add entry if file has multiple references pointing to it:
- if (isset($tempCount[$rec['ref_string']])) {
- if (!is_array($resultArray['multipleReferencesList'][$rec['ref_string']])) {
- $resultArray['multipleReferencesList'][$rec['ref_string']] = [];
- $resultArray['multipleReferencesList'][$rec['ref_string']][$tempCount[$rec['ref_string']][1]] = $tempCount[$rec['ref_string']][0];
- }
- $resultArray['multipleReferencesList'][$rec['ref_string']][$rec['hash']] = $infoString;
- ksort($resultArray['multipleReferencesList'][$rec['ref_string']]);
- } else {
- $tempCount[$rec['ref_string']] = [$infoString, $rec['hash']];
- }
- }
-
- ksort($resultArray['missingFiles']);
- ksort($resultArray['multipleReferencesList']);
- // Add count for multi-references:
- $resultArray['multipleReferencesList_count']['count'] = count($resultArray['multipleReferencesList']);
- $resultArray['singleReferencesList_count']['count'] = count($tempCount) - $resultArray['multipleReferencesList_count']['count'];
- // Sort dirname registry and add warnings for directories outside uploads/
- ksort($resultArray['dirname_registry']);
- foreach ($resultArray['dirname_registry'] as $dir => $temp) {
- ksort($resultArray['dirname_registry'][$dir]);
- if (!GeneralUtility::isFirstPartOfStr($dir, 'uploads/')) {
- $resultArray['warnings'][GeneralUtility::shortMD5($dir)] = 'Directory "' . $dir . '" was outside uploads/ which is unusual practice in TYPO3 although not forbidden. Directory used by the following table:field pairs: ' . implode(',', array_keys($temp));
- }
- }
- return $resultArray;
- }
-
- /**
- * Mandatory autofix function
- * Will run auto-fix on the result array. Echos status during processing.
- *
- * @param array $resultArray Result array from main() function
- * @return void
- */
- public function main_autoFix($resultArray)
- {
- foreach ($resultArray['multipleReferencesList'] as $key => $value) {
- $absFileName = GeneralUtility::getFileAbsFileName($key);
- if ($absFileName && @is_file($absFileName)) {
- echo 'Processing file: ' . $key . LF;
- $c = 0;
- foreach ($value as $hash => $recReference) {
- if ($c == 0) {
- echo ' Keeping ' . $key . ' for record "' . $recReference . '"' . LF;
- } else {
- // Create unique name for file:
- $fileFunc = GeneralUtility::makeInstance(BasicFileUtility::class);
- $newName = $fileFunc->getUniqueName(basename($key), dirname($absFileName));
- echo ' Copying ' . $key . ' to ' . \TYPO3\CMS\Core\Utility\PathUtility::stripPathSitePrefix($newName) . ' for record "' . $recReference . '": ';
- if ($bypass = $this->cli_noExecutionCheck($recReference)) {
- echo $bypass;
- } else {
- GeneralUtility::upload_copy_move($absFileName, $newName);
- clearstatcache();
- if (@is_file($newName)) {
- $sysRefObj = GeneralUtility::makeInstance(ReferenceIndex::class);
- $error = $sysRefObj->setReferenceValue($hash, basename($newName));
- if ($error) {
- echo ' ERROR: TYPO3\\CMS\\Core\\Database\\ReferenceIndex::setReferenceValue(): ' . $error . LF;
- die;
- } else {
- echo 'DONE';
- }
- } else {
- echo ' ERROR: File "' . $newName . '" was not created!';
- }
- }
- echo LF;
- }
- $c++;
- }
- } else {
- echo ' ERROR: File "' . $absFileName . '" was not found!';
- }
- }
- }
-}
diff --git a/typo3/sysext/lowlevel/Configuration/Commands.php b/typo3/sysext/lowlevel/Configuration/Commands.php
index c5b563dd553748a8a2b9517b7b8a5069ef02ad5c..2496e084f80762135e3c25c3645a240959f28269 100644
--- a/typo3/sysext/lowlevel/Configuration/Commands.php
+++ b/typo3/sysext/lowlevel/Configuration/Commands.php
@@ -20,6 +20,11 @@ return [
// needed for updating the reference index (optional)
'user' => '_cli_lowlevel'
],
+ 'cleanup:multiplereferencedfiles' => [
+ 'class' => \TYPO3\CMS\Lowlevel\Command\FilesWithMultipleReferencesCommand::class,
+ // needed for updating the reference index (optional)
+ 'user' => '_cli_lowlevel'
+ ],
'cleanup:missingrelations' => [
'class' => \TYPO3\CMS\Lowlevel\Command\MissingRelationsCommand::class,
// needed for updating the reference index (optional)
diff --git a/typo3/sysext/lowlevel/ext_localconf.php b/typo3/sysext/lowlevel/ext_localconf.php
index 204cf8e160974fd9b8558567f4e63db774d7d4bf..98412fe329e8a8ea6881cfc35b1aee0e3ab645d1 100644
--- a/typo3/sysext/lowlevel/ext_localconf.php
+++ b/typo3/sysext/lowlevel/ext_localconf.php
@@ -10,7 +10,6 @@ if (TYPO3_MODE === 'BE') {
},
'_CLI_lowlevel'
];
- $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['double_files'] = [\TYPO3\CMS\Lowlevel\DoubleFilesCommand::class];
$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['rte_images'] = [\TYPO3\CMS\Lowlevel\RteImagesCommand::class];
$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['versions'] = [\TYPO3\CMS\Lowlevel\VersionsCommand::class];
}