From 5bb7eb6e61256678bdcd44dd9c32e1dd24b07119 Mon Sep 17 00:00:00 2001
From: Benni Mack <benni@typo3.org>
Date: Tue, 25 Oct 2016 09:30:03 +0200
Subject: [PATCH] [!!!][TASK] Migrate lowlevel deleted records command to
SymfonyConsole
The lowlevel cleaner command for permanently delete records
in the database that have been previously marked as "deleted=1"
in the database is migrated to a Symfony Console command,
reducing the complexity and enhances the readability of the function.
Call it like this:
typo3/sysext/core/bin/typo3 cleanup:deletedrecords --dry-run -vv --pid=49 --depth=4
You can also use "-p" instead of "--pid", or "-d" instead of "--depth".
Resolves: #78417
Releases: master
Change-Id: I79fb2292d96c38c1406896cbe9d0bac6494d1fa9
Reviewed-on: https://review.typo3.org/50358
Tested-by: TYPO3com <no-reply@typo3.com>
Reviewed-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Tested-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Reviewed-by: Wouter Wolters <typo3@wouterwolters.nl>
Tested-by: Wouter Wolters <typo3@wouterwolters.nl>
---
...DeletedRecordsCommandParametersChanged.rst | 44 +++
.../Classes/Command/DeletedRecordsCommand.php | 318 ++++++++++++++++++
.../Classes/DeletedRecordsCommand.php | 116 -------
.../lowlevel/Configuration/Commands.php | 4 +
typo3/sysext/lowlevel/ext_localconf.php | 1 -
5 files changed, 366 insertions(+), 117 deletions(-)
create mode 100644 typo3/sysext/core/Documentation/Changelog/master/Breaking-78417-LowlevelDeletedRecordsCommandParametersChanged.rst
create mode 100644 typo3/sysext/lowlevel/Classes/Command/DeletedRecordsCommand.php
delete mode 100644 typo3/sysext/lowlevel/Classes/DeletedRecordsCommand.php
diff --git a/typo3/sysext/core/Documentation/Changelog/master/Breaking-78417-LowlevelDeletedRecordsCommandParametersChanged.rst b/typo3/sysext/core/Documentation/Changelog/master/Breaking-78417-LowlevelDeletedRecordsCommandParametersChanged.rst
new file mode 100644
index 000000000000..151a39757a4b
--- /dev/null
+++ b/typo3/sysext/core/Documentation/Changelog/master/Breaking-78417-LowlevelDeletedRecordsCommandParametersChanged.rst
@@ -0,0 +1,44 @@
+.. include:: ../../Includes.txt
+
+====================================================================
+Breaking: #78417 - Lowlevel DeletedRecordsCommand parameters changed
+====================================================================
+
+See :issue:`78417`
+
+Description
+===========
+
+The DeletedRecordsCommand is now using Symfony Console. The new command which behaves like the old code, but uses certain
+different parameters and is located under the following path now:
+
+`./typo3/sysext/core/bin/typo3 cleanup:deletedrecords`
+
+The following options can be set
+`--dry-run` to only show the deleted records
+`-v` and `-vv` to show additional information
+`--pid=23` or `-p=23` to only find and delete records below page ID 23 (otherwise "0" is taken)
+`--depth=4` or `-d=4` to only delete recursively until a certain page tree level.
+
+The PHP class `TYPO3\CMS\Lowlevel\DeletedRecordsCommand` has been removed.
+
+Impact
+======
+
+Calling `typo3/cli_dispatch lowlevel cleaner deleted` will not work anymore.
+
+Calling the PHP class results in a fatal PHP error.
+
+
+Affected Installations
+======================
+
+Any TYPO3 installation using the previously functioned command or the related PHP class.
+
+
+Migration
+=========
+
+Use the new CLI command as shown above.
+
+.. index:: CLI
diff --git a/typo3/sysext/lowlevel/Classes/Command/DeletedRecordsCommand.php b/typo3/sysext/lowlevel/Classes/Command/DeletedRecordsCommand.php
new file mode 100644
index 000000000000..7774e06d3344
--- /dev/null
+++ b/typo3/sysext/lowlevel/Classes/Command/DeletedRecordsCommand.php
@@ -0,0 +1,318 @@
+<?php
+declare(strict_types=1);
+namespace TYPO3\CMS\Lowlevel\Command;
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Query\QueryBuilder;
+use TYPO3\CMS\Core\DataHandling\DataHandler;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Core\Utility\MathUtility;
+
+/**
+ * Force-deletes all records in the database which have a deleted=1 flag
+ */
+class DeletedRecordsCommand extends Command
+{
+
+ /**
+ * Configure the command by defining the name, options and arguments
+ */
+ public function configure()
+ {
+ $this
+ ->setDescription('Permanently deletes all records marked as "deleted" in the database.')
+ ->setHelp('Traverse page tree and find and flush deleted records. If you want to get more detailed information, use the --verbose option.')
+ ->addOption(
+ 'pid',
+ 'p',
+ InputOption::VALUE_REQUIRED,
+ 'Setting start page in page tree. Default is the page tree root, 0 (zero)'
+ )
+ ->addOption(
+ 'depth',
+ 'd',
+ InputOption::VALUE_REQUIRED,
+ 'Setting traversal depth. 0 (zero) will only analyse start page (see --pid), 1 will traverse one level of subpages etc.'
+ )
+ ->addOption(
+ 'dry-run',
+ null,
+ InputOption::VALUE_NONE,
+ 'If this option is set, the records will not actually be deleted, but just the output which records would be deleted are shown'
+ );
+ }
+
+ /**
+ * Executes the command to find and permanently delete records which are marked as deleted
+ *
+ * @param InputInterface $input
+ * @param OutputInterface $output
+ *
+ * @return void
+ */
+ protected function execute(InputInterface $input, OutputInterface $output)
+ {
+ // The backend user needs super-powers because datahandler is executed
+ $previouslyAppliedAdminRights = $this->getBackendUser()->user['admin'];
+ $this->getBackendUser()->user['admin'] = 1;
+
+ $io = new SymfonyStyle($input, $output);
+ $io->title($this->getDescription());
+
+ $startingPoint = 0;
+ if ($input->hasOption('pid') && MathUtility::canBeInterpretedAsInteger($input->getOption('pid'))) {
+ $startingPoint = MathUtility::forceIntegerInRange((int)$input->getOption('pid'), 0);
+ }
+
+ $depth = 1000;
+ if ($input->hasOption('depth') && MathUtility::canBeInterpretedAsInteger($input->getOption('depth'))) {
+ $depth = MathUtility::forceIntegerInRange((int)$input->getOption('depth'), 0);
+ }
+
+ if ($io->isVerbose()) {
+ $io->section('Searching the database now for deleted records.');
+ }
+
+ // type unsafe comparison and explicit boolean setting on purpose
+ $dryRun = $input->hasOption('dry-run') && $input->getOption('dry-run') != false ? true : false;
+
+ // find all records that should be deleted
+ $deletedRecords = $this->findAllFlaggedRecordsInPage($startingPoint, $depth);
+
+ if (!$io->isQuiet()) {
+ $totalAmountOfTables = count($deletedRecords);
+ $totalAmountOfRecords = 0;
+ foreach ($deletedRecords as $tableName => $itemsInTable) {
+ $totalAmountOfRecords += count($itemsInTable);
+
+ if ($io->isVeryVerbose()) {
+ $io->writeln('Found ' . count($itemsInTable) . ' deleted records in table "' . $tableName . '".');
+ }
+ }
+ $io->note('Found ' . $totalAmountOfRecords . ' records in ' . $totalAmountOfTables . ' database tables ready to be deleted.');
+ }
+
+ $io->section('Deletion process starting now.' . ($dryRun ? ' (Not deleting now, just a dry run)' : ''));
+
+ // actually permanently delete them
+ $this->deleteRecords($deletedRecords, $dryRun, $io);
+
+ // Restore backend user administration rights
+ $this->getBackendUser()->user['admin'] = $previouslyAppliedAdminRights;
+
+ $io->success('All done!');
+ }
+
+ /**
+ * Recursive traversal of page tree to fetch all records marekd as "deleted",
+ * via option $GLOBALS[TCA][$tableName][ctrl][delete]
+ * This also takes deleted versioned records into account.
+ *
+ * @param int $pageId the uid of the pages record (can also be 0)
+ * @param int $depth The current depth of levels to go down
+ * @param array $deletedRecords the records that are already marked as deleted (used when going recursive)
+ *
+ * @return array the modified $deletedRecords array
+ */
+ protected function findAllFlaggedRecordsInPage(int $pageId, int $depth, array $deletedRecords = []): array
+ {
+ /** @var QueryBuilder $queryBuilderForPages */
+ $queryBuilderForPages = GeneralUtility::makeInstance(ConnectionPool::class)
+ ->getQueryBuilderForTable('pages');
+ $queryBuilderForPages->getRestrictions()->removeAll();
+
+ $pageId = (int)$pageId;
+ if ($pageId > 0) {
+ $pageRecordIsDeleted = $queryBuilderForPages
+ ->select('uid', 'deleted')
+ ->from('pages')
+ ->where(
+ $queryBuilderForPages->expr()->andX(
+ $queryBuilderForPages->expr()->eq(
+ 'uid',
+ $queryBuilderForPages->createNamedParameter($pageId, \PDO::PARAM_INT)
+ ),
+ $queryBuilderForPages->expr()->neq('deleted', 0)
+ )
+ )
+ ->execute();
+
+ // Register if page itself is deleted
+ if ($pageRecordIsDeleted->rowCount() > 0) {
+ $deletedRecords['pages'][$pageId] = $pageId;
+ }
+ }
+
+ $databaseTables = $this->getTablesWithDeletedFlags();
+ // Traverse tables of records that belongs to page
+ foreach ($databaseTables as $tableName => $deletedField) {
+ // Select all records belonging to page
+ $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
+ ->getQueryBuilderForTable($tableName);
+
+ $queryBuilder->getRestrictions()->removeAll();
+
+ $result = $queryBuilder
+ ->select('uid', $deletedField)
+ ->from($tableName)
+ ->where($queryBuilder->expr()->eq(
+ 'pid',
+ $queryBuilder->createNamedParameter($pageId, \PDO::PARAM_INT))
+ )
+ ->execute();
+
+ while ($recordOnPage = $result->fetch()) {
+ // Register record as deleted
+ if ($recordOnPage[$deletedField]) {
+ $deletedRecords[$tableName][$recordOnPage['uid']] = $recordOnPage['uid'];
+ }
+ // Add any versions of those records
+ $versions = BackendUtility::selectVersionsOfRecord($tableName, $recordOnPage['uid'],
+ 'uid,t3ver_wsid,t3ver_count,' . $deletedField, null, true) ?: [];
+ if (is_array($versions)) {
+ foreach ($versions as $verRec) {
+ // Mark as deleted
+ if (!$verRec['_CURRENT_VERSION'] && $verRec[$deletedField]) {
+ $deletedRecords[$tableName][$verRec['uid']] = $verRec['uid'];
+ }
+ }
+ }
+ }
+ }
+
+ // Find subpages to root ID and go recursive
+ if ($depth > 0) {
+ $depth--;
+ $result = $queryBuilderForPages
+ ->select('uid')
+ ->from('pages')
+ ->where(
+ $queryBuilderForPages->expr()->eq('pid', $pageId)
+ )
+ ->orderBy('sorting')
+ ->execute();
+
+ while ($subPage = $result->fetch()) {
+ $deletedRecords = $this->findAllFlaggedRecordsInPage($subPage['uid'], $depth, $deletedRecords);
+ }
+ }
+
+ // Add any versions of the page
+ if ($pageId > 0) {
+ $versions = BackendUtility::selectVersionsOfRecord(
+ 'pages',
+ $pageId,
+ 'uid,t3ver_oid,t3ver_wsid,t3ver_count',
+ null,
+ true
+ ) ?: [];
+ if (is_array($versions)) {
+ foreach ($versions as $verRec) {
+ if (!$verRec['_CURRENT_VERSION']) {
+ $deletedRecords = $this->findAllFlaggedRecordsInPage($verRec['uid'], $depth, $deletedRecords);
+ }
+ }
+ }
+ }
+
+ return $deletedRecords;
+ }
+
+ /**
+ * Fetches all tables registered in the TCA with a deleted
+ * and that are not pages (which are handled separately)
+ *
+ * @return array an associative array with the table as key and the
+ */
+ protected function getTablesWithDeletedFlags(): array
+ {
+ static $tables;
+ if (!is_array($tables)) {
+ $tables = [];
+ foreach ($GLOBALS['TCA'] as $tableName => $configuration) {
+ if ($tableName !== 'pages' && isset($GLOBALS['TCA'][$tableName]['ctrl']['delete'])) {
+ $tables[$tableName] = $GLOBALS['TCA'][$tableName]['ctrl']['delete'];
+ }
+ }
+ ksort($tables);
+ }
+ return $tables;
+ }
+
+ /**
+ * Deletes records via DataHandler
+ *
+ * @param array $deletedRecords two level array with tables and uids
+ * @param bool $dryRun check if the records should NOT be deleted (use --dry-run to avoid)
+ * @param SymfonyStyle $io
+ * @return void
+ */
+ protected function deleteRecords(array $deletedRecords, bool $dryRun, SymfonyStyle $io)
+ {
+ // Putting "pages" table in the bottom
+ if (isset($deletedRecords['pages'])) {
+ $_pages = $deletedRecords['pages'];
+ unset($deletedRecords['pages']);
+ // To delete sub pages first assuming they are accumulated from top of page tree.
+ $deletedRecords['pages'] = array_reverse($_pages);
+ }
+
+ // set up the data handler instance
+ $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
+ $dataHandler->start([], []);
+
+ // Loop through all tables and their records
+ foreach ($deletedRecords as $table => $list) {
+ if ($io->isVerbose()) {
+ $io->writeln('Flushing ' . count($list) . ' deleted records from table "' . $table . '"');
+ }
+ foreach ($list as $uid) {
+ if ($io->isVeryVerbose()) {
+ $io->writeln('Flushing record "' . $table . ':' . $uid . '"');
+ }
+ if (!$dryRun) {
+ // Notice, we are deleting pages with no regard to subpages/subrecords - we do this since they
+ // should also be included in the set of deleted pages of course (no un-deleted record can exist
+ // under a deleted page...)
+ $dataHandler->deleteRecord($table, $uid, true, true);
+ // Return errors if any:
+ if (!empty($dataHandler->errorLog)) {
+ $errorMessage = array_merge(['DataHandler reported an error'], $dataHandler->errorLog);
+ $io->error($errorMessage);
+ } elseif (!$io->isQuiet()) {
+ $io->writeln('Permanently deleted record "' . $table . ':' . $uid . '".');
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Short-hand function for accessing the current backend user
+ * @return BackendUserAuthentication
+ */
+ protected function getBackendUser(): BackendUserAuthentication
+ {
+ return $GLOBALS['BE_USER'];
+ }
+}
diff --git a/typo3/sysext/lowlevel/Classes/DeletedRecordsCommand.php b/typo3/sysext/lowlevel/Classes/DeletedRecordsCommand.php
deleted file mode 100644
index 05991e97a384..000000000000
--- a/typo3/sysext/lowlevel/Classes/DeletedRecordsCommand.php
+++ /dev/null
@@ -1,116 +0,0 @@
-<?php
-namespace TYPO3\CMS\Lowlevel;
-
-/*
- * This file is part of the TYPO3 CMS project.
- *
- * It is free software; you can redistribute it and/or modify it under
- * the terms of the GNU General Public License, either version 2
- * of the License, or any later version.
- *
- * For the full copyright and license information, please read the
- * LICENSE.txt file that was distributed with this source code.
- *
- * The TYPO3 project - inspiring people to share!
- */
-use TYPO3\CMS\Core\DataHandling\DataHandler;
-use TYPO3\CMS\Core\Utility\GeneralUtility;
-use TYPO3\CMS\Core\Utility\MathUtility;
-
-/**
- * Looking for Deleted records
- */
-class DeletedRecordsCommand extends CleanerCommand
-{
- /**
- * Constructor
- */
- public function __construct()
- {
- parent::__construct();
- // Setting up help:
- $this->cli_options[] = ['--echotree level', 'When "level" is set to 1 or higher you will see the page of the page tree outputted as it is traversed. A value of 2 for "level" will show even more information.'];
- $this->cli_options[] = ['--pid id', 'Setting start page in page tree. Default is the page tree root, 0 (zero)'];
- $this->cli_options[] = ['--depth int', 'Setting traversal depth. 0 (zero) will only analyse start page (see --pid), 1 will traverse one level of subpages etc.'];
- $this->cli_help['name'] = 'deleted -- To find and flush deleted records in the page tree';
- $this->cli_help['description'] = trim('
-Traversing page tree and finding deleted records
-
-Automatic Repair:
-Although deleted records are not errors to be repaired, this tool allows you to flush the deleted records completely from the system as an automatic action. Limiting this lookup by --pid and --depth can help you to narrow in the operation to a part of the page tree.
-');
- $this->cli_help['examples'] = '';
- }
-
- /**
- * Find orphan records
- * VERY CPU and memory intensive since it will look up the whole page tree!
- *
- * @return array
- */
- public function main()
- {
- // Initialize result array:
- $resultArray = [
- 'message' => $this->cli_help['name'] . LF . LF . $this->cli_help['description'],
- 'headers' => [
- 'deleted' => ['Index of deleted records', 'These are records from the page tree having the deleted-flag set. The --AUTOFIX option will flush them completely!', 1]
- ],
- 'deleted' => []
- ];
- $startingPoint = $this->cli_isArg('--pid') ? MathUtility::forceIntegerInRange($this->cli_argValue('--pid'), 0) : 0;
- $depth = $this->cli_isArg('--depth') ? MathUtility::forceIntegerInRange($this->cli_argValue('--depth'), 0) : 1000;
- $this->genTree($startingPoint, $depth, (int)$this->cli_argValue('--echotree'));
- $resultArray['deleted'] = $this->recStats['deleted'];
- return $resultArray;
- }
-
- /**
- * Mandatory autofix function
- * Will run auto-fix on the result array. Echos status during processing.
- *
- * @param array $resultArray Result array from main() function
- * @return void
- */
- public function main_autoFix($resultArray)
- {
- // Putting "tx_templavoila_datastructure" table in the bottom:
- if (isset($resultArray['deleted']['tx_templavoila_datastructure'])) {
- $_tx_templavoila_datastructure = $resultArray['deleted']['tx_templavoila_datastructure'];
- unset($resultArray['deleted']['tx_templavoila_datastructure']);
- $resultArray['deleted']['tx_templavoila_datastructure'] = $_tx_templavoila_datastructure;
- }
- // Putting "pages" table in the bottom:
- if (isset($resultArray['deleted']['pages'])) {
- $_pages = $resultArray['deleted']['pages'];
- unset($resultArray['deleted']['pages']);
- // To delete sub pages first assuming they are accumulated from top of page tree.
- $resultArray['deleted']['pages'] = array_reverse($_pages);
- }
- // Traversing records:
- foreach ($resultArray['deleted'] as $table => $list) {
- echo 'Flushing deleted records from table "' . $table . '":' . LF;
- foreach ($list as $uid) {
- echo ' Flushing record "' . $table . ':' . $uid . '": ';
- if ($bypass = $this->cli_noExecutionCheck($table . ':' . $uid)) {
- echo $bypass;
- } else {
- // Execute CMD array:
- $tce = GeneralUtility::makeInstance(DataHandler::class);
- $tce->start([], []);
- // Notice, we are deleting pages with no regard to subpages/subrecords - we do this since they
- // should also be included in the set of deleted pages of course (no un-deleted record can exist
- // under a deleted page...)
- $tce->deleteRecord($table, $uid, true, true);
- // Return errors if any:
- if (count($tce->errorLog)) {
- echo ' ERROR from "TCEmain":' . LF . 'TCEmain:' . implode((LF . 'TCEmain:'), $tce->errorLog);
- } else {
- echo 'DONE';
- }
- }
- echo LF;
- }
- }
- }
-}
diff --git a/typo3/sysext/lowlevel/Configuration/Commands.php b/typo3/sysext/lowlevel/Configuration/Commands.php
index f65afa0f0692..4c1ae1be10b9 100644
--- a/typo3/sysext/lowlevel/Configuration/Commands.php
+++ b/typo3/sysext/lowlevel/Configuration/Commands.php
@@ -9,5 +9,9 @@
return [
'syslog:list' => [
'class' => \TYPO3\CMS\Lowlevel\Command\ListSysLogCommand::class
+ ],
+ 'cleanup:deletedrecords' => [
+ 'class' => \TYPO3\CMS\Lowlevel\Command\DeletedRecordsCommand::class,
+ 'user' => '_cli_lowlevel'
]
];
diff --git a/typo3/sysext/lowlevel/ext_localconf.php b/typo3/sysext/lowlevel/ext_localconf.php
index ef542017bb46..74fbbbd76755 100644
--- a/typo3/sysext/lowlevel/ext_localconf.php
+++ b/typo3/sysext/lowlevel/ext_localconf.php
@@ -16,7 +16,6 @@ if (TYPO3_MODE === 'BE') {
$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['rte_images'] = [\TYPO3\CMS\Lowlevel\RteImagesCommand::class];
$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['lost_files'] = [\TYPO3\CMS\Lowlevel\LostFilesCommand::class];
$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['orphan_records'] = [\TYPO3\CMS\Lowlevel\OrphanRecordsCommand::class];
- $GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['deleted'] = [\TYPO3\CMS\Lowlevel\DeletedRecordsCommand::class];
$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['versions'] = [\TYPO3\CMS\Lowlevel\VersionsCommand::class];
$GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['lowlevel']['cleanerModules']['cleanflexform'] = [\TYPO3\CMS\Lowlevel\CleanFlexformCommand::class];
}
--
GitLab