From 59289da4c3deba97a47b5c4b37bfbca663de6669 Mon Sep 17 00:00:00 2001
From: Dmitry Dulepov <dmitry.dulepov@gmail.com>
Date: Wed, 17 Mar 2021 15:09:51 +0300
Subject: [PATCH] [BUGFIX] Normalize slug before converting Unicode characters
to ascii
This change adds a normalization for slug source before converting it
to ascii. This helps to avoid issues with various ways to encode the
same Unicode characters.
Used command:
composer req symfony/polyfill-intl-normalizer
Resolves: #93764
Releases: master, 10.4
Change-Id: I9982fafd9c34c69bb6ca47ee8242f504b9974121
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/68492
Tested-by: core-ci <typo3@b13.com>
Tested-by: Daniel Goerz <daniel.goerz@posteo.de>
Tested-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
Reviewed-by: Daniel Goerz <daniel.goerz@posteo.de>
Reviewed-by: Anja Leichsenring <aleichsenring@ab-softlab.de>
---
composer.json | 1 +
composer.lock | 16 ++++++++--------
.../core/Classes/DataHandling/SlugHelper.php | 4 ++++
.../Tests/Unit/DataHandling/SlugHelperTest.php | 5 +++++
typo3/sysext/core/composer.json | 1 +
5 files changed, 19 insertions(+), 8 deletions(-)
diff --git a/composer.json b/composer.json
index 358802bc67c6..6d3ab7d639aa 100644
--- a/composer.json
+++ b/composer.json
@@ -68,6 +68,7 @@
"symfony/mime": "^5.2",
"symfony/polyfill-intl-icu": "^1.6",
"symfony/polyfill-intl-idn": "^1.10",
+ "symfony/polyfill-intl-normalizer": "^1.22",
"symfony/polyfill-mbstring": "^1.2",
"symfony/property-access": "^5.2",
"symfony/property-info": "^5.2",
diff --git a/composer.lock b/composer.lock
index 71e18d337647..746190119bc8 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
- "content-hash": "88af0d54fa97c2f9129dceb8dc331a28",
+ "content-hash": "2dc2c8cb2e24d2bd5a60678b0e5f48ad",
"packages": [
{
"name": "bacon/bacon-qr-code",
@@ -3243,16 +3243,16 @@
},
{
"name": "symfony/polyfill-intl-normalizer",
- "version": "v1.20.0",
+ "version": "v1.22.1",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-intl-normalizer.git",
- "reference": "727d1096295d807c309fb01a851577302394c897"
+ "reference": "43a0283138253ed1d48d352ab6d0bdb3f809f248"
},
"dist": {
"type": "zip",
- "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/727d1096295d807c309fb01a851577302394c897",
- "reference": "727d1096295d807c309fb01a851577302394c897",
+ "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/43a0283138253ed1d48d352ab6d0bdb3f809f248",
+ "reference": "43a0283138253ed1d48d352ab6d0bdb3f809f248",
"shasum": ""
},
"require": {
@@ -3264,7 +3264,7 @@
"type": "library",
"extra": {
"branch-alias": {
- "dev-main": "1.20-dev"
+ "dev-main": "1.22-dev"
},
"thanks": {
"name": "symfony/polyfill",
@@ -3307,7 +3307,7 @@
"shim"
],
"support": {
- "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.20.0"
+ "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.22.1"
},
"funding": [
{
@@ -3323,7 +3323,7 @@
"type": "tidelift"
}
],
- "time": "2020-10-23T14:02:19+00:00"
+ "time": "2021-01-22T09:19:47+00:00"
},
{
"name": "symfony/polyfill-mbstring",
diff --git a/typo3/sysext/core/Classes/DataHandling/SlugHelper.php b/typo3/sysext/core/Classes/DataHandling/SlugHelper.php
index d5fca085c6e3..630cbc8c5df2 100644
--- a/typo3/sysext/core/Classes/DataHandling/SlugHelper.php
+++ b/typo3/sysext/core/Classes/DataHandling/SlugHelper.php
@@ -114,6 +114,10 @@ class SlugHelper
$fallbackCharacter = (string)($this->configuration['fallbackCharacter'] ?? '-');
$slug = preg_replace('/[ \t\x{00A0}\-+_]+/u', $fallbackCharacter, $slug);
+ if (!\Normalizer::isNormalized((string)$slug)) {
+ $slug = \Normalizer::normalize((string)$slug);
+ }
+
// Convert extended letters to ascii equivalents
// The specCharsToASCII() converts "€" to "EUR"
$slug = GeneralUtility::makeInstance(CharsetConverter::class)->specCharsToASCII('utf-8', $slug);
diff --git a/typo3/sysext/core/Tests/Unit/DataHandling/SlugHelperTest.php b/typo3/sysext/core/Tests/Unit/DataHandling/SlugHelperTest.php
index 580754e89bf1..18e81adbb10d 100644
--- a/typo3/sysext/core/Tests/Unit/DataHandling/SlugHelperTest.php
+++ b/typo3/sysext/core/Tests/Unit/DataHandling/SlugHelperTest.php
@@ -132,6 +132,11 @@ class SlugHelperTest extends UnitTestCase
'bla-arg应---用-ascii',
'bla-arg应-用-ascii'
],
+ 'non-normalized characters' => [
+ [],
+ hex2bin('667275cc88686e65757a6569746c696368656e'),
+ 'fruehneuzeitlichen'
+ ],
];
}
diff --git a/typo3/sysext/core/composer.json b/typo3/sysext/core/composer.json
index 3ddc2c75b82b..a3ecb827976b 100644
--- a/typo3/sysext/core/composer.json
+++ b/typo3/sysext/core/composer.json
@@ -55,6 +55,7 @@
"symfony/mime": "^5.2",
"symfony/polyfill-intl-icu": "^1.6",
"symfony/polyfill-intl-idn": "^1.10",
+ "symfony/polyfill-intl-normalizer": "^1.22",
"symfony/polyfill-mbstring": "^1.2",
"symfony/routing": "^5.2",
"symfony/yaml": "^5.2",
--
GitLab