From d5441f1873c31cd72cf0297e66d5b347a2d19c79 Mon Sep 17 00:00:00 2001
From: Benni Mack <benni@typo3.org>
Date: Thu, 24 Feb 2022 14:24:34 +0100
Subject: [PATCH] [!!!][TASK] Always render frontend in UTF-8
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This removes the option config.metaCharset and the
conversion of content everywhere.
The public method "TSFE->convOutputCharset" is gone.
The property "config.metaCharset" has no effect anymore.
Resolves: #97065
Releases: main
Change-Id: I657b68adeba3501612ae0959fe54bdeb6470305c
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/73698
Tested-by: core-ci <typo3@b13.com>
Tested-by: Oliver Bartsch <bo@cedev.de>
Tested-by: Susanne Moog <look@susi.dev>
Tested-by: Stefan Bürk <stefan@buerk.tech>
Tested-by: Benni Mack <benni@typo3.org>
Reviewed-by: Oliver Bartsch <bo@cedev.de>
Reviewed-by: Susanne Moog <look@susi.dev>
Reviewed-by: Stefan Bürk <stefan@buerk.tech>
Reviewed-by: Benni Mack <benni@typo3.org>
---
Build/phpstan/phpstan-baseline.neon | 10 ----
.../PageRendererBackendSetupTrait.php | 1 -
.../sysext/core/Classes/Page/PageRenderer.php | 14 ++---
...065-TYPO3FrontendAlwaysRenderedInUTF-8.rst | 59 +++++++++++++++++++
.../Functional/Page/PageRendererTest.php | 3 -
.../core/Tests/Unit/Page/PageRendererTest.php | 15 -----
.../sysext/extbase/Classes/Core/Bootstrap.php | 2 +-
.../TypoScriptFrontendController.php | 46 ++-------------
.../frontend/Classes/Http/RequestHandler.php | 10 +---
.../PrepareTypoScriptFrontendRendering.php | 27 +--------
.../ContentObjectRendererTest.php | 13 +---
.../Classes/Controller/SearchController.php | 5 --
.../Classes/Hook/TypoScriptFrontendHook.php | 28 +--------
.../sysext/indexed_search/Classes/Indexer.php | 10 +---
.../Php/MethodCallMatcher.php | 7 +++
.../Php/PropertyPublicMatcher.php | 5 ++
.../t3editor/Resources/Private/tsref.xml | 9 ---
.../JavaScript/Mode/typoscript/typoscript.js | 1 -
18 files changed, 94 insertions(+), 171 deletions(-)
create mode 100644 typo3/sysext/core/Documentation/Changelog/12.0/Breaking-97065-TYPO3FrontendAlwaysRenderedInUTF-8.rst
diff --git a/Build/phpstan/phpstan-baseline.neon b/Build/phpstan/phpstan-baseline.neon
index fad5df0237c1..344023ba23a7 100644
--- a/Build/phpstan/phpstan-baseline.neon
+++ b/Build/phpstan/phpstan-baseline.neon
@@ -1200,11 +1200,6 @@ parameters:
count: 2
path: ../../typo3/sysext/core/Classes/Package/PackageManager.php
- -
- message: "#^Property TYPO3\\\\CMS\\\\Core\\\\Page\\\\PageRenderer\\:\\:\\$charSet \\(string\\) in isset\\(\\) is not nullable\\.$#"
- count: 1
- path: ../../typo3/sysext/core/Classes/Page/PageRenderer.php
-
-
message: "#^Property TYPO3\\\\CMS\\\\Core\\\\Page\\\\PageRenderer\\:\\:\\$jsInline \\(array\\<string, array\\>\\) on left side of \\?\\? is not nullable\\.$#"
count: 1
@@ -1215,11 +1210,6 @@ parameters:
count: 1
path: ../../typo3/sysext/core/Classes/Page/PageRenderer.php
- -
- message: "#^Result of \\|\\| is always false\\.$#"
- count: 1
- path: ../../typo3/sysext/core/Classes/Page/PageRenderer.php
-
-
message: "#^Strict comparison using \\=\\=\\= between array\\{settings\\?\\: non\\-empty\\-array, lang\\?\\: non\\-empty\\-array\\}&non\\-empty\\-array and array\\{\\} will always evaluate to false\\.$#"
count: 1
diff --git a/typo3/sysext/backend/Classes/Template/PageRendererBackendSetupTrait.php b/typo3/sysext/backend/Classes/Template/PageRendererBackendSetupTrait.php
index 758c5483d0b9..a65e49223679 100644
--- a/typo3/sysext/backend/Classes/Template/PageRendererBackendSetupTrait.php
+++ b/typo3/sysext/backend/Classes/Template/PageRendererBackendSetupTrait.php
@@ -54,7 +54,6 @@ trait PageRendererBackendSetupTrait
): void {
// Yes, hardcoded on purpose
$pageRenderer->setXmlPrologAndDocType('<!DOCTYPE html>');
- $pageRenderer->setCharSet('utf-8');
$pageRenderer->setLanguage($languageService->lang);
$pageRenderer->setMetaTag('name', 'viewport', 'width=device-width, initial-scale=1');
$pageRenderer->setFavIcon($this->getBackendFavicon($extensionConfiguration, $request));
diff --git a/typo3/sysext/core/Classes/Page/PageRenderer.php b/typo3/sysext/core/Classes/Page/PageRenderer.php
index cc0f0cb9b6f3..a78f320e44bd 100644
--- a/typo3/sysext/core/Classes/Page/PageRenderer.php
+++ b/typo3/sysext/core/Classes/Page/PageRenderer.php
@@ -141,7 +141,7 @@ class PageRenderer implements SingletonInterface
*
* @var string
*/
- protected $charSet;
+ protected $charSet = 'utf-8';
/**
* @var string
@@ -2097,8 +2097,8 @@ class PageRenderer implements SingletonInterface
}
$this->inlineLanguageLabelFiles = [];
// Convert settings back to UTF-8 since json_encode() only works with UTF-8:
- if ($this->getCharSet() && $this->getCharSet() !== 'utf-8' && is_array($this->inlineSettings)) {
- $this->convertCharsetRecursivelyToUtf8($this->inlineSettings, $this->getCharSet());
+ if ($this->charSet !== 'utf-8' && is_array($this->inlineSettings)) {
+ $this->convertCharsetRecursivelyToUtf8($this->inlineSettings, $this->charSet);
}
}
@@ -2376,12 +2376,12 @@ class PageRenderer implements SingletonInterface
*/
protected function includeLanguageFileForInline($fileRef, $selectionPrefix = '', $stripFromSelectionName = '')
{
- if (!isset($this->lang) || !isset($this->charSet)) {
- throw new \RuntimeException('Language and character encoding are not set.', 1284906026);
+ if (!isset($this->lang)) {
+ throw new \RuntimeException('Language is are not set.', 1284906026);
}
$labelsFromFile = [];
$allLabels = $this->readLLfile($fileRef);
- if ($allLabels !== false) {
+ if (is_array($allLabels)) {
// Merge language specific translations:
if ($this->lang !== 'default' && isset($allLabels[$this->lang])) {
$labels = array_merge($allLabels['default'], $allLabels[$this->lang]);
@@ -2391,7 +2391,7 @@ class PageRenderer implements SingletonInterface
// Iterate through all locallang labels:
foreach ($labels as $label => $value) {
// If $selectionPrefix is set, only respect labels that start with $selectionPrefix
- if ($selectionPrefix === '' || strpos($label, $selectionPrefix) === 0) {
+ if ($selectionPrefix === '' || str_starts_with($label, $selectionPrefix)) {
// Remove substring $stripFromSelectionName from label
$label = str_replace($stripFromSelectionName, '', $label);
$labelsFromFile[$label] = $value;
diff --git a/typo3/sysext/core/Documentation/Changelog/12.0/Breaking-97065-TYPO3FrontendAlwaysRenderedInUTF-8.rst b/typo3/sysext/core/Documentation/Changelog/12.0/Breaking-97065-TYPO3FrontendAlwaysRenderedInUTF-8.rst
new file mode 100644
index 000000000000..2848ab78719d
--- /dev/null
+++ b/typo3/sysext/core/Documentation/Changelog/12.0/Breaking-97065-TYPO3FrontendAlwaysRenderedInUTF-8.rst
@@ -0,0 +1,59 @@
+.. include:: ../../Includes.txt
+
+==========================================================
+Breaking: #97065 - TYPO3 Frontend always rendered in UTF-8
+==========================================================
+
+See :issue:`97065`
+
+Description
+===========
+
+For historical reasons, it was possible to change the actual rendering charset
+of TYPO3's Frontend Output to a specific character set, and also to modify the
+"renderCharset", which was removed in TYPO3 v8.0. Since TYPO3 v6, the default
+rendering output was set to "utf-8", and nowadays, it has become a niche
+to change the output rendering charset to a different value than UTF-8.
+
+For this reason, the TypoScript setting `config.metaCharset` has no effect
+anymore as all rendering for Frontend is "utf-8" and not changeable anymore.
+
+If this TypoScript setting was set to "utf-8" in previous installations,
+this line could have been removed anyways already.
+
+The public PHP property :php:`TypoScriptFrontendController->metaCharset` is
+removed, along with the public method
+:php:`TypoScriptFrontendController->convOutputCharset()`.
+
+
+Impact
+======
+
+TYPO3 installations with a different setting than "utf-8" will now output
+"utf-8" output at all times.
+
+TYPO3 extensions accessing the removed property will trigger a PHP warning, or
+calling the removed method :php:`convOutputCharset()` will see a fatal PHP error.
+
+
+Affected Installations
+======================
+
+TYPO3 installations using `config.metaCharset` set to a value other than
+`utf-8`, or accessing the removed property or method. The Extension Scanner
+in the Install Tool will detect usages of the removed property and method.
+
+
+Migration
+=========
+
+TYPO3 Installations with a different charset than UTF-8 should convert their own
+content in a custom middleware, as this specific use-case is not supported by
+TYPO3 Core anymore.
+
+TYPO3 installations with TypoScript option set `config.metaCharset = utf-8` can
+remove the TypoScript line in previous supported TYPO3 versions.
+
+Any usage of the removed property / method should be removed.
+
+.. index:: Frontend, PHP-API, TypoScript, FullyScanned, ext:frontend
diff --git a/typo3/sysext/core/Tests/Functional/Page/PageRendererTest.php b/typo3/sysext/core/Tests/Functional/Page/PageRendererTest.php
index 53339c0e6f2a..1b0e8d6a582a 100644
--- a/typo3/sysext/core/Tests/Functional/Page/PageRendererTest.php
+++ b/typo3/sysext/core/Tests/Functional/Page/PageRendererTest.php
@@ -228,7 +228,6 @@ class PageRendererTest extends FunctionalTestCase
$GLOBALS['TYPO3_REQUEST'] = (new ServerRequest('https://www.example.com/'))
->withAttribute('applicationType', $requestType);
$subject = $this->createPageRenderer();
- $subject->setCharSet('utf-8');
$subject->setLanguage('default');
$subject->enableMoveJsFromHeaderToFooter();
@@ -315,7 +314,6 @@ class PageRendererTest extends FunctionalTestCase
$GLOBALS['TYPO3_REQUEST'] = (new ServerRequest('https://www.example.com/'))
->withAttribute('applicationType', SystemEnvironmentBuilder::REQUESTTYPE_BE);
$subject = $this->createPageRenderer();
- $subject->setCharSet('utf-8');
$subject->setLanguage('default');
$subject->addJsFooterLibrary(
@@ -415,7 +413,6 @@ class PageRendererTest extends FunctionalTestCase
->withAttribute('applicationType', SystemEnvironmentBuilder::REQUESTTYPE_BE);
$subject = $this->createPageRenderer();
- $subject->setCharSet('utf-8');
$subject->setLanguage('default');
$packages = [
diff --git a/typo3/sysext/core/Tests/Unit/Page/PageRendererTest.php b/typo3/sysext/core/Tests/Unit/Page/PageRendererTest.php
index ba2c5c520a11..e7b356b9364a 100644
--- a/typo3/sysext/core/Tests/Unit/Page/PageRendererTest.php
+++ b/typo3/sysext/core/Tests/Unit/Page/PageRendererTest.php
@@ -163,21 +163,6 @@ class PageRendererTest extends UnitTestCase
/** @var PageRenderer|\PHPUnit\Framework\MockObject\MockObject|\TYPO3\TestingFramework\Core\AccessibleObjectInterface $subject */
$subject = $this->getAccessibleMock(PageRenderer::class, ['dummy'], [], '', false);
- $subject->_set('charSet', 'utf-8');
- $subject->_call('includeLanguageFileForInline', 'someLLFile.xml');
- }
-
- /**
- * @test
- */
- public function includeLanguageFileForInlineThrowsExceptionIfCharSetIsNotSet(): void
- {
- $this->expectException(\RuntimeException::class);
- $this->expectExceptionCode(1284906026);
-
- /** @var PageRenderer|\PHPUnit\Framework\MockObject\MockObject|\TYPO3\TestingFramework\Core\AccessibleObjectInterface $subject */
- $subject = $this->getAccessibleMock(PageRenderer::class, ['dummy'], [], '', false);
- $subject->_set('lang', 'default');
$subject->_call('includeLanguageFileForInline', 'someLLFile.xml');
}
diff --git a/typo3/sysext/extbase/Classes/Core/Bootstrap.php b/typo3/sysext/extbase/Classes/Core/Bootstrap.php
index ff75861f8f69..d7679fd87796 100644
--- a/typo3/sysext/extbase/Classes/Core/Bootstrap.php
+++ b/typo3/sysext/extbase/Classes/Core/Bootstrap.php
@@ -179,7 +179,7 @@ class Bootstrap
// Although the charset header is disabled in configuration, we *must* send a Content-Type header here.
// Content-Type headers optionally carry charset information at the same time.
// Since we have the information about the charset, there is no reason to not include the charset information although disabled in TypoScript.
- $response = $response->withHeader('Content-Type', 'application/json; charset=' . trim($typoScriptFrontendController->metaCharset));
+ $response = $response->withHeader('Content-Type', 'application/json; charset=utf-8');
}
}
diff --git a/typo3/sysext/frontend/Classes/Controller/TypoScriptFrontendController.php b/typo3/sysext/frontend/Classes/Controller/TypoScriptFrontendController.php
index 60befa754e8c..05da516e51a1 100644
--- a/typo3/sysext/frontend/Classes/Controller/TypoScriptFrontendController.php
+++ b/typo3/sysext/frontend/Classes/Controller/TypoScriptFrontendController.php
@@ -23,8 +23,6 @@ use Psr\Log\LoggerAwareTrait;
use Psr\Log\LogLevel;
use TYPO3\CMS\Backend\FrontendBackendUserAuthentication;
use TYPO3\CMS\Core\Cache\CacheManager;
-use TYPO3\CMS\Core\Charset\CharsetConverter;
-use TYPO3\CMS\Core\Charset\UnknownCharsetException;
use TYPO3\CMS\Core\Configuration\PageTsConfig;
use TYPO3\CMS\Core\Context\Context;
use TYPO3\CMS\Core\Context\DateTimeAspect;
@@ -413,14 +411,6 @@ class TypoScriptFrontendController implements LoggerAwareInterface
*/
public $content = '';
- /**
- * Output charset of the websites content. This is the charset found in the
- * header, meta tag etc. If different than utf-8 a conversion
- * happens before output to browser. Defaults to utf-8.
- * @var string
- */
- public $metaCharset = 'utf-8';
-
/**
* Internal calculations for labels
*
@@ -1554,10 +1544,6 @@ class TypoScriptFrontendController implements LoggerAwareInterface
if (is_array($this->pSetup['config.'] ?? null)) {
$this->config['config'] = array_replace_recursive($this->config['config'], $this->pSetup['config.']);
}
- // Rendering charset of HTML page.
- if (isset($this->config['config']['metaCharset']) && $this->config['config']['metaCharset'] !== 'utf-8') {
- $this->metaCharset = $this->config['config']['metaCharset'];
- }
// Processing for the config_array:
$this->config['rootLine'] = $this->tmpl->rootLine;
}
@@ -2109,9 +2095,6 @@ class TypoScriptFrontendController implements LoggerAwareInterface
GeneralUtility::callUserFunction($_funcRef, $_params, $this);
}
}
- // Convert charset for output. Any hooks before (including indexed search) will have to convert from UTF-8 to the target
- // charset as well.
- $this->content = $this->convOutputCharset($this->content);
// Storing for cache:
if (!$this->no_cache) {
$this->realPageCacheContent();
@@ -2238,8 +2221,8 @@ class TypoScriptFrontendController implements LoggerAwareInterface
'<!--FD_' . $this->config['INTincScript_ext']['divKey'] . '-->',
],
[
- $this->convOutputCharset(implode(LF, $this->additionalHeaderData)),
- $this->convOutputCharset(implode(LF, $this->additionalFooterData)),
+ implode(LF, $this->additionalHeaderData),
+ implode(LF, $this->additionalFooterData),
],
$this->pageRenderer->renderJavaScriptAndCssForProcessingOfUncachedContentObjects($this->content, $this->config['INTincScript_ext']['divKey'])
);
@@ -2306,7 +2289,7 @@ class TypoScriptFrontendController implements LoggerAwareInterface
$nonCacheableContent = $contentObjectRendererForNonCacheable->callUserFunction($nonCacheableData[$nonCacheableKey]['postUserFunc'], $nonCacheableData[$nonCacheableKey]['conf'], $nonCacheableData[$nonCacheableKey]['content']);
break;
}
- $this->content .= $this->convOutputCharset($nonCacheableContent);
+ $this->content .= $nonCacheableContent;
$this->content .= substr($contentPart, 35);
$timeTracker->pull($nonCacheableContent);
} else {
@@ -2364,7 +2347,7 @@ class TypoScriptFrontendController implements LoggerAwareInterface
{
// Set header for charset-encoding unless disabled
if (empty($this->config['config']['disableCharsetHeader'])) {
- $response = $response->withHeader('Content-Type', $this->contentType . '; charset=' . trim($this->metaCharset));
+ $response = $response->withHeader('Content-Type', $this->contentType . '; charset=utf-8');
}
// Set header for content language unless disabled
$contentLanguage = $this->language->getTwoLetterIsoCode();
@@ -2714,27 +2697,6 @@ class TypoScriptFrontendController implements LoggerAwareInterface
$this->languageService->debugKey = false;
}
- /**
- * Converts input string from utf-8 to metaCharset IF the two charsets are different.
- *
- * @param string $content Content to be converted.
- * @return string Converted content string.
- * @throws \RuntimeException if an invalid charset was configured
- */
- public function convOutputCharset($content)
- {
- if ($this->metaCharset !== 'utf-8') {
- /** @var CharsetConverter $charsetConverter */
- $charsetConverter = GeneralUtility::makeInstance(CharsetConverter::class);
- try {
- $content = $charsetConverter->conv($content, 'utf-8', $this->metaCharset);
- } catch (UnknownCharsetException $e) {
- throw new \RuntimeException('Invalid config.metaCharset: ' . $e->getMessage(), 1508916185);
- }
- }
- return $content;
- }
-
/**
* Returns the originally requested page uid when TSFE was instantiated initially.
*/
diff --git a/typo3/sysext/frontend/Classes/Http/RequestHandler.php b/typo3/sysext/frontend/Classes/Http/RequestHandler.php
index 2175e2bf88f8..fdaac130265e 100644
--- a/typo3/sysext/frontend/Classes/Http/RequestHandler.php
+++ b/typo3/sysext/frontend/Classes/Http/RequestHandler.php
@@ -257,8 +257,6 @@ class RequestHandler implements RequestHandlerInterface
if ($headerComment) {
$pageRenderer->addInlineComment("\t" . str_replace(LF, LF . "\t", $headerComment) . LF);
}
- // Setting charset:
- $theCharset = $controller->metaCharset;
$htmlTagAttributes = [];
$htmlLang = $siteLanguage->getHreflang() ?: '';
@@ -274,14 +272,14 @@ class RequestHandler implements RequestHandlerInterface
$xmlDocument = false;
break;
case 'xml_10':
- $docTypeParts[] = '<?xml version="1.0" encoding="' . $theCharset . '"?>';
+ $docTypeParts[] = '<?xml version="1.0" encoding="utf-8"?>';
break;
case 'xml_11':
- $docTypeParts[] = '<?xml version="1.1" encoding="' . $theCharset . '"?>';
+ $docTypeParts[] = '<?xml version="1.1" encoding="utf-8"?>';
break;
case '':
if ($controller->xhtmlVersion) {
- $docTypeParts[] = '<?xml version="1.0" encoding="' . $theCharset . '"?>';
+ $docTypeParts[] = '<?xml version="1.0" encoding="utf-8"?>';
} else {
$xmlDocument = false;
}
@@ -377,8 +375,6 @@ class RequestHandler implements RequestHandlerInterface
$headTag = $controller->cObj->stdWrap($headTag, $controller->pSetup['headTag.']);
}
$pageRenderer->setHeadTag($headTag);
- // Setting charset meta tag:
- $pageRenderer->setCharSet($theCharset);
$pageRenderer->addInlineComment(GeneralUtility::makeInstance(Typo3Information::class)->getInlineHeaderComment());
if ($controller->baseUrl) {
$pageRenderer->setBaseUrl($controller->baseUrl);
diff --git a/typo3/sysext/frontend/Classes/Middleware/PrepareTypoScriptFrontendRendering.php b/typo3/sysext/frontend/Classes/Middleware/PrepareTypoScriptFrontendRendering.php
index bd382fe3b259..9f18d6c1b31f 100644
--- a/typo3/sysext/frontend/Classes/Middleware/PrepareTypoScriptFrontendRendering.php
+++ b/typo3/sysext/frontend/Classes/Middleware/PrepareTypoScriptFrontendRendering.php
@@ -52,7 +52,7 @@ class PrepareTypoScriptFrontendRendering implements MiddlewareInterface
*/
public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
{
- /** @var TypoScriptFrontendController */
+ /** @var TypoScriptFrontendController $controller */
$controller = $request->getAttribute('frontend.controller');
// as long as TSFE throws errors with the global object, this needs to be set, but
@@ -67,14 +67,6 @@ class PrepareTypoScriptFrontendRendering implements MiddlewareInterface
// After this, we should have a valid config-array ready
$controller->getConfigArray($request);
- // Convert POST data to utf-8 for internal processing if metaCharset is different
- if ($controller->metaCharset !== 'utf-8' && $request->getMethod() === 'POST') {
- $parsedBody = $request->getParsedBody();
- if (is_array($parsedBody) && !empty($parsedBody)) {
- $this->convertCharsetRecursivelyToUtf8($parsedBody, $controller->metaCharset);
- $request = $request->withParsedBody($parsedBody);
- }
- }
$response = $handler->handle($request);
/**
@@ -87,21 +79,4 @@ class PrepareTypoScriptFrontendRendering implements MiddlewareInterface
return $response;
}
-
- /**
- * Small helper function to convert charsets for arrays to UTF-8
- *
- * @param mixed $data given by reference (string/array usually)
- * @param string $fromCharset convert FROM this charset
- */
- protected function convertCharsetRecursivelyToUtf8(&$data, string $fromCharset)
- {
- foreach ($data as $key => $value) {
- if (is_array($data[$key])) {
- $this->convertCharsetRecursivelyToUtf8($data[$key], $fromCharset);
- } elseif (is_string($data[$key])) {
- $data[$key] = mb_convert_encoding($data[$key], 'utf-8', $fromCharset);
- }
- }
- }
}
diff --git a/typo3/sysext/frontend/Tests/Unit/ContentObject/ContentObjectRendererTest.php b/typo3/sysext/frontend/Tests/Unit/ContentObject/ContentObjectRendererTest.php
index d0a6e938bcb4..918b785c35f3 100644
--- a/typo3/sysext/frontend/Tests/Unit/ContentObject/ContentObjectRendererTest.php
+++ b/typo3/sysext/frontend/Tests/Unit/ContentObject/ContentObjectRendererTest.php
@@ -1175,7 +1175,8 @@ class ContentObjectRendererTest extends UnitTestCase
*/
public function getDataWithTypeTsfe(): void
{
- self::assertEquals($GLOBALS['TSFE']->metaCharset, $this->subject->getData('tsfe:metaCharset'));
+ $GLOBALS['TSFE']->intTarget = 'foo';
+ self::assertEquals($GLOBALS['TSFE']->intTarget, $this->subject->getData('tsfe:intTarget'));
}
/**
@@ -1327,16 +1328,6 @@ class ContentObjectRendererTest extends UnitTestCase
self::assertEquals(2, $this->subject->getData('level'));
}
- /**
- * Checks if getData() works with type "global"
- *
- * @test
- */
- public function getDataWithTypeGlobal(): void
- {
- self::assertEquals($GLOBALS['TSFE']->metaCharset, $this->subject->getData('global:TSFE|metaCharset'));
- }
-
/**
* Checks if getData() works with type "leveltitle"
*
diff --git a/typo3/sysext/indexed_search/Classes/Controller/SearchController.php b/typo3/sysext/indexed_search/Classes/Controller/SearchController.php
index 5e683cd6d397..76683846c0ff 100644
--- a/typo3/sysext/indexed_search/Classes/Controller/SearchController.php
+++ b/typo3/sysext/indexed_search/Classes/Controller/SearchController.php
@@ -891,11 +891,6 @@ class SearchController extends ActionController
{
// Shorten search-word string to max 200 bytes - shortening the string here is only a run-away feature!
$searchWords = mb_substr($this->getSword(), 0, 200);
- // Convert to UTF-8 + conv. entities (was also converted during indexing!)
- if ($GLOBALS['TSFE']->metaCharset && $GLOBALS['TSFE']->metaCharset !== 'utf-8') {
- $searchWords = mb_convert_encoding($searchWords, 'utf-8', $GLOBALS['TSFE']->metaCharset);
- $searchWords = html_entity_decode($searchWords);
- }
$sWordArray = false;
if ($hookObj = $this->hookRequest('getSearchWords')) {
$sWordArray = $hookObj->getSearchWords_splitSWords($searchWords, $defaultOperator);
diff --git a/typo3/sysext/indexed_search/Classes/Hook/TypoScriptFrontendHook.php b/typo3/sysext/indexed_search/Classes/Hook/TypoScriptFrontendHook.php
index 0f9f2e52b5f2..f0c0c3998759 100644
--- a/typo3/sysext/indexed_search/Classes/Hook/TypoScriptFrontendHook.php
+++ b/typo3/sysext/indexed_search/Classes/Hook/TypoScriptFrontendHook.php
@@ -15,8 +15,6 @@
namespace TYPO3\CMS\IndexedSearch\Hook;
-use TYPO3\CMS\Core\Charset\CharsetConverter;
-use TYPO3\CMS\Core\Charset\UnknownCharsetException;
use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
use TYPO3\CMS\Core\Context\Context;
use TYPO3\CMS\Core\Context\LanguageAspect;
@@ -107,12 +105,10 @@ class TypoScriptFrontendHook
$configuration['rootline_uids'][$rlkey] = $rldat['uid'];
}
// Content of page
- $configuration['content'] = $this->convOutputCharset($tsfe->content, $tsfe->metaCharset);
+ $configuration['content'] = $tsfe->content;
// Content string (HTML of TYPO3 page)
- $configuration['indexedDocTitle'] = $this->convOutputCharset($tsfe->indexedDocTitle, $tsfe->metaCharset);
+ $configuration['indexedDocTitle'] = $tsfe->indexedDocTitle;
// Alternative title for indexing
- $configuration['metaCharset'] = $tsfe->metaCharset;
- // Character set of content (will be converted to utf-8 during indexing)
$configuration['mtime'] = $tsfe->register['SYS_LASTCHANGED'] ?? $tsfe->page['SYS_LASTCHANGED'];
// Most recent modification time (seconds) of the content on the page. Used to evaluate whether it should be re-indexed.
// Configuration of behavior
@@ -127,24 +123,4 @@ class TypoScriptFrontendHook
$configuration['freeIndexSetId'] = 0;
return $configuration;
}
-
- /**
- * Converts input string from utf-8 to metaCharset IF the two charsets are different.
- *
- * @param string $content Content to be converted.
- * @param string $metaCharset
- * @return string Converted content string.
- */
- protected function convOutputCharset(string $content, string $metaCharset): string
- {
- if ($metaCharset !== 'utf-8') {
- $charsetConverter = GeneralUtility::makeInstance(CharsetConverter::class);
- try {
- $content = $charsetConverter->conv($content, 'utf-8', $metaCharset);
- } catch (UnknownCharsetException $e) {
- throw new \RuntimeException('Invalid config.metaCharset: ' . $e->getMessage(), 1508916285);
- }
- }
- return $content;
- }
}
diff --git a/typo3/sysext/indexed_search/Classes/Indexer.php b/typo3/sysext/indexed_search/Classes/Indexer.php
index 259210b9ffde..4a9186b736eb 100644
--- a/typo3/sysext/indexed_search/Classes/Indexer.php
+++ b/typo3/sysext/indexed_search/Classes/Indexer.php
@@ -334,8 +334,8 @@ class Indexer
$checkCHash = $this->checkContentHash();
if (!is_array($checkCHash) || $check === 1) {
$Pstart = IndexedSearchUtility::milliseconds();
- $this->log_push('Converting charset of content (' . $this->conf['metaCharset'] . ') to utf-8', '');
- $this->charsetEntity2utf8($this->contentParts, $this->conf['metaCharset']);
+ $this->log_push('Converting entities of content', '');
+ $this->charsetEntity2utf8($this->contentParts);
$this->log_pull();
// Splitting words
$this->log_push('Extract words from content', '');
@@ -995,16 +995,12 @@ class Indexer
* Convert character set and HTML entities in the value of input content array keys
*
* @param array $contentArr Standard content array
- * @param string $charset Charset of the input content (converted to utf-8)
*/
- public function charsetEntity2utf8(&$contentArr, $charset)
+ public function charsetEntity2utf8(&$contentArr)
{
// Convert charset if necessary
foreach ($contentArr as $key => $value) {
if ((string)$contentArr[$key] !== '') {
- if ($charset !== 'utf-8') {
- $contentArr[$key] = mb_convert_encoding($contentArr[$key], 'utf-8', $charset);
- }
// decode all numeric / html-entities in the string to real characters:
$contentArr[$key] = html_entity_decode($contentArr[$key]);
}
diff --git a/typo3/sysext/install/Configuration/ExtensionScanner/Php/MethodCallMatcher.php b/typo3/sysext/install/Configuration/ExtensionScanner/Php/MethodCallMatcher.php
index 7fea2506ddfa..af009e8ee9ee 100644
--- a/typo3/sysext/install/Configuration/ExtensionScanner/Php/MethodCallMatcher.php
+++ b/typo3/sysext/install/Configuration/ExtensionScanner/Php/MethodCallMatcher.php
@@ -5218,4 +5218,11 @@ return [
'Deprecation-97027-ContentObjectRenderer-getTreeList.rst',
],
],
+ 'TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController->convOutputCharset' => [
+ 'numberOfMandatoryArguments' => 1,
+ 'maximumNumberOfArguments' => 1,
+ 'restFiles' => [
+ 'Breaking-97065-TYPO3FrontendAlwaysRenderedInUTF-8.rst',
+ ],
+ ],
];
diff --git a/typo3/sysext/install/Configuration/ExtensionScanner/Php/PropertyPublicMatcher.php b/typo3/sysext/install/Configuration/ExtensionScanner/Php/PropertyPublicMatcher.php
index e91621eea64d..68149ca07192 100644
--- a/typo3/sysext/install/Configuration/ExtensionScanner/Php/PropertyPublicMatcher.php
+++ b/typo3/sysext/install/Configuration/ExtensionScanner/Php/PropertyPublicMatcher.php
@@ -944,4 +944,9 @@ return [
'Breaking-96708-RemovedSupportForAccesskeysInHMENU.rst',
],
],
+ 'TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController->metaCharset' => [
+ 'restFiles' => [
+ 'Breaking-97065-TYPO3FrontendAlwaysRenderedInUTF-8.rst',
+ ],
+ ],
];
diff --git a/typo3/sysext/t3editor/Resources/Private/tsref.xml b/typo3/sysext/t3editor/Resources/Private/tsref.xml
index 058e3ff9293d..03e0db2601a4 100644
--- a/typo3/sysext/t3editor/Resources/Private/tsref.xml
+++ b/typo3/sysext/t3editor/Resources/Private/tsref.xml
@@ -476,15 +476,6 @@ config.message_preview_workspace = <div class="previewbox">Displaying workspace
<default><![CDATA[
]]></default>
</property>
- <property name="metaCharset" type="string">
- <description><![CDATA[Charset used for the output document. For example in the meta tag:
-<meta http-equiv="Content-Type" content="text/html; charset=...>
-
-Is used for a) HTML meta-tag, b) HTTP header (unless disabled with .disableCharsetHeader) and c) xhtml prologues (if available)
-
-If metaCharset is different than utf-8 the output content is automatically converted to metaCharset before output and likewise are values posted back to the page converted from metaCharset to utf-8 for internal processing. This conversion takes time of course so there is another good reason to use the same charset for both.]]></description>
- <default><![CDATA[utf-8]]></default>
- </property>
<property name="moveJsFromHeaderToFooter" type="boolean">
<description><![CDATA[
If set, all Javascript (includes and inline) will be moved to the bottom of the HTML document, which is after content and before the closing body tag
diff --git a/typo3/sysext/t3editor/Resources/Public/JavaScript/Mode/typoscript/typoscript.js b/typo3/sysext/t3editor/Resources/Public/JavaScript/Mode/typoscript/typoscript.js
index 5c6b2fb70944..7b369579f628 100644
--- a/typo3/sysext/t3editor/Resources/Public/JavaScript/Mode/typoscript/typoscript.js
+++ b/typo3/sysext/t3editor/Resources/Public/JavaScript/Mode/typoscript/typoscript.js
@@ -546,7 +546,6 @@
'message_preview': kw('message_preview'),
'META': kw('META'),
'meta': kw('meta'),
- 'metaCharset': kw('metaCharset'),
'method': kw('method'),
'min': kw('min'),
'minH': kw('minH'),
--
GitLab