From 1e52b4014102fd8067fa1b2a134066f1194e5faf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stefan=20B=C3=BCrk?= <stefan@buerk.tech>
Date: Thu, 25 Aug 2022 10:17:11 +0200
Subject: [PATCH] [TASK] Ensure php files contain correct legal file header
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Core php file header check got silently be disabled. Since
then the one or other file header error has slipped into
the core. It also pops up from time to time in reviews of
patches that file headers are missing (or are wrong).
This change ensures that file headers are checked again
and can be fixed by given contributers the proper tools
at hand.
Changes:
* add cglFixMyCommitFileHeader.(sh|bat)
* add cglHeader and cglHeaderGit to runTests.sh along
with proper help text
* extend pre-commit hook to execute both fixer scripts
side-by-side with error output and fixing possibility
* fix minor issue with header comment php-cs-fixer config
Additionally the cglHeader check is added to gitlab workflow
pre-merge (commit files) and nightly (all files).
Slipped in errors has been fixed with a dedicated change #98204.
Resolves: #98203
Related: #98204
Releases: main
Change-Id: I9567d1dcf8fb53d4901a575213c0ff16dc33ffd2
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/75549
Tested-by: core-ci <typo3@b13.com>
Tested-by: Benjamin Franzke <bfr@qbus.de>
Tested-by: Christian Kuhn <lolli@schwarzbu.ch>
Tested-by: Andreas Fernandez <a.fernandez@scripting-base.de>
Tested-by: Stefan Bürk <stefan@buerk.tech>
Reviewed-by: Benjamin Franzke <bfr@qbus.de>
Reviewed-by: Christian Kuhn <lolli@schwarzbu.ch>
Reviewed-by: Andreas Fernandez <a.fernandez@scripting-base.de>
Reviewed-by: Stefan Bürk <stefan@buerk.tech>
---
Build/Scripts/cglFixMyCommitFileHeader.bat | 4 +
Build/Scripts/cglFixMyCommitFileHeader.sh | 145 ++++++++++++++++++
Build/Scripts/runTests.sh | 20 ++-
Build/git-hooks/unix+mac/pre-commit | 67 +++++---
Build/gitlab-ci/nightly/integrity.yml | 1 +
Build/gitlab-ci/pre-merge/integrity.yml | 1 +
Build/php-cs-fixer/header-comment.php | 7 +-
Build/testing-docker/local/docker-compose.yml | 29 ++++
8 files changed, 253 insertions(+), 21 deletions(-)
create mode 100644 Build/Scripts/cglFixMyCommitFileHeader.bat
create mode 100755 Build/Scripts/cglFixMyCommitFileHeader.sh
diff --git a/Build/Scripts/cglFixMyCommitFileHeader.bat b/Build/Scripts/cglFixMyCommitFileHeader.bat
new file mode 100644
index 000000000000..8c4f96ddd094
--- /dev/null
+++ b/Build/Scripts/cglFixMyCommitFileHeader.bat
@@ -0,0 +1,4 @@
+@ECHO OFF
+FOR /F %%i in ('git diff-tree --no-commit-id --name-only -r HEAD ^| findstr -i ".*\.php"') DO (
+ bin\php-cs-fixer.bat fix --path-mode intersection --config=Build/php-cs-fixer/header-comment.php %%i
+)
diff --git a/Build/Scripts/cglFixMyCommitFileHeader.sh b/Build/Scripts/cglFixMyCommitFileHeader.sh
new file mode 100755
index 000000000000..a637b6af542f
--- /dev/null
+++ b/Build/Scripts/cglFixMyCommitFileHeader.sh
@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+
+#########################
+#
+# CGL fix.
+#
+# It expects to be run from the core root.
+#
+# To auto-fix single files, use the php-cs-fixer command directly
+# substitute $FILE with a filename
+#
+##########################
+
+# --------------------------
+# --- default parameters ---
+# --------------------------
+# check files in last commit
+filestype=commit
+# non-dryrun is default
+DRYRUN=""
+DRYRUN_OPTIONS="--dry-run --diff"
+
+# ----------------------
+# --- automatic vars ---
+# ----------------------
+progname=$(basename $0)
+
+# ------------------------
+# --- print usage info ---
+# ------------------------
+usage()
+{
+ echo "Usage: $0 [options] "
+ echo " "
+ echo "no arguments/default: fix all php files in last commit "
+ echo " "
+ echo "Options: "
+ echo " -f <commit|cache|stdin> "
+ echo " specifies which files to check: "
+ echo " - commit (default): all files in latest commit "
+ echo " - cache : all files in git cache (staging area) "
+ echo " - stdin : read list of files from stdin "
+ echo " "
+ echo " -n "
+ echo " dryrun only, do not fix anything! "
+ echo " "
+ echo " -h "
+ echo " help "
+ echo " "
+ echo "Note: In order to still support command line options of "
+ echo " older versions of this script, you can use the argument "
+ echo " dryrun. "
+ echo " "
+ echo " THIS IS NOT RECOMMENDED but will still work for now "
+ echo " Usage: $0 [options] [dryrun] "
+ exit 0
+}
+
+# -----------------------
+# --- parsing of args ---
+# -----------------------
+OPTIND=1
+
+while getopts "hnf:" opt;do
+ case "$opt" in
+ h)
+ usage
+ ;;
+ f)
+ filestype=$OPTARG
+ echo "$0 files type=$filestype"
+ ;;
+ n)
+ echo "$progname: dryrun mode"
+ DRYRUN="$DRYRUN_OPTIONS"
+ ;;
+ esac
+done
+
+shift $((OPTIND-1))
+
+if [ "$1" = "dryrun" ]
+then
+ echo "$progname: dryrun mode"
+ DRYRUN="$DRYRUN_OPTIONS"
+fi
+
+# --------------------------------------
+# --- check if php executable exists ---
+# --------------------------------------
+exist_php_executable() {
+ which php >/dev/null 2>/dev/null
+ if [ $? -ne 0 ];then
+ echo "$progname: No php executable found\n"
+ exit 1
+ fi
+}
+
+
+# ------------------------------
+# --- run php without xdebug ---
+# ------------------------------
+php_no_xdebug ()
+{
+ temporaryPath="$(mktemp -t php.XXXXXX).ini"
+ php -i | grep "\.ini" | grep -o -e '\(/[A-Za-z0-9._-]\+\)\+\.ini' | grep -v xdebug | xargs awk 'FNR==1{print ""}1' > "${temporaryPath}"
+ php -n -c "${temporaryPath}" "$@"
+ RETURN=$?
+ rm -f "${temporaryPath}"
+ exit $RETURN
+}
+
+# ------------------------------------
+# --- get a list of files to check ---
+# ------------------------------------
+if [[ $filestype == commit ]];then
+ echo "$progname: Searching for php files in latest git commit ..."
+ DETECTED_FILES=`git diff-tree --no-commit-id --name-only -r HEAD | grep '.php$' 2>/dev/null`
+elif [[ $filestype == cache ]];then
+ echo "$progname: Searching for php files in git cache ..."
+ DETECTED_FILES=`git diff --cached --name-only | grep '.php$' 2>/dev/null`
+elif [[ $filestype == stdin ]];then
+ echo "$progname: reading list of php files to check from stdin"
+ DETECTED_FILES=$(cat)
+else
+ echo "$progname: ERROR: unknown filetype, possibly used -f with wrong argument"
+ usage
+fi
+if [ -z "${DETECTED_FILES}" ]
+then
+ echo "$progname: No PHP files to check, all is well."
+ exit 0
+fi
+
+# ---------------------------------
+# --- run php-cs-fixer on files ---
+# ---------------------------------
+exist_php_executable
+php_no_xdebug ./bin/php-cs-fixer fix \
+ -v ${DRYRUN} \
+ --path-mode intersection \
+ --config=Build/php-cs-fixer/header-comment.php \
+ `echo ${DETECTED_FILES} | xargs ls -d 2>/dev/null`
+
+exit $?
diff --git a/Build/Scripts/runTests.sh b/Build/Scripts/runTests.sh
index 5a0b0e2cbd60..a4e9cbfc052f 100755
--- a/Build/Scripts/runTests.sh
+++ b/Build/Scripts/runTests.sh
@@ -135,6 +135,8 @@ Options:
- buildJavascript: execute typescript to javascript builder
- cgl: test and fix all core php files
- cglGit: test and fix latest committed patch for CGL compliance
+ - cglHeader: test and fix file header for all core php files
+ - cglHeaderGit: test and fix latest committed patch for CGL file header compliance
- checkAnnotations: check php code for allowed annotations
- checkBom: check UTF-8 files do not contain BOM
- checkComposer: check composer.json files for version integrity
@@ -246,7 +248,7 @@ Options:
replay the unit tests in that order.
-n
- Only with -s cgl|cglGit
+ Only with -s cgl|cglGit|cglHeader|cglGitHeader
Activate dry-run in CGL check that does not actively change files and only prints broken ones.
-u
@@ -559,6 +561,22 @@ case ${TEST_SUITE} in
SUITE_EXIT_CODE=$?
docker-compose down
;;
+ cglHeader)
+ # Active dry-run for cgl needs not "-n" but specific options
+ if [ -n "${CGLCHECK_DRY_RUN}" ]; then
+ CGLCHECK_DRY_RUN="--dry-run --diff"
+ fi
+ setUpDockerComposeDotEnv
+ docker-compose run cgl_header_all
+ SUITE_EXIT_CODE=$?
+ docker-compose down
+ ;;
+ cglHeaderGit)
+ setUpDockerComposeDotEnv
+ docker-compose run cgl_header_git
+ SUITE_EXIT_CODE=$?
+ docker-compose down
+ ;;
checkAnnotations)
setUpDockerComposeDotEnv
docker-compose run check_annotations
diff --git a/Build/git-hooks/unix+mac/pre-commit b/Build/git-hooks/unix+mac/pre-commit
index eb964220f8b5..f8751b85b6ae 100755
--- a/Build/git-hooks/unix+mac/pre-commit
+++ b/Build/git-hooks/unix+mac/pre-commit
@@ -1,26 +1,34 @@
#!/usr/bin/env bash
# pre-commit hook
-# - check if staged files adhere to Coding Guidelines
+# - check if staged files adhere to Coding Guidelines and have proper file header
# - abort on error:no for now, override with environment variable TYPO3_GIT_HOOK_ABORT_ON_ERROR
#
# Dependencies:
# uses: Build/Scripts/cglFixMyCommit.sh
#
+USE_EXIT_CODE=0
script=Build/Scripts/cglFixMyCommit.sh
+scriptFileHeader=Build/Scripts/cglFixMyCommitFileHeader.sh
ABORT_ON_ERROR="no"
ERROR_TEXT="\n"
ERROR_CODE=0
+ERROR_CODE_FILE_HEADER=0
if [ -z "${TYPO3_GIT_HOOK_ABORT_ON_ERROR+x}" ]; then
ABORT_ON_ERROR=${TYPO3_GIT_HOOK_ABORT_ON_ERROR}
fi
if [ ! -x $script ];then
- "echo "$script does not exist or is not executable"
- "exit 0
+ echo "$script does not exist or is not executable"
+ exit 0
+fi
+
+if [ ! -x $scriptFileHeader ];then
+ echo "$scriptFileHeader does not exist or is not executable"
+ exit 0
fi
# call script with -f <cache> parameter: check files in git cache (staging area)
@@ -28,21 +36,42 @@ fi
$script -f cache -n
ERROR_CODE=$?
-
if [ ${ERROR_CODE} -ne 0 ];then
- echo -e "\n-----------------------------------------------------------------\n"
- echo -e " >> ERROR: There was a coding guideline problem in one or more of "
- echo -e " your php files. "
- echo -e " Please refer to [1] for details on the coding guidelines "
- echo -e " Please refer to [2] for details on contribution "
- echo -e " [1] https://docs.typo3.org/typo3cms/CoreApiReference/CodingGuidelines/Index.html"
- echo -e " [2] https://docs.typo3.org/typo3cms/ContributionWorkflowGuide/ "
- echo -e "------------------------------------------------------------------\n"
- if [[ ${ABORT_ON_ERROR} == "yes" ]];then
- echo -e " Your commit is being aborted ... Fix and try again! "
- exit 1
- else
- echo -e " You must fix this and then commit again (git commit --amend) "
- fi
+ echo -e "\n-----------------------------------------------------------------\n"
+ echo -e " >> ERROR: There was a coding guideline problem in one or more of "
+ echo -e " your php files. "
+ echo -e " Please refer to [1] for details on the coding guidelines "
+ echo -e " Please refer to [2] for details on contribution "
+ echo -e " [1] https://docs.typo3.org/typo3cms/CoreApiReference/CodingGuidelines/Index.html"
+ echo -e " [2] https://docs.typo3.org/typo3cms/ContributionWorkflowGuide/ "
+ echo -e "------------------------------------------------------------------\n"
+ if [[ ${ABORT_ON_ERROR} == "yes" ]];then
+ echo -e " Your commit is being aborted ... Fix and try again! "
+ USE_EXIT_CODE=1
+ else
+ echo -e " You must fix this and then commit again (git commit --amend) "
+ fi
+fi
+
+$scriptFileHeader -f cache -n
+ERROR_CODE_FILE_HEADER=$?
+
+if [ ${ERROR_CODE_FILE_HEADER} -ne 0 ];then
+
+ echo -e "\n-----------------------------------------------------------------\n"
+ echo -e " >> ERROR: There was a missing or wrong php file header in one or "
+ echo -e " more of your php files. "
+ echo -e " Please refer to [1] for details on the coding guidelines "
+ echo -e " Please refer to [2] for details on contribution "
+ echo -e " [1] https://docs.typo3.org/typo3cms/CoreApiReference/CodingGuidelines/Index.html"
+ echo -e " [2] https://docs.typo3.org/typo3cms/ContributionWorkflowGuide/ "
+ echo -e "------------------------------------------------------------------\n"
+ if [[ ${ABORT_ON_ERROR} == "yes" ]];then
+ echo -e " Your commit is being aborted ... Fix and try again! "
+ USE_EXIT_CODE=1
+ else
+ echo -e " You must fix this and then commit again (git commit --amend) "
+ fi
fi
-exit 0
+
+exit ${USE_EXIT_CODE}
diff --git a/Build/gitlab-ci/nightly/integrity.yml b/Build/gitlab-ci/nightly/integrity.yml
index 5592c03d4d10..b43c2f2c2e5e 100644
--- a/Build/gitlab-ci/nightly/integrity.yml
+++ b/Build/gitlab-ci/nightly/integrity.yml
@@ -15,6 +15,7 @@ cgl:
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
- Build/Scripts/runTests.sh -s cgl -n -p 8.1
+ - Build/Scripts/runTests.sh -s cglHeader -n -p 8.1
grunt clean:
stage: integrity
diff --git a/Build/gitlab-ci/pre-merge/integrity.yml b/Build/gitlab-ci/pre-merge/integrity.yml
index 37b919990213..5a380a9e79fb 100644
--- a/Build/gitlab-ci/pre-merge/integrity.yml
+++ b/Build/gitlab-ci/pre-merge/integrity.yml
@@ -17,6 +17,7 @@ cgl pre-merge:
script:
- Build/Scripts/runTests.sh -s composerInstall -p 8.1
- Build/Scripts/runTests.sh -s cglGit -n -p 8.1
+ - Build/Scripts/runTests.sh -s cglHeaderGit -n -p 8.1
grunt clean pre-merge:
stage: main
diff --git a/Build/php-cs-fixer/header-comment.php b/Build/php-cs-fixer/header-comment.php
index 417b9859199a..19e416c90bed 100644
--- a/Build/php-cs-fixer/header-comment.php
+++ b/Build/php-cs-fixer/header-comment.php
@@ -30,6 +30,10 @@ $finder = PhpCsFixer\Finder::create()
->notName('ext_localconf.php')
->notName('ext_tables.php')
->notName('ext_emconf.php')
+ // ClassAliasMap files do not need header comments
+ ->notName('ClassAliasMap.php')
+ // CodeSnippets and Examples in Documentation do not need header comments
+ ->exclude('Documentation')
// Third-party inclusion files should not have a changed comment
->notName('Rfc822AddressesParser.php')
->notName('ClassMapGenerator.php')
@@ -48,9 +52,10 @@ LICENSE.txt file that was distributed with this source code.
The TYPO3 project - inspiring people to share!
COMMENT;
-return PhpCsFixer\Config::create()
+return (new \PhpCsFixer\Config())
->setRiskyAllowed(false)
->setRules([
+ 'no_extra_blank_lines' => true,
'header_comment' => [
'header' => $headerComment,
'comment_type' => 'comment',
diff --git a/Build/testing-docker/local/docker-compose.yml b/Build/testing-docker/local/docker-compose.yml
index 7e41126850b2..9ac0628e453d 100644
--- a/Build/testing-docker/local/docker-compose.yml
+++ b/Build/testing-docker/local/docker-compose.yml
@@ -513,6 +513,35 @@ services:
--config=Build/php-cs-fixer.php typo3/
"
+ cgl_header_git:
+ image: typo3/core-testing-${DOCKER_PHP_IMAGE}:latest
+ user: "${HOST_UID}"
+ volumes:
+ - ${CORE_ROOT}:${CORE_ROOT}
+ working_dir: ${CORE_ROOT}
+ command: >
+ /bin/sh -c "
+ if [ ${SCRIPT_VERBOSE} -eq 1 ]; then
+ set -x
+ fi
+ Build/Scripts/cglFixMyCommitFileHeader.sh ${CGLCHECK_DRY_RUN};
+ "
+
+ cgl_header_all:
+ image: typo3/core-testing-${DOCKER_PHP_IMAGE}:latest
+ user: "${HOST_UID}"
+ volumes:
+ - ${CORE_ROOT}:${CORE_ROOT}
+ working_dir: ${CORE_ROOT}
+ command: >
+ /bin/sh -c "
+ if [ ${SCRIPT_VERBOSE} -eq 1 ]; then
+ set -x
+ fi
+ php -dxdebug.mode=off bin/php-cs-fixer fix -v ${CGLCHECK_DRY_RUN} --path-mode intersection \
+ --config=Build/php-cs-fixer/header-comment.php typo3/
+ "
+
check_annotations:
image: typo3/core-testing-${DOCKER_PHP_IMAGE}:latest
user: "${HOST_UID}"
--
GitLab