CI: Overhaul static checks to use `pre-commit`

8 files changed, 211 insertions, 201 deletions

diff --git a/misc/scripts/codespell.sh b/misc/scripts/codespell.sh
deleted file mode 100755
index ef361a6495..0000000000
--- a/misc/scripts/codespell.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-SKIP_LIST="./.*,./**/.*,./bin,./thirdparty,*.desktop,*.gen.*,*.po,*.pot,*.rc,./AUTHORS.md,./COPYRIGHT.txt,./DONORS.md,"
-SKIP_LIST+="./core/input/gamecontrollerdb.txt,./core/string/locales.h,./editor/renames_map_3_to_4.cpp,./misc/scripts/codespell.sh,"
-SKIP_LIST+="./platform/android/java/lib/src/com,./platform/web/node_modules,./platform/web/package-lock.json,"
-
-IGNORE_LIST="breaked,cancelled,colour,curvelinear,doubleclick,expct,findn,gird,hel,inout,lod,mis,nd,numer,ot,requestor,te,thirdparty,vai"
-
-codespell -w -q 3 -S "${SKIP_LIST}" -L "${IGNORE_LIST}" --builtin "clear,rare,en-GB_to_en-US"
diff --git a/misc/scripts/dotnet_format.py b/misc/scripts/dotnet_format.py
index 83265be7c5..51fd7a1223 100644
--- a/misc/scripts/dotnet_format.py
+++ b/misc/scripts/dotnet_format.py
@@ -5,10 +5,16 @@ import glob
 import os
 import sys
 
-# Create dummy generated files.
+if len(sys.argv) < 2:
+    print("Invalid usage of dotnet_format.py, it should be called with a path to one or multiple files.")
+    sys.exit(1)
+
+# Create dummy generated files, if needed.
 for path in [
     "modules/mono/SdkPackageVersions.props",
 ]:
+    if os.path.exists(path):
+        continue
     os.makedirs(os.path.dirname(path), exist_ok=True)
     with open(path, "w", encoding="utf-8", newline="\n") as f:
         f.write("<Project />")
@@ -17,14 +23,12 @@ for path in [
 os.environ["GodotSkipGenerated"] = "true"
 
 # Match all the input files to their respective C# project.
-input_files = [os.path.normpath(x) for x in sys.argv]
 projects = {
-    path: [f for f in sys.argv if os.path.commonpath([f, path]) == path]
+    path: " ".join([f for f in sys.argv[1:] if os.path.commonpath([f, path]) == path])
     for path in [os.path.dirname(f) for f in glob.glob("**/*.csproj", recursive=True)]
 }
 
 # Run dotnet format on all projects with more than 0 modified files.
 for path, files in projects.items():
-    if len(files) > 0:
-        command = f"dotnet format {path} --include {' '.join(files)}"
-        os.system(command)
+    if files:
+        os.system(f"dotnet format {path} --include {files}")
diff --git a/misc/scripts/file_format.py b/misc/scripts/file_format.py
new file mode 100644
index 0000000000..a4ea544a45
--- /dev/null
+++ b/misc/scripts/file_format.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+
+if len(sys.argv) < 2:
+    print("Invalid usage of file_format.py, it should be called with a path to one or multiple files.")
+    sys.exit(1)
+
+BOM = b"\xef\xbb\xbf"
+
+changed = []
+invalid = []
+
+for file in sys.argv[1:]:
+    try:
+        with open(file, "rt", encoding="utf-8") as f:
+            original = f.read()
+    except UnicodeDecodeError:
+        invalid.append(file)
+        continue
+
+    if original == "":
+        continue
+
+    EOL = "\r\n" if file.endswith((".csproj", ".sln", ".bat")) or file.startswith("misc/msvs") else "\n"
+    WANTS_BOM = file.endswith((".csproj", ".sln"))
+
+    revamp = EOL.join([line.rstrip("\n\r\t ") for line in original.splitlines(True)]).rstrip(EOL) + EOL
+
+    new_raw = revamp.encode(encoding="utf-8")
+    if not WANTS_BOM and new_raw.startswith(BOM):
+        new_raw = new_raw[len(BOM) :]
+    elif WANTS_BOM and not new_raw.startswith(BOM):
+        new_raw = BOM + new_raw
+
+    with open(file, "rb") as f:
+        old_raw = f.read()
+
+    if old_raw != new_raw:
+        changed.append(file)
+        with open(file, "wb") as f:
+            f.write(new_raw)
+
+if changed:
+    for file in changed:
+        print(f"FIXED: {file}")
+if invalid:
+    for file in invalid:
+        print(f"REQUIRES MANUAL CHANGES: {file}")
+    sys.exit(1)
diff --git a/misc/scripts/file_format.sh b/misc/scripts/file_format.sh
deleted file mode 100755
index ad58657883..0000000000
--- a/misc/scripts/file_format.sh
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env bash
-
-# This script ensures proper POSIX text file formatting and a few other things.
-# This is supplementary to clang_format.sh and black_format.sh, but should be
-# run before them.
-
-# We need dos2unix and isutf8.
-if [ ! -x "$(command -v dos2unix)" -o ! -x "$(command -v isutf8)" ]; then
-    printf "Install 'dos2unix' and 'isutf8' (moreutils package) to use this script.\n"
-    exit 1
-fi
-
-set -uo pipefail
-
-if [ $# -eq 0 ]; then
-    # Loop through all code files tracked by Git.
-    mapfile -d '' files < <(git grep -zIl '')
-else
-    # $1 should be a file listing file paths to process. Used in CI.
-    mapfile -d ' ' < <(cat "$1")
-fi
-
-for f in "${files[@]}"; do
-    # Exclude some types of files.
-    if [[ "$f" == *"csproj" ]]; then
-        continue
-    elif [[ "$f" == *"sln" ]]; then
-        continue
-    elif [[ "$f" == *".bat" ]]; then
-        continue
-    elif [[ "$f" == *".out" ]]; then
-        # GDScript integration testing files.
-        continue
-    elif [[ "$f" == *"patch" ]]; then
-        continue
-    elif [[ "$f" == *"pot" ]]; then
-        continue
-    elif [[ "$f" == *"po" ]]; then
-        continue
-    elif [[ "$f" == "thirdparty/"* ]]; then
-        continue
-    elif [[ "$f" == *"/thirdparty/"* ]]; then
-        continue
-    elif [[ "$f" == "platform/android/java/lib/src/com/google"* ]]; then
-        continue
-    elif [[ "$f" == *"-so_wrap."* ]]; then
-        continue
-    elif [[ "$f" == *".test.txt" ]]; then
-        continue
-    fi
-    # Ensure that files are UTF-8 formatted.
-    isutf8 "$f" >> utf8-validation.txt 2>&1
-    # Ensure that files have LF line endings and do not contain a BOM.
-    dos2unix "$f" 2> /dev/null
-    # Remove trailing space characters and ensures that files end
-    # with newline characters. -l option handles newlines conveniently.
-    perl -i -ple 's/\s*$//g' "$f"
-done
-
-diff=$(git diff --color)
-
-if [ ! -s utf8-validation.txt ] && [ -z "$diff" ] ; then
-    # If no UTF-8 violations were collected (the file is empty) and
-    # no diff has been generated all is OK, clean up, and exit.
-    printf "\e[1;32m*** Files in this commit comply with the file formatting rules.\e[0m\n"
-    rm -f utf8-validation.txt
-    exit 0
-fi
-
-if [ -s utf8-validation.txt ]
-then
-    # If the file has content and is not empty, violations
-    # detected, notify the user, clean up, and exit.
-    printf "\n\e[1;33m*** The following files contain invalid UTF-8 character sequences:\e[0m\n\n"
-    cat utf8-validation.txt
-fi
-
-rm -f utf8-validation.txt
-
-if [ ! -z "$diff" ]
-then
-    # A diff has been created, notify the user, clean up, and exit.
-    printf "\n\e[1;33m*** The following changes must be made to comply with the formatting rules:\e[0m\n\n"
-    # Perl commands replace trailing spaces with `·` and tabs with `<TAB>`.
-    printf "%s\n" "$diff" | perl -pe 's/(.*[^ ])( +)(\e\[m)$/my $spaces="·" x length($2); sprintf("$1$spaces$3")/ge' | perl -pe 's/(.*[^\t])(\t+)(\e\[m)$/my $tabs="<TAB>" x length($2); sprintf("$1$tabs$3")/ge'
-fi
-
-printf "\n\e[1;91m*** Please fix your commit(s) with 'git commit --amend' or 'git rebase -i <hash>'\e[0m\n"
-exit 1
diff --git a/misc/scripts/header_guards.py b/misc/scripts/header_guards.py
new file mode 100644
index 0000000000..b554be5159
--- /dev/null
+++ b/misc/scripts/header_guards.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+from pathlib import Path
+
+if len(sys.argv) < 2:
+    print("Invalid usage of header_guards.py, it should be called with a path to one or multiple files.")
+    sys.exit(1)
+
+HEADER_CHECK_OFFSET = 30
+HEADER_BEGIN_OFFSET = 31
+HEADER_END_OFFSET = -1
+
+changed = []
+invalid = []
+
+for file in sys.argv[1:]:
+    with open(file, "rt", encoding="utf-8", newline="\n") as f:
+        lines = f.readlines()
+
+    if len(lines) <= HEADER_BEGIN_OFFSET:
+        continue  # Most likely a dummy file.
+
+    if lines[HEADER_CHECK_OFFSET].startswith("#import"):
+        continue  # Early catch obj-c file.
+
+    split = file.split("/")  # Already in posix-format.
+
+    prefix = ""
+    if split[0] == "modules" and split[-1] == "register_types.h":
+        prefix = f"{split[1]}_"  # Name of module.
+    elif split[0] == "platform" and (file.endswith("api/api.h") or "/export/" in file):
+        prefix = f"{split[1]}_"  # Name of platform.
+    elif file.startswith("modules/mono/utils") and "mono" not in split[-1]:
+        prefix = "MONO_"
+    elif file == "servers/rendering/storage/utilities.h":
+        prefix = "RENDERER_"
+
+    suffix = ""
+    if "dummy" in file and "dummy" not in split[-1]:
+        suffix = "_DUMMY"
+    elif "gles3" in file and "gles3" not in split[-1]:
+        suffix = "_GLES3"
+    elif "renderer_rd" in file and "rd" not in split[-1]:
+        suffix = "_RD"
+    elif split[-1] == "ustring.h":
+        suffix = "_GODOT"
+
+    name = (f"{prefix}{Path(file).stem}{suffix}{Path(file).suffix}".upper()
+            .replace(".", "_").replace("-", "_").replace(" ", "_"))  # fmt: skip
+
+    HEADER_CHECK = f"#ifndef {name}\n"
+    HEADER_BEGIN = f"#define {name}\n"
+    HEADER_END = f"#endif // {name}\n"
+
+    if (
+        lines[HEADER_CHECK_OFFSET] == HEADER_CHECK
+        and lines[HEADER_BEGIN_OFFSET] == HEADER_BEGIN
+        and lines[HEADER_END_OFFSET] == HEADER_END
+    ):
+        continue
+
+    # Guards might exist but with the wrong names.
+    if (
+        lines[HEADER_CHECK_OFFSET].startswith("#ifndef")
+        and lines[HEADER_BEGIN_OFFSET].startswith("#define")
+        and lines[HEADER_END_OFFSET].startswith("#endif")
+    ):
+        lines[HEADER_CHECK_OFFSET] = HEADER_CHECK
+        lines[HEADER_BEGIN_OFFSET] = HEADER_BEGIN
+        lines[HEADER_END_OFFSET] = HEADER_END
+        with open(file, "wt", encoding="utf-8", newline="\n") as f:
+            f.writelines(lines)
+        changed.append(file)
+        continue
+
+    header_check = -1
+    header_begin = -1
+    header_end = -1
+    pragma_once = -1
+    objc = False
+
+    for idx, line in enumerate(lines):
+        if not line.startswith("#"):
+            continue
+        elif line.startswith("#ifndef") and header_check == -1:
+            header_check = idx
+        elif line.startswith("#define") and header_begin == -1:
+            header_begin = idx
+        elif line.startswith("#endif") and header_end == -1:
+            header_end = idx
+        elif line.startswith("#pragma once"):
+            pragma_once = idx
+            break
+        elif line.startswith("#import"):
+            objc = True
+            break
+
+    if objc:
+        continue
+
+    if pragma_once != -1:
+        lines.pop(pragma_once)
+        lines.insert(HEADER_CHECK_OFFSET, HEADER_CHECK)
+        lines.insert(HEADER_BEGIN_OFFSET, HEADER_BEGIN)
+        lines.append("\n")
+        lines.append(HEADER_END)
+        with open(file, "wt", encoding="utf-8", newline="\n") as f:
+            f.writelines(lines)
+        changed.append(file)
+        continue
+
+    if header_check == -1 and header_begin == -1 and header_end == -1:
+        # Guards simply didn't exist
+        lines.insert(HEADER_CHECK_OFFSET, HEADER_CHECK)
+        lines.insert(HEADER_BEGIN_OFFSET, HEADER_BEGIN)
+        lines.append("\n")
+        lines.append(HEADER_END)
+        with open(file, "wt", encoding="utf-8", newline="\n") as f:
+            f.writelines(lines)
+        changed.append(file)
+        continue
+
+    if header_check != -1 and header_begin != -1 and header_end != -1:
+        # All prepends "found", see if we can salvage this.
+        if header_check == header_begin - 1 and header_begin < header_end:
+            lines.pop(header_check)
+            lines.pop(header_begin - 1)
+            lines.pop(header_end - 2)
+            if lines[header_end - 3] == "\n":
+                lines.pop(header_end - 3)
+            lines.insert(HEADER_CHECK_OFFSET, HEADER_CHECK)
+            lines.insert(HEADER_BEGIN_OFFSET, HEADER_BEGIN)
+            lines.append("\n")
+            lines.append(HEADER_END)
+            with open(file, "wt", encoding="utf-8", newline="\n") as f:
+                f.writelines(lines)
+            changed.append(file)
+            continue
+
+    invalid.append(file)
+
+if changed:
+    for file in changed:
+        print(f"FIXED: {file}")
+if invalid:
+    for file in invalid:
+        print(f"REQUIRES MANUAL CHANGES: {file}")
+    sys.exit(1)
diff --git a/misc/scripts/header_guards.sh b/misc/scripts/header_guards.sh
deleted file mode 100755
index a79ccd4bee..0000000000
--- a/misc/scripts/header_guards.sh
+++ /dev/null
@@ -1,87 +0,0 @@
-#!/bin/bash
-
-if [ ! -f "version.py" ]; then
-  echo "Warning: This script is intended to be run from the root of the Godot repository."
-  echo "Some of the paths checks may not work as intended from a different folder."
-fi
-
-if [ $# -eq 0 ]; then
-    # Loop through all code files tracked by Git.
-    files=$(find -name "thirdparty" -prune -o -name "*.h" -print | sed "s@^\./@@g")
-else
-    # $1 should be a file listing file paths to process. Used in CI.
-    files=$(cat "$1" | grep -v "thirdparty/" | grep -E "\.h$" | sed "s@^\./@@g")
-fi
-
-files_invalid_guard=""
-
-for file in $files; do
-  # Skip *.gen.h and *-so_wrap.h, they're generated.
-  if [[ "$file" == *".gen.h" || "$file" == *"-so_wrap.h" ]]; then continue; fi
-  # Has important define before normal header guards.
-  if [[ "$file" == *"thread.h" || "$file" == *"platform_config.h" || "$file" == *"platform_gl.h" ]]; then continue; fi
-  # Obj-C files don't use header guards.
-  if grep -q "#import " "$file"; then continue; fi
-
-  bname=$(basename $file .h)
-
-  # Add custom prefix or suffix for generic filenames with a well-defined namespace.
-
-  prefix=
-  if [[ "$file" == "modules/"*"/register_types.h" ]]; then
-    module=$(echo $file | sed "s@.*modules/\([^/]*\).*@\1@")
-    prefix="${module^^}_"
-  fi
-  if [[ "$file" == "platform/"*"/api/api.h" || "$file" == "platform/"*"/export/"* ]]; then
-    platform=$(echo $file | sed "s@.*platform/\([^/]*\).*@\1@")
-    prefix="${platform^^}_"
-  fi
-  if [[ "$file" == "modules/mono/utils/"* && "$bname" != *"mono"* ]]; then prefix="MONO_"; fi
-  if [[ "$file" == "servers/rendering/storage/utilities.h" ]]; then prefix="RENDERER_"; fi
-
-  suffix=
-  if [[ "$file" == *"dummy"* && "$bname" != *"dummy"* ]]; then suffix="_DUMMY"; fi
-  if [[ "$file" == *"gles3"* && "$bname" != *"gles3"* ]]; then suffix="_GLES3"; fi
-  if [[ "$file" == *"renderer_rd"* && "$bname" != *"rd"* ]]; then suffix="_RD"; fi
-  if [[ "$file" == *"ustring.h" ]]; then suffix="_GODOT"; fi
-
-  # ^^ is bash builtin for UPPERCASE.
-  guard="${prefix}${bname^^}${suffix}_H"
-
-  # Replaces guards to use computed name.
-  # We also add some \n to make sure there's a proper separation.
-  sed -i $file -e "0,/ifndef/s/#ifndef.*/\n#ifndef $guard/"
-  sed -i $file -e "0,/define/s/#define.*/#define $guard\n/"
-  sed -i $file -e "$ s/#endif.*/\n#endif \/\/ $guard/"
-  # Removes redundant \n added before, if they weren't needed.
-  sed -i $file -e "/^$/N;/^\n$/D"
-
-  # Check that first ifndef (should be header guard) is at the expected position.
-  # If not it can mean we have some code before the guard that should be after.
-  # "31" is the expected line with the copyright header.
-  first_ifndef=$(grep -n -m 1 "ifndef" $file | sed 's/\([0-9]*\).*/\1/')
-  if [[ "$first_ifndef" != "31" ]]; then
-    files_invalid_guard+="$file\n"
-  fi
-done
-
-if [[ ! -z "$files_invalid_guard" ]]; then
-  echo -e "The following files were found to have potentially invalid header guard:\n"
-  echo -e "$files_invalid_guard"
-fi
-
-diff=$(git diff --color)
-
-# If no diff has been generated all is OK, clean up, and exit.
-if [ -z "$diff" ] ; then
-    printf "\e[1;32m*** Files in this commit comply with the header guards formatting rules.\e[0m\n"
-    exit 0
-fi
-
-# A diff has been created, notify the user, clean up, and exit.
-printf "\n\e[1;33m*** The following changes must be made to comply with the formatting rules:\e[0m\n\n"
-# Perl commands replace trailing spaces with `·` and tabs with `<TAB>`.
-printf "%s\n" "$diff" | perl -pe 's/(.*[^ ])( +)(\e\[m)$/my $spaces="·" x length($2); sprintf("$1$spaces$3")/ge' | perl -pe 's/(.*[^\t])(\t+)(\e\[m)$/my $tabs="<TAB>" x length($2); sprintf("$1$tabs$3")/ge'
-
-printf "\n\e[1;91m*** Please fix your commit(s) with 'git commit --amend' or 'git rebase -i <hash>'\e[0m\n"
-exit 1
diff --git a/misc/scripts/mypy_check.sh b/misc/scripts/mypy_check.sh
deleted file mode 100755
index 2a06486d67..0000000000
--- a/misc/scripts/mypy_check.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-set -uo pipefail
-
-echo -e "Python: mypy static analysis..."
-mypy --config-file=./misc/scripts/mypy.ini .
diff --git a/misc/scripts/pytest_builders.sh b/misc/scripts/pytest_builders.sh
deleted file mode 100755
index eb2ddbcddc..0000000000
--- a/misc/scripts/pytest_builders.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-set -uo pipefail
-
-echo "Running Python checks for builder system"
-pytest ./tests/python_build