Merge pull request #92087 from Repiteo/scons/windows-color-fix-maybe

SCons: Fix potential Windows ANSI exception

4 files changed, 67 insertions, 38 deletions

diff --git a/SConstruct b/SConstruct
index c668ac7df3..afa44fef4a 100644
--- a/SConstruct
+++ b/SConstruct
@@ -15,16 +15,6 @@ from collections import OrderedDict
 from importlib.util import spec_from_file_location, module_from_spec
 from SCons import __version__ as scons_raw_version
 
-# Enable ANSI escape code support on Windows 10 and later (for colored console output).
-# <https://github.com/python/cpython/issues/73245>
-if sys.platform == "win32":
-    from ctypes import windll, c_int, byref
-
-    stdout_handle = windll.kernel32.GetStdHandle(c_int(-11))
-    mode = c_int(0)
-    windll.kernel32.GetConsoleMode(c_int(stdout_handle), byref(mode))
-    mode = c_int(mode.value | 4)
-    windll.kernel32.SetConsoleMode(c_int(stdout_handle), mode)
 
 # Explicitly resolve the helper modules, this is done to avoid clash with
 # modules of the same name that might be randomly added (e.g. someone adding
@@ -74,6 +64,24 @@ if ARGUMENTS.get("target", "editor") == "editor":
     _helper_module("editor.editor_builders", "editor/editor_builders.py")
     _helper_module("editor.template_builders", "editor/template_builders.py")
 
+# Enable ANSI escape code support on Windows 10 and later (for colored console output).
+# <https://github.com/python/cpython/issues/73245>
+if sys.stdout.isatty() and sys.platform == "win32":
+    try:
+        from ctypes import windll, byref, WinError  # type: ignore
+        from ctypes.wintypes import DWORD  # type: ignore
+
+        stdout_handle = windll.kernel32.GetStdHandle(DWORD(-11))
+        mode = DWORD(0)
+        if not windll.kernel32.GetConsoleMode(stdout_handle, byref(mode)):
+            raise WinError()
+        mode = DWORD(mode.value | 4)
+        if not windll.kernel32.SetConsoleMode(stdout_handle, mode):
+            raise WinError()
+    except Exception as e:
+        methods._colorize = False
+        print_error(f"Failed to enable ANSI escape code support, disabling color output.\n{e}")
+
 # Scan possible build platforms
 
 platform_list = []  # list of platforms
diff --git a/doc/tools/make_rst.py b/doc/tools/make_rst.py
index 0b722757f9..43e1afe720 100755
--- a/doc/tools/make_rst.py
+++ b/doc/tools/make_rst.py
@@ -677,17 +677,6 @@ class ScriptLanguageParityCheck:
 
 # Entry point for the RST generator.
 def main() -> None:
-    # Enable ANSI escape code support on Windows 10 and later (for colored console output).
-    # <https://bugs.python.org/issue29059>
-    if platform.system().lower() == "windows":
-        from ctypes import windll, c_int, byref  # type: ignore
-
-        stdout_handle = windll.kernel32.GetStdHandle(c_int(-11))
-        mode = c_int(0)
-        windll.kernel32.GetConsoleMode(c_int(stdout_handle), byref(mode))
-        mode = c_int(mode.value | 4)
-        windll.kernel32.SetConsoleMode(c_int(stdout_handle), mode)
-
     parser = argparse.ArgumentParser()
     parser.add_argument("path", nargs="+", help="A path to an XML file or a directory containing XML files to parse.")
     parser.add_argument("--filter", default="", help="The filepath pattern for XML files to filter.")
@@ -711,7 +700,25 @@ def main() -> None:
     )
     args = parser.parse_args()
 
-    should_color = args.color or (hasattr(sys.stdout, "isatty") and sys.stdout.isatty())
+    should_color = bool(args.color or sys.stdout.isatty() or os.environ.get("CI"))
+
+    # Enable ANSI escape code support on Windows 10 and later (for colored console output).
+    # <https://github.com/python/cpython/issues/73245>
+    if should_color and sys.stdout.isatty() and sys.platform == "win32":
+        try:
+            from ctypes import windll, byref, WinError  # type: ignore
+            from ctypes.wintypes import DWORD  # type: ignore
+
+            stdout_handle = windll.kernel32.GetStdHandle(DWORD(-11))
+            mode = DWORD(0)
+            if not windll.kernel32.GetConsoleMode(stdout_handle, byref(mode)):
+                raise WinError()
+            mode = DWORD(mode.value | 4)
+            if not windll.kernel32.SetConsoleMode(stdout_handle, mode):
+                raise WinError()
+        except Exception:
+            should_color = False
+
     STYLES["red"] = "\x1b[91m" if should_color else ""
     STYLES["green"] = "\x1b[92m" if should_color else ""
     STYLES["yellow"] = "\x1b[93m" if should_color else ""
diff --git a/modules/text_server_adv/gdextension_build/SConstruct b/modules/text_server_adv/gdextension_build/SConstruct
index 018984a52f..58e1868e31 100644
--- a/modules/text_server_adv/gdextension_build/SConstruct
+++ b/modules/text_server_adv/gdextension_build/SConstruct
@@ -6,14 +6,21 @@ import time
 
 # Enable ANSI escape code support on Windows 10 and later (for colored console output).
 # <https://github.com/python/cpython/issues/73245>
-if sys.platform == "win32":
-    from ctypes import windll, c_int, byref
-
-    stdout_handle = windll.kernel32.GetStdHandle(c_int(-11))
-    mode = c_int(0)
-    windll.kernel32.GetConsoleMode(c_int(stdout_handle), byref(mode))
-    mode = c_int(mode.value | 4)
-    windll.kernel32.SetConsoleMode(c_int(stdout_handle), mode)
+if sys.stdout.isatty() and sys.platform == "win32":
+    try:
+        from ctypes import windll, byref, WinError  # type: ignore
+        from ctypes.wintypes import DWORD  # type: ignore
+
+        stdout_handle = windll.kernel32.GetStdHandle(DWORD(-11))
+        mode = DWORD(0)
+        if not windll.kernel32.GetConsoleMode(stdout_handle, byref(mode)):
+            raise WinError()
+        mode = DWORD(mode.value | 4)
+        if not windll.kernel32.SetConsoleMode(stdout_handle, mode):
+            raise WinError()
+    except Exception as e:
+        methods._colorize = False
+        methods.print_error(f"Failed to enable ANSI escape code support, disabling color output.\n{e}")
 
 # For the reference:
 # - CCFLAGS are compilation flags shared between C and C++
diff --git a/modules/text_server_fb/gdextension_build/SConstruct b/modules/text_server_fb/gdextension_build/SConstruct
index 07940719eb..9326a53026 100644
--- a/modules/text_server_fb/gdextension_build/SConstruct
+++ b/modules/text_server_fb/gdextension_build/SConstruct
@@ -6,14 +6,21 @@ import time
 
 # Enable ANSI escape code support on Windows 10 and later (for colored console output).
 # <https://github.com/python/cpython/issues/73245>
-if sys.platform == "win32":
-    from ctypes import windll, c_int, byref
-
-    stdout_handle = windll.kernel32.GetStdHandle(c_int(-11))
-    mode = c_int(0)
-    windll.kernel32.GetConsoleMode(c_int(stdout_handle), byref(mode))
-    mode = c_int(mode.value | 4)
-    windll.kernel32.SetConsoleMode(c_int(stdout_handle), mode)
+if sys.stdout.isatty() and sys.platform == "win32":
+    try:
+        from ctypes import windll, byref, WinError  # type: ignore
+        from ctypes.wintypes import DWORD  # type: ignore
+
+        stdout_handle = windll.kernel32.GetStdHandle(DWORD(-11))
+        mode = DWORD(0)
+        if not windll.kernel32.GetConsoleMode(stdout_handle, byref(mode)):
+            raise WinError()
+        mode = DWORD(mode.value | 4)
+        if not windll.kernel32.SetConsoleMode(stdout_handle, mode):
+            raise WinError()
+    except Exception as e:
+        methods._colorize = False
+        methods.print_error(f"Failed to enable ANSI escape code support, disabling color output.\n{e}")
 
 # For the reference:
 # - CCFLAGS are compilation flags shared between C and C++