diff options
Diffstat (limited to 'methods.py')
| -rw-r--r-- | methods.py | 1019 |
1 files changed, 737 insertions, 282 deletions
diff --git a/methods.py b/methods.py index 7a7758e24b..30c7cb0331 100644 --- a/methods.py +++ b/methods.py @@ -5,6 +5,7 @@ import glob import subprocess from collections import OrderedDict from collections.abc import Mapping +from enum import Enum from typing import Iterator from pathlib import Path from os.path import normpath, basename @@ -15,6 +16,10 @@ base_folder_only = os.path.basename(os.path.normpath(base_folder_path)) # Listing all the folders we have converted # for SCU in scu_builders.py _scu_folders = set() +# Colors are disabled in non-TTY environments such as pipes. This means +# that if output is redirected to a file, it won't contain color codes. +# Colors are always enabled on continuous integration. +_colorize = bool(sys.stdout.isatty() or os.environ.get("CI")) def set_scu_folders(scu_folders): @@ -22,13 +27,55 @@ def set_scu_folders(scu_folders): _scu_folders = scu_folders +class ANSI(Enum): + """ + Enum class for adding ansi colorcodes directly into strings. + Automatically converts values to strings representing their + internal value, or an empty string in a non-colorized scope. + """ + + GRAY = "\x1b[0;30m" + RED = "\x1b[0;31m" + GREEN = "\x1b[0;32m" + YELLOW = "\x1b[0;33m" + BLUE = "\x1b[0;34m" + PURPLE = "\x1b[0;35m" + CYAN = "\x1b[0;36m" + WHITE = "\x1b[0;37m" + + BOLD_GRAY = "\x1b[1;90m" + BOLD_RED = "\x1b[1;91m" + BOLD_GREEN = "\x1b[1;92m" + BOLD_YELLOW = "\x1b[1;93m" + BOLD_BLUE = "\x1b[1;94m" + BOLD_PURPLE = "\x1b[1;95m" + BOLD_CYAN = "\x1b[1;96m" + BOLD_WHITE = "\x1b[1;97m" + + RESET = "\x1b[0m" + + def __str__(self): + global _colorize + return self.value if _colorize else "" + + +def print_warning(*values: object) -> None: + """Prints a warning message with formatting.""" + print(f"{ANSI.BOLD_YELLOW}WARNING:{ANSI.YELLOW}", *values, ANSI.RESET, file=sys.stderr) + + +def print_error(*values: object) -> None: + """Prints an error message with formatting.""" + print(f"{ANSI.BOLD_RED}ERROR:{ANSI.RED}", *values, ANSI.RESET, file=sys.stderr) + + def add_source_files_orig(self, sources, files, allow_gen=False): # Convert string to list of absolute paths (including expanding wildcard) if isinstance(files, (str, bytes)): # Keep SCons project-absolute path as they are (no wildcard support) if files.startswith("#"): if "*" in files: - print("ERROR: Wildcards can't be expanded in SCons project-absolute path: '{}'".format(files)) + print_error("Wildcards can't be expanded in SCons project-absolute path: '{}'".format(files)) return files = [files] else: @@ -44,7 +91,7 @@ def add_source_files_orig(self, sources, files, allow_gen=False): for path in files: obj = self.Object(path) if obj in sources: - print('WARNING: Object "{}" already included in environment sources.'.format(obj)) + print_warning('Object "{}" already included in environment sources.'.format(obj)) continue sources.append(obj) @@ -163,7 +210,6 @@ def get_version_info(module_version_string="", silent=False): "status": str(version.status), "build": str(build_name), "module_config": str(version.module_config) + module_version_string, - "year": int(version.year), "website": str(version.website), "docs_branch": str(version.docs), } @@ -180,12 +226,14 @@ def get_version_info(module_version_string="", silent=False): gitfolder = ".git" if os.path.isfile(".git"): - module_folder = open(".git", "r").readline().strip() + with open(".git", "r", encoding="utf-8") as file: + module_folder = file.readline().strip() if module_folder.startswith("gitdir: "): gitfolder = module_folder[8:] if os.path.isfile(os.path.join(gitfolder, "HEAD")): - head = open(os.path.join(gitfolder, "HEAD"), "r", encoding="utf8").readline().strip() + with open(os.path.join(gitfolder, "HEAD"), "r", encoding="utf8") as file: + head = file.readline().strip() if head.startswith("ref: "): ref = head[5:] # If this directory is a Git worktree instead of a root clone. @@ -195,11 +243,12 @@ def get_version_info(module_version_string="", silent=False): head = os.path.join(gitfolder, ref) packedrefs = os.path.join(gitfolder, "packed-refs") if os.path.isfile(head): - githash = open(head, "r").readline().strip() + with open(head, "r", encoding="utf-8") as file: + githash = file.readline().strip() elif os.path.isfile(packedrefs): # Git may pack refs into a single file. This code searches .git/packed-refs file for the current ref's hash. # https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-pack-refs.html - for line in open(packedrefs, "r").read().splitlines(): + for line in open(packedrefs, "r", encoding="utf-8").read().splitlines(): if line.startswith("#"): continue (line_hash, line_ref) = line.split(" ") @@ -210,18 +259,64 @@ def get_version_info(module_version_string="", silent=False): githash = head version_info["git_hash"] = githash + # Fallback to 0 as a timestamp (will be treated as "unknown" in the engine). + version_info["git_timestamp"] = 0 + + # Get the UNIX timestamp of the build commit. + if os.path.exists(".git"): + try: + version_info["git_timestamp"] = subprocess.check_output( + ["git", "log", "-1", "--pretty=format:%ct", githash] + ).decode("utf-8") + except (subprocess.CalledProcessError, OSError): + # `git` not found in PATH. + pass return version_info +_cleanup_env = None +_cleanup_bool = False + + +def write_file_if_needed(path, string): + """Generates a file only if it doesn't already exist or the content has changed. + + Utilizes a dedicated SCons environment to ensure the files are properly removed + during cleanup; will not attempt to create files during cleanup. + + - `path` - Path to the file in question; used to create cleanup logic. + - `string` - Content to compare against an existing file. + """ + global _cleanup_env + global _cleanup_bool + + if _cleanup_env is None: + from SCons.Environment import Environment + + _cleanup_env = Environment() + _cleanup_bool = _cleanup_env.GetOption("clean") + + _cleanup_env.Clean("#", path) + if _cleanup_bool: + return + + try: + with open(path, "r", encoding="utf-8", newline="\n") as f: + if f.read() == string: + return + except FileNotFoundError: + pass + + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write(string) + + def generate_version_header(module_version_string=""): version_info = get_version_info(module_version_string) - # NOTE: It is safe to generate these files here, since this is still executed serially. - - f = open("core/version_generated.gen.h", "w") - f.write( - """/* THIS FILE IS GENERATED DO NOT EDIT */ + version_info_header = """\ +/* THIS FILE IS GENERATED DO NOT EDIT */ #ifndef VERSION_GENERATED_GEN_H #define VERSION_GENERATED_GEN_H #define VERSION_SHORT_NAME "{short_name}" @@ -232,57 +327,53 @@ def generate_version_header(module_version_string=""): #define VERSION_STATUS "{status}" #define VERSION_BUILD "{build}" #define VERSION_MODULE_CONFIG "{module_config}" -#define VERSION_YEAR {year} #define VERSION_WEBSITE "{website}" #define VERSION_DOCS_BRANCH "{docs_branch}" #define VERSION_DOCS_URL "https://docs.godotengine.org/en/" VERSION_DOCS_BRANCH #endif // VERSION_GENERATED_GEN_H """.format( - **version_info - ) + **version_info ) - f.close() - fhash = open("core/version_hash.gen.cpp", "w") - fhash.write( - """/* THIS FILE IS GENERATED DO NOT EDIT */ + version_hash_data = """\ +/* THIS FILE IS GENERATED DO NOT EDIT */ #include "core/version.h" const char *const VERSION_HASH = "{git_hash}"; +const uint64_t VERSION_TIMESTAMP = {git_timestamp}; """.format( - **version_info - ) + **version_info ) - fhash.close() + + write_file_if_needed("core/version_generated.gen.h", version_info_header) + write_file_if_needed("core/version_hash.gen.cpp", version_hash_data) def parse_cg_file(fname, uniforms, sizes, conditionals): - fs = open(fname, "r") - line = fs.readline() + with open(fname, "r", encoding="utf-8") as fs: + line = fs.readline() - while line: - if re.match(r"^\s*uniform", line): - res = re.match(r"uniform ([\d\w]*) ([\d\w]*)") - type = res.groups(1) - name = res.groups(2) + while line: + if re.match(r"^\s*uniform", line): + res = re.match(r"uniform ([\d\w]*) ([\d\w]*)") + type = res.groups(1) + name = res.groups(2) - uniforms.append(name) + uniforms.append(name) - if type.find("texobj") != -1: - sizes.append(1) - else: - t = re.match(r"float(\d)x(\d)", type) - if t: - sizes.append(int(t.groups(1)) * int(t.groups(2))) + if type.find("texobj") != -1: + sizes.append(1) else: - t = re.match(r"float(\d)", type) - sizes.append(int(t.groups(1))) + t = re.match(r"float(\d)x(\d)", type) + if t: + sizes.append(int(t.groups(1)) * int(t.groups(2))) + else: + t = re.match(r"float(\d)", type) + sizes.append(int(t.groups(1))) - if line.find("[branch]") != -1: - conditionals.append(name) + if line.find("[branch]") != -1: + conditionals.append(name) - line = fs.readline() - - fs.close() + line = fs.readline() def get_cmdline_bool(option, default): @@ -373,15 +464,18 @@ def is_module(path): def write_disabled_classes(class_list): - f = open("core/disabled_classes.gen.h", "w") - f.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n") - f.write("#ifndef DISABLED_CLASSES_GEN_H\n") - f.write("#define DISABLED_CLASSES_GEN_H\n\n") + file_contents = "" + + file_contents += "/* THIS FILE IS GENERATED DO NOT EDIT */\n" + file_contents += "#ifndef DISABLED_CLASSES_GEN_H\n" + file_contents += "#define DISABLED_CLASSES_GEN_H\n\n" for c in class_list: cs = c.strip() if cs != "": - f.write("#define ClassDB_Disable_" + cs + " 1\n") - f.write("\n#endif\n") + file_contents += "#define ClassDB_Disable_" + cs + " 1\n" + file_contents += "\n#endif\n" + + write_file_if_needed("core/disabled_classes.gen.h", file_contents) def write_modules(modules): @@ -423,9 +517,7 @@ void uninitialize_modules(ModuleInitializationLevel p_level) { uninitialize_cpp, ) - # NOTE: It is safe to generate this file here, since this is still executed serially - with open("modules/register_module_types.gen.cpp", "w") as f: - f.write(modules_cpp) + write_file_if_needed("modules/register_module_types.gen.cpp", modules_cpp) def convert_custom_modules_path(path): @@ -472,7 +564,7 @@ def module_check_dependencies(self, module): missing_deps.append(dep) if missing_deps != []: - print( + print_warning( "Disabling '{}' module as the following dependencies are not satisfied: {}".format( module, ", ".join(missing_deps) ) @@ -531,9 +623,7 @@ def use_windows_spawn_fix(self, platform=None): _, err = proc.communicate() rv = proc.wait() if rv: - print("=====") - print(err) - print("=====") + print_error(err) return rv def mySpawn(sh, escape, cmd, args, env): @@ -556,57 +646,34 @@ def use_windows_spawn_fix(self, platform=None): self["SPAWN"] = mySpawn -def no_verbose(sys, env): - colors = {} - - # Colors are disabled in non-TTY environments such as pipes. This means - # that if output is redirected to a file, it will not contain color codes - if sys.stdout.isatty(): - colors["blue"] = "\033[0;94m" - colors["bold_blue"] = "\033[1;94m" - colors["reset"] = "\033[0m" - else: - colors["blue"] = "" - colors["bold_blue"] = "" - colors["reset"] = "" +def no_verbose(env): + colors = [ANSI.BLUE, ANSI.BOLD_BLUE, ANSI.RESET] # There is a space before "..." to ensure that source file names can be # Ctrl + clicked in the VS Code terminal. - compile_source_message = "{}Compiling {}$SOURCE{} ...{}".format( - colors["blue"], colors["bold_blue"], colors["blue"], colors["reset"] - ) - java_compile_source_message = "{}Compiling {}$SOURCE{} ...{}".format( - colors["blue"], colors["bold_blue"], colors["blue"], colors["reset"] - ) - compile_shared_source_message = "{}Compiling shared {}$SOURCE{} ...{}".format( - colors["blue"], colors["bold_blue"], colors["blue"], colors["reset"] - ) - link_program_message = "{}Linking Program {}$TARGET{} ...{}".format( - colors["blue"], colors["bold_blue"], colors["blue"], colors["reset"] - ) - link_library_message = "{}Linking Static Library {}$TARGET{} ...{}".format( - colors["blue"], colors["bold_blue"], colors["blue"], colors["reset"] - ) - ranlib_library_message = "{}Ranlib Library {}$TARGET{} ...{}".format( - colors["blue"], colors["bold_blue"], colors["blue"], colors["reset"] - ) - link_shared_library_message = "{}Linking Shared Library {}$TARGET{} ...{}".format( - colors["blue"], colors["bold_blue"], colors["blue"], colors["reset"] - ) - java_library_message = "{}Creating Java Archive {}$TARGET{} ...{}".format( - colors["blue"], colors["bold_blue"], colors["blue"], colors["reset"] - ) - - env.Append(CXXCOMSTR=[compile_source_message]) - env.Append(CCCOMSTR=[compile_source_message]) - env.Append(SHCCCOMSTR=[compile_shared_source_message]) - env.Append(SHCXXCOMSTR=[compile_shared_source_message]) - env.Append(ARCOMSTR=[link_library_message]) - env.Append(RANLIBCOMSTR=[ranlib_library_message]) - env.Append(SHLINKCOMSTR=[link_shared_library_message]) - env.Append(LINKCOMSTR=[link_program_message]) - env.Append(JARCOMSTR=[java_library_message]) - env.Append(JAVACCOMSTR=[java_compile_source_message]) + compile_source_message = "{0}Compiling {1}$SOURCE{0} ...{2}".format(*colors) + java_compile_source_message = "{0}Compiling {1}$SOURCE{0} ...{2}".format(*colors) + compile_shared_source_message = "{0}Compiling shared {1}$SOURCE{0} ...{2}".format(*colors) + link_program_message = "{0}Linking Program {1}$TARGET{0} ...{2}".format(*colors) + link_library_message = "{0}Linking Static Library {1}$TARGET{0} ...{2}".format(*colors) + ranlib_library_message = "{0}Ranlib Library {1}$TARGET{0} ...{2}".format(*colors) + link_shared_library_message = "{0}Linking Shared Library {1}$TARGET{0} ...{2}".format(*colors) + java_library_message = "{0}Creating Java Archive {1}$TARGET{0} ...{2}".format(*colors) + compiled_resource_message = "{0}Creating Compiled Resource {1}$TARGET{0} ...{2}".format(*colors) + generated_file_message = "{0}Generating {1}$TARGET{0} ...{2}".format(*colors) + + env.Append(CXXCOMSTR=compile_source_message) + env.Append(CCCOMSTR=compile_source_message) + env.Append(SHCCCOMSTR=compile_shared_source_message) + env.Append(SHCXXCOMSTR=compile_shared_source_message) + env.Append(ARCOMSTR=link_library_message) + env.Append(RANLIBCOMSTR=ranlib_library_message) + env.Append(SHLINKCOMSTR=link_shared_library_message) + env.Append(LINKCOMSTR=link_program_message) + env.Append(JARCOMSTR=java_library_message) + env.Append(JAVACCOMSTR=java_compile_source_message) + env.Append(RCCOMSTR=compiled_resource_message) + env.Append(GENCOMSTR=generated_file_message) def detect_visual_c_compiler_version(tools_env): @@ -708,25 +775,24 @@ def detect_visual_c_compiler_version(tools_env): def find_visual_c_batch_file(env): - from SCons.Tool.MSCommon.vc import ( - get_default_version, - get_host_target, - find_batch_file, - ) - - # Syntax changed in SCons 4.4.0. - from SCons import __version__ as scons_raw_version - - scons_ver = env._get_major_minor_revision(scons_raw_version) + from SCons.Tool.MSCommon.vc import get_default_version, get_host_target, find_batch_file, find_vc_pdir - version = get_default_version(env) + msvc_version = get_default_version(env) - if scons_ver >= (4, 4, 0): - (host_platform, target_platform, _) = get_host_target(env, version) + # Syntax changed in SCons 4.4.0. + if env.scons_version >= (4, 4, 0): + (host_platform, target_platform, _) = get_host_target(env, msvc_version) else: (host_platform, target_platform, _) = get_host_target(env) - return find_batch_file(env, version, host_platform, target_platform)[0] + if env.scons_version < (4, 6, 0): + return find_batch_file(env, msvc_version, host_platform, target_platform)[0] + + # Scons 4.6.0+ removed passing env, so we need to get the product_dir ourselves first, + # then pass that as the last param instead of env as the first param as before. + # We should investigate if we can avoid relying on SCons internals here. + product_dir = find_vc_pdir(env, msvc_version) + return find_batch_file(msvc_version, host_platform, target_platform, product_dir)[0] def generate_cpp_hint_file(filename): @@ -735,10 +801,10 @@ def generate_cpp_hint_file(filename): pass else: try: - with open(filename, "w") as fd: + with open(filename, "w", encoding="utf-8", newline="\n") as fd: fd.write("#define GDCLASS(m_class, m_inherits)\n") except OSError: - print("Could not write cpp.hint file.") + print_warning("Could not write cpp.hint file.") def glob_recursive(pattern, node="."): @@ -773,154 +839,6 @@ def add_to_vs_project(env, sources): env.vs_srcs += [basename + ".cpp"] -def generate_vs_project(env, original_args, project_name="godot"): - batch_file = find_visual_c_batch_file(env) - filtered_args = original_args.copy() - # Ignore the "vsproj" option to not regenerate the VS project on every build - filtered_args.pop("vsproj", None) - # The "platform" option is ignored because only the Windows platform is currently supported for VS projects - filtered_args.pop("platform", None) - # The "target" option is ignored due to the way how targets configuration is performed for VS projects (there is a separate project configuration for each target) - filtered_args.pop("target", None) - # The "progress" option is ignored as the current compilation progress indication doesn't work in VS - filtered_args.pop("progress", None) - - if batch_file: - - class ModuleConfigs(Mapping): - # This version information (Win32, x64, Debug, Release) seems to be - # required for Visual Studio to understand that it needs to generate an NMAKE - # project. Do not modify without knowing what you are doing. - PLATFORMS = ["Win32", "x64"] - PLATFORM_IDS = ["x86_32", "x86_64"] - CONFIGURATIONS = ["editor", "template_release", "template_debug"] - DEV_SUFFIX = ".dev" if env["dev_build"] else "" - - @staticmethod - def for_every_variant(value): - return [value for _ in range(len(ModuleConfigs.CONFIGURATIONS) * len(ModuleConfigs.PLATFORMS))] - - def __init__(self): - shared_targets_array = [] - self.names = [] - self.arg_dict = { - "variant": [], - "runfile": shared_targets_array, - "buildtarget": shared_targets_array, - "cpppaths": [], - "cppdefines": [], - "cmdargs": [], - } - self.add_mode() # default - - def add_mode( - self, - name: str = "", - includes: str = "", - cli_args: str = "", - defines=None, - ): - if defines is None: - defines = [] - self.names.append(name) - self.arg_dict["variant"] += [ - f'{config}{f"_[{name}]" if name else ""}|{platform}' - for config in ModuleConfigs.CONFIGURATIONS - for platform in ModuleConfigs.PLATFORMS - ] - self.arg_dict["runfile"] += [ - f'bin\\godot.windows.{config}{ModuleConfigs.DEV_SUFFIX}{".double" if env["precision"] == "double" else ""}.{plat_id}{f".{name}" if name else ""}.exe' - for config in ModuleConfigs.CONFIGURATIONS - for plat_id in ModuleConfigs.PLATFORM_IDS - ] - self.arg_dict["cpppaths"] += ModuleConfigs.for_every_variant(env["CPPPATH"] + [includes]) - self.arg_dict["cppdefines"] += ModuleConfigs.for_every_variant(list(env["CPPDEFINES"]) + defines) - self.arg_dict["cmdargs"] += ModuleConfigs.for_every_variant(cli_args) - - def build_commandline(self, commands): - configuration_getter = ( - "$(Configuration" - + "".join([f'.Replace("{name}", "")' for name in self.names[1:]]) - + '.Replace("_[]", "")' - + ")" - ) - - common_build_prefix = [ - 'cmd /V /C set "plat=$(PlatformTarget)"', - '(if "$(PlatformTarget)"=="x64" (set "plat=x86_amd64"))', - 'call "' + batch_file + '" !plat!', - ] - - # Windows allows us to have spaces in paths, so we need - # to double quote off the directory. However, the path ends - # in a backslash, so we need to remove this, lest it escape the - # last double quote off, confusing MSBuild - common_build_postfix = [ - "--directory=\"$(ProjectDir.TrimEnd('\\'))\"", - "platform=windows", - f"target={configuration_getter}", - "progress=no", - ] - - for arg, value in filtered_args.items(): - common_build_postfix.append(f"{arg}={value}") - - result = " ^& ".join(common_build_prefix + [" ".join([commands] + common_build_postfix)]) - return result - - # Mappings interface definitions - - def __iter__(self) -> Iterator[str]: - for x in self.arg_dict: - yield x - - def __len__(self) -> int: - return len(self.names) - - def __getitem__(self, k: str): - return self.arg_dict[k] - - add_to_vs_project(env, env.core_sources) - add_to_vs_project(env, env.drivers_sources) - add_to_vs_project(env, env.main_sources) - add_to_vs_project(env, env.modules_sources) - add_to_vs_project(env, env.scene_sources) - add_to_vs_project(env, env.servers_sources) - if env["tests"]: - add_to_vs_project(env, env.tests_sources) - if env.editor_build: - add_to_vs_project(env, env.editor_sources) - - for header in glob_recursive("**/*.h"): - env.vs_incs.append(str(header)) - - module_configs = ModuleConfigs() - - if env.get("module_mono_enabled"): - mono_defines = [("GD_MONO_HOT_RELOAD",)] if env.editor_build else [] - module_configs.add_mode( - "mono", - cli_args="module_mono_enabled=yes", - defines=mono_defines, - ) - - env["MSVSBUILDCOM"] = module_configs.build_commandline("scons") - env["MSVSREBUILDCOM"] = module_configs.build_commandline("scons vsproj=yes") - env["MSVSCLEANCOM"] = module_configs.build_commandline("scons --clean") - if not env.get("MSVS"): - env["MSVS"]["PROJECTSUFFIX"] = ".vcxproj" - env["MSVS"]["SOLUTIONSUFFIX"] = ".sln" - env.MSVSProject( - target=["#" + project_name + env["MSVSPROJECTSUFFIX"]], - incs=env.vs_incs, - srcs=env.vs_srcs, - auto_build_solution=1, - **module_configs, - ) - else: - print("Could not locate Visual Studio batch file to set up the build environment. Not generating VS project.") - - def precious_program(env, program, sources, **args): program = env.ProgramOriginal(program, sources, **args) env.Precious(program) @@ -951,15 +869,10 @@ def CommandNoCache(env, target, sources, command, **args): return result -def Run(env, function, short_message, subprocess=True): +def Run(env, function): from SCons.Script import Action - from platform_methods import run_in_subprocess - output_print = short_message if not env["verbose"] else "" - if not subprocess: - return Action(function, output_print) - else: - return Action(run_in_subprocess(function), output_print) + return Action(function, "$GENCOMSTR") def detect_darwin_sdk_path(platform, env): @@ -982,7 +895,7 @@ def detect_darwin_sdk_path(platform, env): if sdk_path: env[var_name] = sdk_path except (subprocess.CalledProcessError, OSError): - print("Failed to find SDK path while running xcrun --sdk {} --show-sdk-path.".format(sdk_name)) + print_error("Failed to find SDK path while running xcrun --sdk {} --show-sdk-path.".format(sdk_name)) raise @@ -992,7 +905,7 @@ def is_vanilla_clang(env): try: version = subprocess.check_output([env.subst(env["CXX"]), "--version"]).strip().decode("utf-8") except (subprocess.CalledProcessError, OSError): - print("Couldn't parse CXX environment variable to infer compiler version.") + print_warning("Couldn't parse CXX environment variable to infer compiler version.") return False return not version.startswith("Apple") @@ -1012,15 +925,24 @@ def get_compiler_version(env): "metadata1": None, "metadata2": None, "date": None, + "apple_major": -1, + "apple_minor": -1, + "apple_patch1": -1, + "apple_patch2": -1, + "apple_patch3": -1, } if not env.msvc: # Not using -dumpversion as some GCC distros only return major, and # Clang used to return hardcoded 4.2.1: # https://reviews.llvm.org/D56803 try: - version = subprocess.check_output([env.subst(env["CXX"]), "--version"]).strip().decode("utf-8") + version = ( + subprocess.check_output([env.subst(env["CXX"]), "--version"], shell=(os.name == "nt")) + .strip() + .decode("utf-8") + ) except (subprocess.CalledProcessError, OSError): - print("Couldn't parse CXX environment variable to infer compiler version.") + print_warning("Couldn't parse CXX environment variable to infer compiler version.") return ret else: # TODO: Implement for MSVC @@ -1039,8 +961,32 @@ def get_compiler_version(env): for key, value in match.groupdict().items(): if value is not None: ret[key] = value + + match_apple = re.search( + r"(?:(?<=clang-)|(?<=\) )|(?<=^))" + r"(?P<apple_major>\d+)" + r"(?:\.(?P<apple_minor>\d*))?" + r"(?:\.(?P<apple_patch1>\d*))?" + r"(?:\.(?P<apple_patch2>\d*))?" + r"(?:\.(?P<apple_patch3>\d*))?", + version, + ) + if match_apple is not None: + for key, value in match_apple.groupdict().items(): + if value is not None: + ret[key] = value + # Transform semantic versioning to integers - for key in ["major", "minor", "patch"]: + for key in [ + "major", + "minor", + "patch", + "apple_major", + "apple_minor", + "apple_patch1", + "apple_patch2", + "apple_patch3", + ]: ret[key] = int(ret[key] or -1) return ret @@ -1058,6 +1004,10 @@ def using_emcc(env): def show_progress(env): + if env["ninja"]: + # Has its own progress/tracking tool that clashes with ours + return + import sys from SCons.Script import Progress, Command, AlwaysBuild @@ -1160,7 +1110,7 @@ def show_progress(env): def progress_finish(target, source, env): nonlocal node_count, progressor try: - with open(node_count_fname, "w") as f: + with open(node_count_fname, "w", encoding="utf-8", newline="\n") as f: f.write("%d\n" % node_count) progressor.delete(progressor.file_list()) except Exception: @@ -1190,5 +1140,510 @@ def dump(env): def non_serializable(obj): return "<<non-serializable: %s>>" % (type(obj).__qualname__) - with open(".scons_env.json", "w") as f: + with open(".scons_env.json", "w", encoding="utf-8", newline="\n") as f: dump(env.Dictionary(), f, indent=4, default=non_serializable) + + +# Custom Visual Studio project generation logic that supports any platform that has a msvs.py +# script, so Visual Studio can be used to run scons for any platform, with the right defines per target. +# Invoked with scons vsproj=yes +# +# Only platforms that opt in to vs proj generation by having a msvs.py file in the platform folder are included. +# Platforms with a msvs.py file will be added to the solution, but only the current active platform+target+arch +# will have a build configuration generated, because we only know what the right defines/includes/flags/etc are +# on the active build target. +# +# Platforms that don't support an editor target will have a dummy editor target that won't do anything on build, +# but will have the files and configuration for the windows editor target. +# +# To generate build configuration files for all platforms+targets+arch combinations, users can call +# scons vsproj=yes +# for each combination of platform+target+arch. This will generate the relevant vs project files but +# skip the build process. This lets project files be quickly generated even if there are build errors. +# +# To generate AND build from the command line: +# scons vsproj=yes vsproj_gen_only=yes +def generate_vs_project(env, original_args, project_name="godot"): + # Augmented glob_recursive that also fills the dirs argument with traversed directories that have content. + def glob_recursive_2(pattern, dirs, node="."): + from SCons import Node + from SCons.Script import Glob + + results = [] + for f in Glob(str(node) + "/*", source=True): + if type(f) is Node.FS.Dir: + results += glob_recursive_2(pattern, dirs, f) + r = Glob(str(node) + "/" + pattern, source=True) + if len(r) > 0 and not str(node) in dirs: + d = "" + for part in str(node).split("\\"): + d += part + if not d in dirs: + dirs.append(d) + d += "\\" + results += r + return results + + def get_bool(args, option, default): + from SCons.Variables.BoolVariable import _text2bool + + val = args.get(option, default) + if val is not None: + try: + return _text2bool(val) + except: + return default + else: + return default + + def format_key_value(v): + if type(v) in [tuple, list]: + return v[0] if len(v) == 1 else f"{v[0]}={v[1]}" + return v + + filtered_args = original_args.copy() + + # Ignore the "vsproj" option to not regenerate the VS project on every build + filtered_args.pop("vsproj", None) + + # This flag allows users to regenerate the proj files but skip the building process. + # This lets projects be regenerated even if there are build errors. + filtered_args.pop("vsproj_gen_only", None) + + # This flag allows users to regenerate only the props file without touching the sln or vcxproj files. + # This preserves any customizations users have done to the solution, while still updating the file list + # and build commands. + filtered_args.pop("vsproj_props_only", None) + + # The "progress" option is ignored as the current compilation progress indication doesn't work in VS + filtered_args.pop("progress", None) + + # We add these three manually because they might not be explicitly passed in, and it's important to always set them. + filtered_args.pop("platform", None) + filtered_args.pop("target", None) + filtered_args.pop("arch", None) + + platform = env["platform"] + target = env["target"] + arch = env["arch"] + + vs_configuration = {} + common_build_prefix = [] + confs = [] + for x in sorted(glob.glob("platform/*")): + # Only platforms that opt in to vs proj generation are included. + if not os.path.isdir(x) or not os.path.exists(x + "/msvs.py"): + continue + tmppath = "./" + x + sys.path.insert(0, tmppath) + import msvs + + vs_plats = [] + vs_confs = [] + try: + platform_name = x[9:] + vs_plats = msvs.get_platforms() + vs_confs = msvs.get_configurations() + val = [] + for plat in vs_plats: + val += [{"platform": plat[0], "architecture": plat[1]}] + + vsconf = {"platform": platform_name, "targets": vs_confs, "arches": val} + confs += [vsconf] + + # Save additional information about the configuration for the actively selected platform, + # so we can generate the platform-specific props file with all the build commands/defines/etc + if platform == platform_name: + common_build_prefix = msvs.get_build_prefix(env) + vs_configuration = vsconf + except Exception: + pass + + sys.path.remove(tmppath) + sys.modules.pop("msvs") + + headers = [] + headers_dirs = [] + for file in glob_recursive_2("*.h", headers_dirs): + headers.append(str(file).replace("/", "\\")) + for file in glob_recursive_2("*.hpp", headers_dirs): + headers.append(str(file).replace("/", "\\")) + + sources = [] + sources_dirs = [] + for file in glob_recursive_2("*.cpp", sources_dirs): + sources.append(str(file).replace("/", "\\")) + for file in glob_recursive_2("*.c", sources_dirs): + sources.append(str(file).replace("/", "\\")) + + others = [] + others_dirs = [] + for file in glob_recursive_2("*.natvis", others_dirs): + others.append(str(file).replace("/", "\\")) + for file in glob_recursive_2("*.glsl", others_dirs): + others.append(str(file).replace("/", "\\")) + + skip_filters = False + import hashlib + import json + + md5 = hashlib.md5( + json.dumps(headers + headers_dirs + sources + sources_dirs + others + others_dirs, sort_keys=True).encode( + "utf-8" + ) + ).hexdigest() + + if os.path.exists(f"{project_name}.vcxproj.filters"): + with open(f"{project_name}.vcxproj.filters", "r", encoding="utf-8") as file: + existing_filters = file.read() + match = re.search(r"(?ms)^<!-- CHECKSUM$.([0-9a-f]{32})", existing_filters) + if match is not None and md5 == match.group(1): + skip_filters = True + + import uuid + + # Don't regenerate the filters file if nothing has changed, so we keep the existing UUIDs. + if not skip_filters: + print(f"Regenerating {project_name}.vcxproj.filters") + + with open("misc/msvs/vcxproj.filters.template", "r", encoding="utf-8") as file: + filters_template = file.read() + for i in range(1, 10): + filters_template = filters_template.replace(f"%%UUID{i}%%", str(uuid.uuid4())) + + filters = "" + + for d in headers_dirs: + filters += f'<Filter Include="Header Files\\{d}"><UniqueIdentifier>{{{str(uuid.uuid4())}}}</UniqueIdentifier></Filter>\n' + for d in sources_dirs: + filters += f'<Filter Include="Source Files\\{d}"><UniqueIdentifier>{{{str(uuid.uuid4())}}}</UniqueIdentifier></Filter>\n' + for d in others_dirs: + filters += f'<Filter Include="Other Files\\{d}"><UniqueIdentifier>{{{str(uuid.uuid4())}}}</UniqueIdentifier></Filter>\n' + + filters_template = filters_template.replace("%%FILTERS%%", filters) + + filters = "" + for file in headers: + filters += ( + f'<ClInclude Include="{file}"><Filter>Header Files\\{os.path.dirname(file)}</Filter></ClInclude>\n' + ) + filters_template = filters_template.replace("%%INCLUDES%%", filters) + + filters = "" + for file in sources: + filters += ( + f'<ClCompile Include="{file}"><Filter>Source Files\\{os.path.dirname(file)}</Filter></ClCompile>\n' + ) + + filters_template = filters_template.replace("%%COMPILES%%", filters) + + filters = "" + for file in others: + filters += f'<None Include="{file}"><Filter>Other Files\\{os.path.dirname(file)}</Filter></None>\n' + filters_template = filters_template.replace("%%OTHERS%%", filters) + + filters_template = filters_template.replace("%%HASH%%", md5) + + with open(f"{project_name}.vcxproj.filters", "w", encoding="utf-8", newline="\r\n") as f: + f.write(filters_template) + + envsources = [] + + envsources += env.core_sources + envsources += env.drivers_sources + envsources += env.main_sources + envsources += env.modules_sources + envsources += env.scene_sources + envsources += env.servers_sources + if env.editor_build: + envsources += env.editor_sources + envsources += env.platform_sources + + headers_active = [] + sources_active = [] + others_active = [] + for x in envsources: + fname = "" + if type(x) == type(""): + fname = env.File(x).path + else: + # Some object files might get added directly as a File object and not a list. + try: + fname = env.File(x)[0].path + except: + fname = x.path + pass + + if fname: + fname = fname.replace("\\\\", "/") + parts = os.path.splitext(fname) + basename = parts[0] + ext = parts[1] + idx = fname.find(env["OBJSUFFIX"]) + if ext in [".h", ".hpp"]: + headers_active += [fname] + elif ext in [".c", ".cpp"]: + sources_active += [fname] + elif idx > 0: + basename = fname[:idx] + if os.path.isfile(basename + ".h"): + headers_active += [basename + ".h"] + elif os.path.isfile(basename + ".hpp"): + headers_active += [basename + ".hpp"] + elif basename.endswith(".gen") and os.path.isfile(basename[:-4] + ".h"): + headers_active += [basename[:-4] + ".h"] + if os.path.isfile(basename + ".c"): + sources_active += [basename + ".c"] + elif os.path.isfile(basename + ".cpp"): + sources_active += [basename + ".cpp"] + else: + fname = os.path.relpath(os.path.abspath(fname), env.Dir("").abspath) + others_active += [fname] + + all_items = [] + properties = [] + activeItems = [] + extraItems = [] + + set_headers = set(headers_active) + set_sources = set(sources_active) + set_others = set(others_active) + for file in headers: + base_path = os.path.dirname(file).replace("\\", "_") + all_items.append(f'<ClInclude Include="{file}">') + all_items.append( + f" <ExcludedFromBuild Condition=\"!$(ActiveProjectItemList_{base_path}.Contains(';{file};'))\">true</ExcludedFromBuild>" + ) + all_items.append("</ClInclude>") + if file in set_headers: + activeItems.append(file) + + for file in sources: + base_path = os.path.dirname(file).replace("\\", "_") + all_items.append(f'<ClCompile Include="{file}">') + all_items.append( + f" <ExcludedFromBuild Condition=\"!$(ActiveProjectItemList_{base_path}.Contains(';{file};'))\">true</ExcludedFromBuild>" + ) + all_items.append("</ClCompile>") + if file in set_sources: + activeItems.append(file) + + for file in others: + base_path = os.path.dirname(file).replace("\\", "_") + all_items.append(f'<None Include="{file}">') + all_items.append( + f" <ExcludedFromBuild Condition=\"!$(ActiveProjectItemList_{base_path}.Contains(';{file};'))\">true</ExcludedFromBuild>" + ) + all_items.append("</None>") + if file in set_others: + activeItems.append(file) + + if vs_configuration: + vsconf = "" + for a in vs_configuration["arches"]: + if arch == a["architecture"]: + vsconf = f'{target}|{a["platform"]}' + break + + condition = "'$(GodotConfiguration)|$(GodotPlatform)'=='" + vsconf + "'" + itemlist = {} + for item in activeItems: + key = os.path.dirname(item).replace("\\", "_") + if not key in itemlist: + itemlist[key] = [item] + else: + itemlist[key] += [item] + + for x in itemlist.keys(): + properties.append( + "<ActiveProjectItemList_%s>;%s;</ActiveProjectItemList_%s>" % (x, ";".join(itemlist[x]), x) + ) + output = f'bin\\godot{env["PROGSUFFIX"]}' + + with open("misc/msvs/props.template", "r", encoding="utf-8") as file: + props_template = file.read() + + props_template = props_template.replace("%%VSCONF%%", vsconf) + props_template = props_template.replace("%%CONDITION%%", condition) + props_template = props_template.replace("%%PROPERTIES%%", "\n ".join(properties)) + props_template = props_template.replace("%%EXTRA_ITEMS%%", "\n ".join(extraItems)) + + props_template = props_template.replace("%%OUTPUT%%", output) + + proplist = [format_key_value(v) for v in list(env["CPPDEFINES"])] + proplist += [format_key_value(j) for j in env.get("VSHINT_DEFINES", [])] + props_template = props_template.replace("%%DEFINES%%", ";".join(proplist)) + + proplist = [str(j) for j in env["CPPPATH"]] + proplist += [str(j) for j in env.get("VSHINT_INCLUDES", [])] + props_template = props_template.replace("%%INCLUDES%%", ";".join(proplist)) + + proplist = env["CCFLAGS"] + proplist += [x for x in env["CXXFLAGS"] if not x.startswith("$")] + proplist += [str(j) for j in env.get("VSHINT_OPTIONS", [])] + props_template = props_template.replace("%%OPTIONS%%", " ".join(proplist)) + + # Windows allows us to have spaces in paths, so we need + # to double quote off the directory. However, the path ends + # in a backslash, so we need to remove this, lest it escape the + # last double quote off, confusing MSBuild + common_build_postfix = [ + "--directory="$(ProjectDir.TrimEnd('\\'))"", + "progress=no", + f"platform={platform}", + f"target={target}", + f"arch={arch}", + ] + + for arg, value in filtered_args.items(): + common_build_postfix.append(f"{arg}={value}") + + cmd_rebuild = [ + "vsproj=yes", + "vsproj_props_only=yes", + "vsproj_gen_only=no", + f"vsproj_name={project_name}", + ] + common_build_postfix + + cmd_clean = [ + "--clean", + ] + common_build_postfix + + commands = "scons" + if len(common_build_prefix) == 0: + commands = "echo Starting SCons && cmd /V /C " + commands + else: + common_build_prefix[0] = "echo Starting SCons && cmd /V /C " + common_build_prefix[0] + + cmd = " ^& ".join(common_build_prefix + [" ".join([commands] + common_build_postfix)]) + props_template = props_template.replace("%%BUILD%%", cmd) + + cmd = " ^& ".join(common_build_prefix + [" ".join([commands] + cmd_rebuild)]) + props_template = props_template.replace("%%REBUILD%%", cmd) + + cmd = " ^& ".join(common_build_prefix + [" ".join([commands] + cmd_clean)]) + props_template = props_template.replace("%%CLEAN%%", cmd) + + with open( + f"{project_name}.{platform}.{target}.{arch}.generated.props", "w", encoding="utf-8", newline="\r\n" + ) as f: + f.write(props_template) + + proj_uuid = str(uuid.uuid4()) + sln_uuid = str(uuid.uuid4()) + + if os.path.exists(f"{project_name}.sln"): + for line in open(f"{project_name}.sln", "r", encoding="utf-8").read().splitlines(): + if line.startswith('Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}")'): + proj_uuid = re.search( + r"\"{(\b[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-\b[0-9a-fA-F]{12}\b)}\"$", + line, + ).group(1) + elif line.strip().startswith("SolutionGuid ="): + sln_uuid = re.search( + r"{(\b[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-\b[0-9a-fA-F]{12}\b)}", line + ).group(1) + break + + configurations = [] + imports = [] + properties = [] + section1 = [] + section2 = [] + for conf in confs: + godot_platform = conf["platform"] + for p in conf["arches"]: + sln_plat = p["platform"] + proj_plat = sln_plat + godot_arch = p["architecture"] + + # Redirect editor configurations for non-Windows platforms to the Windows one, so the solution has all the permutations + # and VS doesn't complain about missing project configurations. + # These configurations are disabled, so they show up but won't build. + if godot_platform != "windows": + section1 += [f"editor|{sln_plat} = editor|{proj_plat}"] + section2 += [ + f"{{{proj_uuid}}}.editor|{proj_plat}.ActiveCfg = editor|{proj_plat}", + ] + + for t in conf["targets"]: + godot_target = t + + # Windows x86 is a special little flower that requires a project platform == Win32 but a solution platform == x86. + if godot_platform == "windows" and godot_target == "editor" and godot_arch == "x86_32": + sln_plat = "x86" + + configurations += [ + f'<ProjectConfiguration Include="{godot_target}|{proj_plat}">', + f" <Configuration>{godot_target}</Configuration>", + f" <Platform>{proj_plat}</Platform>", + "</ProjectConfiguration>", + ] + + properties += [ + f"<PropertyGroup Condition=\"'$(Configuration)|$(Platform)'=='{godot_target}|{proj_plat}'\">", + f" <GodotConfiguration>{godot_target}</GodotConfiguration>", + f" <GodotPlatform>{proj_plat}</GodotPlatform>", + "</PropertyGroup>", + ] + + if godot_platform != "windows": + configurations += [ + f'<ProjectConfiguration Include="editor|{proj_plat}">', + f" <Configuration>editor</Configuration>", + f" <Platform>{proj_plat}</Platform>", + "</ProjectConfiguration>", + ] + + properties += [ + f"<PropertyGroup Condition=\"'$(Configuration)|$(Platform)'=='editor|{proj_plat}'\">", + f" <GodotConfiguration>editor</GodotConfiguration>", + f" <GodotPlatform>{proj_plat}</GodotPlatform>", + "</PropertyGroup>", + ] + + p = f"{project_name}.{godot_platform}.{godot_target}.{godot_arch}.generated.props" + imports += [ + f'<Import Project="$(MSBuildProjectDirectory)\\{p}" Condition="Exists(\'$(MSBuildProjectDirectory)\\{p}\')"/>' + ] + + section1 += [f"{godot_target}|{sln_plat} = {godot_target}|{sln_plat}"] + + section2 += [ + f"{{{proj_uuid}}}.{godot_target}|{sln_plat}.ActiveCfg = {godot_target}|{proj_plat}", + f"{{{proj_uuid}}}.{godot_target}|{sln_plat}.Build.0 = {godot_target}|{proj_plat}", + ] + + # Add an extra import for a local user props file at the end, so users can add more overrides. + imports += [ + f'<Import Project="$(MSBuildProjectDirectory)\\{project_name}.vs.user.props" Condition="Exists(\'$(MSBuildProjectDirectory)\\{project_name}.vs.user.props\')"/>' + ] + section1 = sorted(section1) + section2 = sorted(section2) + + if not get_bool(original_args, "vsproj_props_only", False): + with open("misc/msvs/vcxproj.template", "r", encoding="utf-8") as file: + proj_template = file.read() + proj_template = proj_template.replace("%%UUID%%", proj_uuid) + proj_template = proj_template.replace("%%CONFS%%", "\n ".join(configurations)) + proj_template = proj_template.replace("%%IMPORTS%%", "\n ".join(imports)) + proj_template = proj_template.replace("%%DEFAULT_ITEMS%%", "\n ".join(all_items)) + proj_template = proj_template.replace("%%PROPERTIES%%", "\n ".join(properties)) + + with open(f"{project_name}.vcxproj", "w", encoding="utf-8", newline="\n") as f: + f.write(proj_template) + + if not get_bool(original_args, "vsproj_props_only", False): + with open("misc/msvs/sln.template", "r", encoding="utf-8") as file: + sln_template = file.read() + sln_template = sln_template.replace("%%NAME%%", project_name) + sln_template = sln_template.replace("%%UUID%%", proj_uuid) + sln_template = sln_template.replace("%%SLNUUID%%", sln_uuid) + sln_template = sln_template.replace("%%SECTION1%%", "\n\t\t".join(section1)) + sln_template = sln_template.replace("%%SECTION2%%", "\n\t\t".join(section2)) + + with open(f"{project_name}.sln", "w", encoding="utf-8", newline="\r\n") as f: + f.write(sln_template) + + if get_bool(original_args, "vsproj_gen_only", True): + sys.exit() |