diff options
Diffstat (limited to 'methods.py')
| -rw-r--r-- | methods.py | 563 |
1 files changed, 216 insertions, 347 deletions
diff --git a/methods.py b/methods.py index 9e881773c9..203f0dd8a5 100644 --- a/methods.py +++ b/methods.py @@ -1,5 +1,7 @@ +import atexit import contextlib import glob +import math import os import re import subprocess @@ -8,7 +10,7 @@ from collections import OrderedDict from enum import Enum from io import StringIO, TextIOWrapper from pathlib import Path -from typing import Generator, List, Optional, Union +from typing import Generator, List, Optional, Union, cast # Get the "Godot" folder name ahead of time base_folder_path = str(os.path.abspath(Path(__file__).parent)) + "/" @@ -73,21 +75,13 @@ def print_error(*values: object) -> None: def add_source_files_orig(self, sources, files, allow_gen=False): # Convert string to list of absolute paths (including expanding wildcard) - if isinstance(files, (str, bytes)): - # Keep SCons project-absolute path as they are (no wildcard support) - if files.startswith("#"): - if "*" in files: - print_error("Wildcards can't be expanded in SCons project-absolute path: '{}'".format(files)) - return - files = [files] - else: - # Exclude .gen.cpp files from globbing, to avoid including obsolete ones. - # They should instead be added manually. - skip_gen_cpp = "*" in files - dir_path = self.Dir(".").abspath - files = sorted(glob.glob(dir_path + "/" + files)) - if skip_gen_cpp and not allow_gen: - files = [f for f in files if not f.endswith(".gen.cpp")] + if isinstance(files, str): + # Exclude .gen.cpp files from globbing, to avoid including obsolete ones. + # They should instead be added manually. + skip_gen_cpp = "*" in files + files = self.Glob(files) + if skip_gen_cpp and not allow_gen: + files = [f for f in files if not str(f).endswith(".gen.cpp")] # Add each path as compiled Object following environment (self) configuration for path in files: @@ -98,35 +92,6 @@ def add_source_files_orig(self, sources, files, allow_gen=False): sources.append(obj) -# The section name is used for checking -# the hash table to see whether the folder -# is included in the SCU build. -# It will be something like "core/math". -def _find_scu_section_name(subdir): - section_path = os.path.abspath(subdir) + "/" - - folders = [] - folder = "" - - for i in range(8): - folder = os.path.dirname(section_path) - folder = os.path.basename(folder) - if folder == base_folder_only: - break - folders += [folder] - section_path += "../" - section_path = os.path.abspath(section_path) + "/" - - section_name = "" - for n in range(len(folders)): - # section_name += folders[len(folders) - n - 1] + " " - section_name += folders[len(folders) - n - 1] - if n != (len(folders) - 1): - section_name += "/" - - return section_name - - def add_source_files_scu(self, sources, files, allow_gen=False): if self["scu_build"] and isinstance(files, str): if "*." not in files: @@ -135,10 +100,8 @@ def add_source_files_scu(self, sources, files, allow_gen=False): # If the files are in a subdirectory, we want to create the scu gen # files inside this subdirectory. subdir = os.path.dirname(files) - if subdir != "": - subdir += "/" - - section_name = _find_scu_section_name(subdir) + subdir = subdir if subdir == "" else subdir + "/" + section_name = self.Dir(subdir).tpath # if the section name is in the hash table? # i.e. is it part of the SCU build? global _scu_folders @@ -277,34 +240,6 @@ def get_version_info(module_version_string="", silent=False): return version_info -def parse_cg_file(fname, uniforms, sizes, conditionals): - with open(fname, "r", encoding="utf-8") as fs: - line = fs.readline() - - while line: - if re.match(r"^\s*uniform", line): - res = re.match(r"uniform ([\d\w]*) ([\d\w]*)") - type = res.groups(1) - name = res.groups(2) - - uniforms.append(name) - - if type.find("texobj") != -1: - sizes.append(1) - else: - t = re.match(r"float(\d)x(\d)", type) - if t: - sizes.append(int(t.groups(1)) * int(t.groups(2))) - else: - t = re.match(r"float(\d)", type) - sizes.append(int(t.groups(1))) - - if line.find("[branch]") != -1: - conditionals.append(name) - - line = fs.readline() - - def get_cmdline_bool(option, default): """We use `ARGUMENTS.get()` to check if options were manually overridden on the command line, and SCons' _text2bool helper to convert them to booleans, otherwise they're handled as strings. @@ -404,10 +339,6 @@ def convert_custom_modules_path(path): return path -def disable_module(self): - self.disabled_modules.append(self.current_module) - - def module_add_dependencies(self, module, dependencies, optional=False): """ Adds dependencies for a given module. @@ -428,19 +359,21 @@ def module_check_dependencies(self, module): Meant to be used in module `can_build` methods. Returns a boolean (True if dependencies are satisfied). """ - missing_deps = [] + missing_deps = set() required_deps = self.module_dependencies[module][0] if module in self.module_dependencies else [] for dep in required_deps: opt = "module_{}_enabled".format(dep) - if opt not in self or not self[opt]: - missing_deps.append(dep) - - if missing_deps != []: - print_warning( - "Disabling '{}' module as the following dependencies are not satisfied: {}".format( - module, ", ".join(missing_deps) + if opt not in self or not self[opt] or not module_check_dependencies(self, dep): + missing_deps.add(dep) + + if missing_deps: + if module not in self.disabled_modules: + print_warning( + "Disabling '{}' module as the following dependencies are not satisfied: {}".format( + module, ", ".join(missing_deps) + ) ) - ) + self.disabled_modules.add(module) return False else: return True @@ -478,8 +411,7 @@ def use_windows_spawn_fix(self, platform=None): "shell": False, "env": env, } - if sys.version_info >= (3, 7, 0): - popen_args["text"] = True + popen_args["text"] = True proc = subprocess.Popen(cmdline, **popen_args) _, err = proc.communicate() rv = proc.wait() @@ -565,40 +497,7 @@ def detect_visual_c_compiler_version(tools_env): vc_chosen_compiler_index = -1 vc_chosen_compiler_str = "" - # Start with Pre VS 2017 checks which uses VCINSTALLDIR: - if "VCINSTALLDIR" in tools_env: - # print("Checking VCINSTALLDIR") - - # find() works with -1 so big ifs below are needed... the simplest solution, in fact - # First test if amd64 and amd64_x86 compilers are present in the path - vc_amd64_compiler_detection_index = tools_env["PATH"].find(tools_env["VCINSTALLDIR"] + "BIN\\amd64;") - if vc_amd64_compiler_detection_index > -1: - vc_chosen_compiler_index = vc_amd64_compiler_detection_index - vc_chosen_compiler_str = "amd64" - - vc_amd64_x86_compiler_detection_index = tools_env["PATH"].find(tools_env["VCINSTALLDIR"] + "BIN\\amd64_x86;") - if vc_amd64_x86_compiler_detection_index > -1 and ( - vc_chosen_compiler_index == -1 or vc_chosen_compiler_index > vc_amd64_x86_compiler_detection_index - ): - vc_chosen_compiler_index = vc_amd64_x86_compiler_detection_index - vc_chosen_compiler_str = "amd64_x86" - - # Now check the 32 bit compilers - vc_x86_compiler_detection_index = tools_env["PATH"].find(tools_env["VCINSTALLDIR"] + "BIN;") - if vc_x86_compiler_detection_index > -1 and ( - vc_chosen_compiler_index == -1 or vc_chosen_compiler_index > vc_x86_compiler_detection_index - ): - vc_chosen_compiler_index = vc_x86_compiler_detection_index - vc_chosen_compiler_str = "x86" - - vc_x86_amd64_compiler_detection_index = tools_env["PATH"].find(tools_env["VCINSTALLDIR"] + "BIN\\x86_amd64;") - if vc_x86_amd64_compiler_detection_index > -1 and ( - vc_chosen_compiler_index == -1 or vc_chosen_compiler_index > vc_x86_amd64_compiler_detection_index - ): - vc_chosen_compiler_index = vc_x86_amd64_compiler_detection_index - vc_chosen_compiler_str = "x86_amd64" - - # and for VS 2017 and newer we check VCTOOLSINSTALLDIR: + # VS 2017 and newer should set VCTOOLSINSTALLDIR if "VCTOOLSINSTALLDIR" in tools_env: # Newer versions have a different path available vc_amd64_compiler_detection_index = ( @@ -695,23 +594,6 @@ def glob_recursive(pattern, node="."): return results -def add_to_vs_project(env, sources): - for x in sources: - fname = env.File(x).path if isinstance(x, str) else env.File(x)[0].path - pieces = fname.split(".") - if len(pieces) > 0: - basename = pieces[0] - basename = basename.replace("\\\\", "/") - if os.path.isfile(basename + ".h"): - env.vs_incs += [basename + ".h"] - elif os.path.isfile(basename + ".hpp"): - env.vs_incs += [basename + ".hpp"] - if os.path.isfile(basename + ".c"): - env.vs_srcs += [basename + ".c"] - elif os.path.isfile(basename + ".cpp"): - env.vs_srcs += [basename + ".cpp"] - - def precious_program(env, program, sources, **args): program = env.ProgramOriginal(program, sources, **args) env.Precious(program) @@ -772,7 +654,9 @@ def detect_darwin_sdk_path(platform, env): raise -def is_vanilla_clang(env): +def is_apple_clang(env): + if env["platform"] not in ["macos", "ios"]: + return False if not using_clang(env): return False try: @@ -780,7 +664,7 @@ def is_vanilla_clang(env): except (subprocess.CalledProcessError, OSError): print_warning("Couldn't parse CXX environment variable to infer compiler version.") return False - return not version.startswith("Apple") + return version.startswith("Apple") def get_compiler_version(env): @@ -904,159 +788,165 @@ def using_emcc(env): def show_progress(env): - if env["ninja"]: - # Has its own progress/tracking tool that clashes with ours + # Progress reporting is not available in non-TTY environments since it messes with the output + # (for example, when writing to a file). Ninja has its own progress/tracking tool that clashes + # with ours. + if not env["progress"] or not sys.stdout.isatty() or env["ninja"]: return - import sys - - from SCons.Script import AlwaysBuild, Command, Progress - - screen = sys.stdout - # Progress reporting is not available in non-TTY environments since it - # messes with the output (for example, when writing to a file) - show_progress = env["progress"] and sys.stdout.isatty() - node_count = 0 - node_count_max = 0 - node_count_interval = 1 - node_count_fname = str(env.Dir("#")) + "/.scons_node_count" - - import math - - class cache_progress: - # The default is 1 GB cache - def __init__(self, path=None, limit=pow(1024, 3)): - self.path = path - self.limit = limit - if env["verbose"] and path is not None: - screen.write( - "Current cache limit is {} (used: {})\n".format( - self.convert_size(limit), self.convert_size(self.get_size(path)) - ) - ) + NODE_COUNT_FILENAME = f"{base_folder_path}.scons_node_count" + + class ShowProgress: + def __init__(self): + self.count = 0 + self.max = 0 + try: + with open(NODE_COUNT_FILENAME, "r", encoding="utf-8") as f: + self.max = int(f.readline()) + except OSError: + pass + if self.max == 0: + print("NOTE: Performing initial build, progress percentage unavailable!") def __call__(self, node, *args, **kw): - nonlocal node_count, node_count_max, node_count_interval, node_count_fname, show_progress - if show_progress: - # Print the progress percentage - node_count += node_count_interval - if node_count_max > 0 and node_count <= node_count_max: - screen.write("\r[%3d%%] " % (node_count * 100 / node_count_max)) - screen.flush() - elif node_count_max > 0 and node_count > node_count_max: - screen.write("\r[100%] ") - screen.flush() - else: - screen.write("\r[Initial build] ") - screen.flush() - - def convert_size(self, size_bytes): - if size_bytes == 0: - return "0 bytes" - size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") - i = int(math.floor(math.log(size_bytes, 1024))) - p = math.pow(1024, i) - s = round(size_bytes / p, 2) - return "%s %s" % (int(s) if i == 0 else s, size_name[i]) - - def get_size(self, start_path="."): - total_size = 0 - for dirpath, dirnames, filenames in os.walk(start_path): - for f in filenames: - fp = os.path.join(dirpath, f) - total_size += os.path.getsize(fp) - return total_size + self.count += 1 + if self.max != 0: + percent = int(min(self.count * 100 / self.max, 100)) + sys.stdout.write(f"\r[{percent:3d}%] ") + sys.stdout.flush() + + from SCons.Script import Progress + + progressor = ShowProgress() + Progress(progressor) def progress_finish(target, source, env): - nonlocal node_count, progressor try: - with open(node_count_fname, "w", encoding="utf-8", newline="\n") as f: - f.write("%d\n" % node_count) - except Exception: + with open(NODE_COUNT_FILENAME, "w", encoding="utf-8", newline="\n") as f: + f.write(f"{progressor.count}\n") + except OSError: pass - try: - with open(node_count_fname, "r", encoding="utf-8") as f: - node_count_max = int(f.readline()) - except Exception: - pass + env.AlwaysBuild( + env.CommandNoCache( + "progress_finish", [], env.Action(progress_finish, "Building node count database .scons_node_count") + ) + ) + + +def convert_size(size_bytes: int) -> str: + if size_bytes == 0: + return "0 bytes" + SIZE_NAMES = ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"] + index = math.floor(math.log(size_bytes, 1024)) + power = math.pow(1024, index) + size = round(size_bytes / power, 2) + return f"{size} {SIZE_NAMES[index]}" - cache_directory = os.environ.get("SCONS_CACHE") - # Simple cache pruning, attached to SCons' progress callback. Trim the - # cache directory to a size not larger than cache_limit. - cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024 - progressor = cache_progress(cache_directory, cache_limit) - Progress(progressor, interval=node_count_interval) - - progress_finish_command = Command("progress_finish", [], progress_finish) - AlwaysBuild(progress_finish_command) - - -def clean_cache(env): - import atexit - import time - - class cache_clean: - def __init__(self, path=None, limit=pow(1024, 3)): - self.path = path - self.limit = limit - - def clean(self): - self.delete(self.file_list()) - - def delete(self, files): - if len(files) == 0: - return - if env["verbose"]: - # Utter something - print("Purging %d %s from cache..." % (len(files), "files" if len(files) > 1 else "file")) - [os.remove(f) for f in files] - - def file_list(self): - if self.path is None: - # Nothing to do - return [] - # Gather a list of (filename, (size, atime)) within the - # cache directory - file_stat = [(x, os.stat(x)[6:8]) for x in glob.glob(os.path.join(self.path, "*", "*"))] - if file_stat == []: - # Nothing to do - return [] - # Weight the cache files by size (assumed to be roughly - # proportional to the recompilation time) times an exponential - # decay since the ctime, and return a list with the entries - # (filename, size, weight). - current_time = time.time() - file_stat = [(x[0], x[1][0], (current_time - x[1][1])) for x in file_stat] - # Sort by the most recently accessed files (most sensible to keep) first - file_stat.sort(key=lambda x: x[2]) - # Search for the first entry where the storage limit is - # reached - sum, mark = 0, None - for i, x in enumerate(file_stat): - sum += x[1] - if sum > self.limit: - mark = i - break - if mark is None: - return [] - else: - return [x[0] for x in file_stat[mark:]] - def cache_finally(): - nonlocal cleaner +def get_size(start_path: str = ".") -> int: + total_size = 0 + for dirpath, _, filenames in os.walk(start_path): + for file in filenames: + path = os.path.join(dirpath, file) + total_size += os.path.getsize(path) + return total_size + + +def clean_cache(cache_path: str, cache_limit: int, verbose: bool): + files = glob.glob(os.path.join(cache_path, "*", "*")) + if not files: + return + + # Remove all text files, store binary files in list of (filename, size, atime). + purge = [] + texts = [] + stats = [] + for file in files: try: - cleaner.clean() - except Exception: - pass + # Save file stats to rewrite after modifying. + tmp_stat = os.stat(file) + # Failing a utf-8 decode is the easiest way to determine if a file is binary. + try: + with open(file, encoding="utf-8") as out: + out.read(1024) + except UnicodeDecodeError: + stats.append((file, *tmp_stat[6:8])) + # Restore file stats after reading. + os.utime(file, (tmp_stat[7], tmp_stat[8])) + else: + texts.append(file) + except OSError: + print_error(f'Failed to access cache file "{file}"; skipping.') + + if texts: + count = len(texts) + for file in texts: + try: + os.remove(file) + except OSError: + print_error(f'Failed to remove cache file "{file}"; skipping.') + count -= 1 + if verbose: + print("Purging %d text %s from cache..." % (count, "files" if count > 1 else "file")) + + if cache_limit: + # Sort by most recent access (most sensible to keep) first. Search for the first entry where + # the cache limit is reached. + stats.sort(key=lambda x: x[2], reverse=True) + sum = 0 + for index, stat in enumerate(stats): + sum += stat[1] + if sum > cache_limit: + purge.extend([x[0] for x in stats[index:]]) + break - cache_directory = os.environ.get("SCONS_CACHE") - # Simple cache pruning, attached to SCons' progress callback. Trim the - # cache directory to a size not larger than cache_limit. - cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024 - cleaner = cache_clean(cache_directory, cache_limit) + if purge: + count = len(purge) + for file in purge: + try: + os.remove(file) + except OSError: + print_error(f'Failed to remove cache file "{file}"; skipping.') + count -= 1 + if verbose: + print("Purging %d %s from cache..." % (count, "files" if count > 1 else "file")) + + +def prepare_cache(env) -> None: + if env.GetOption("clean"): + return + + cache_path = "" + if env["cache_path"]: + cache_path = cast(str, env["cache_path"]) + elif os.environ.get("SCONS_CACHE"): + print_warning("Environment variable `SCONS_CACHE` is deprecated; use `cache_path` argument instead.") + cache_path = cast(str, os.environ.get("SCONS_CACHE")) + + if not cache_path: + return + + env.CacheDir(cache_path) + print(f'SCons cache enabled... (path: "{cache_path}")') + + if env["cache_limit"]: + cache_limit = float(env["cache_limit"]) + elif os.environ.get("SCONS_CACHE_LIMIT"): + print_warning("Environment variable `SCONS_CACHE_LIMIT` is deprecated; use `cache_limit` argument instead.") + cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", "0")) / 1024 # Old method used MiB, convert to GiB + + # Convert GiB to bytes; treat negative numbers as 0 (unlimited). + cache_limit = max(0, int(cache_limit * 1024 * 1024 * 1024)) + if env["verbose"]: + print( + "Current cache limit is {} (used: {})".format( + convert_size(cache_limit) if cache_limit else "∞", + convert_size(get_size(cache_path)), + ) + ) - atexit.register(cache_finally) + atexit.register(clean_cache, cache_path, cache_limit, env["verbose"]) def dump(env): @@ -1127,6 +1017,30 @@ def generate_vs_project(env, original_args, project_name="godot"): return v[0] if len(v) == 1 else f"{v[0]}={v[1]}" return v + def get_dependencies(file, env, exts, headers, sources, others): + for child in file.children(): + if isinstance(child, str): + child = env.File(x) + fname = "" + try: + fname = child.path + except AttributeError: + # It's not a file. + pass + + if fname: + parts = os.path.splitext(fname) + if len(parts) > 1: + ext = parts[1].lower() + if ext in exts["sources"]: + sources += [fname] + elif ext in exts["headers"]: + headers += [fname] + elif ext in exts["others"]: + others += [fname] + + get_dependencies(child, env, exts, headers, sources, others) + filtered_args = original_args.copy() # Ignore the "vsproj" option to not regenerate the VS project on every build @@ -1188,35 +1102,35 @@ def generate_vs_project(env, original_args, project_name="godot"): sys.path.remove(tmppath) sys.modules.pop("msvs") + extensions = {} + extensions["headers"] = [".h", ".hh", ".hpp", ".hxx", ".inc"] + extensions["sources"] = [".c", ".cc", ".cpp", ".cxx", ".m", ".mm", ".java"] + extensions["others"] = [".natvis", ".glsl", ".rc"] + headers = [] headers_dirs = [] - for file in glob_recursive_2("*.h", headers_dirs): - headers.append(str(file).replace("/", "\\")) - for file in glob_recursive_2("*.hpp", headers_dirs): - headers.append(str(file).replace("/", "\\")) + for ext in extensions["headers"]: + for file in glob_recursive_2("*" + ext, headers_dirs): + headers.append(str(file).replace("/", "\\")) sources = [] sources_dirs = [] - for file in glob_recursive_2("*.cpp", sources_dirs): - sources.append(str(file).replace("/", "\\")) - for file in glob_recursive_2("*.c", sources_dirs): - sources.append(str(file).replace("/", "\\")) + for ext in extensions["sources"]: + for file in glob_recursive_2("*" + ext, sources_dirs): + sources.append(str(file).replace("/", "\\")) others = [] others_dirs = [] - for file in glob_recursive_2("*.natvis", others_dirs): - others.append(str(file).replace("/", "\\")) - for file in glob_recursive_2("*.glsl", others_dirs): - others.append(str(file).replace("/", "\\")) + for ext in extensions["others"]: + for file in glob_recursive_2("*" + ext, others_dirs): + others.append(str(file).replace("/", "\\")) skip_filters = False import hashlib import json md5 = hashlib.md5( - json.dumps(headers + headers_dirs + sources + sources_dirs + others + others_dirs, sort_keys=True).encode( - "utf-8" - ) + json.dumps(sorted(headers + headers_dirs + sources + sources_dirs + others + others_dirs)).encode("utf-8") ).hexdigest() if os.path.exists(f"{project_name}.vcxproj.filters"): @@ -1273,58 +1187,13 @@ def generate_vs_project(env, original_args, project_name="godot"): with open(f"{project_name}.vcxproj.filters", "w", encoding="utf-8", newline="\r\n") as f: f.write(filters_template) - envsources = [] - - envsources += env.core_sources - envsources += env.drivers_sources - envsources += env.main_sources - envsources += env.modules_sources - envsources += env.scene_sources - envsources += env.servers_sources - if env.editor_build: - envsources += env.editor_sources - envsources += env.platform_sources - headers_active = [] sources_active = [] others_active = [] - for x in envsources: - fname = "" - if isinstance(x, str): - fname = env.File(x).path - else: - # Some object files might get added directly as a File object and not a list. - try: - fname = env.File(x)[0].path - except Exception: - fname = x.path - pass - if fname: - fname = fname.replace("\\\\", "/") - parts = os.path.splitext(fname) - basename = parts[0] - ext = parts[1] - idx = fname.find(env["OBJSUFFIX"]) - if ext in [".h", ".hpp"]: - headers_active += [fname] - elif ext in [".c", ".cpp"]: - sources_active += [fname] - elif idx > 0: - basename = fname[:idx] - if os.path.isfile(basename + ".h"): - headers_active += [basename + ".h"] - elif os.path.isfile(basename + ".hpp"): - headers_active += [basename + ".hpp"] - elif basename.endswith(".gen") and os.path.isfile(basename[:-4] + ".h"): - headers_active += [basename[:-4] + ".h"] - if os.path.isfile(basename + ".c"): - sources_active += [basename + ".c"] - elif os.path.isfile(basename + ".cpp"): - sources_active += [basename + ".cpp"] - else: - fname = os.path.relpath(os.path.abspath(fname), env.Dir("").abspath) - others_active += [fname] + get_dependencies( + env.File(f"#bin/godot{env['PROGSUFFIX']}"), env, extensions, headers_active, sources_active, others_active + ) all_items = [] properties = [] |