From b4d0a09f15c60c88bbf516d2f6dcdb451dcad9c7 Mon Sep 17 00:00:00 2001 From: George Marques Date: Mon, 22 Jan 2024 11:31:55 -0300 Subject: GDScript: Reintroduce binary tokenization on export This adds back a function available in 3.x: exporting the GDScript files in a binary form by converting the tokens recognized by the tokenizer into a data format. It is enabled by default on export but can be manually disabled. The format helps with loading times since, the tokens are easily reconstructed, and with hiding the source code, since recovering it would require a specialized tool. Code comments are not stored in this format. The `--test` command can also include a `--use-binary-tokens` flag which will run the GDScript tests with the binary format instead of the regular source code by converting them in-memory before the test runs. --- modules/gdscript/gdscript.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'modules/gdscript/gdscript.cpp') diff --git a/modules/gdscript/gdscript.cpp b/modules/gdscript/gdscript.cpp index e78c113c6c..551adcb320 100644 --- a/modules/gdscript/gdscript.cpp +++ b/modules/gdscript/gdscript.cpp @@ -35,6 +35,7 @@ #include "gdscript_compiler.h" #include "gdscript_parser.h" #include "gdscript_rpc_callable.h" +#include "gdscript_tokenizer_buffer.h" #include "gdscript_warning.h" #ifdef TOOLS_ENABLED @@ -740,7 +741,12 @@ Error GDScript::reload(bool p_keep_state) { valid = false; GDScriptParser parser; - Error err = parser.parse(source, path, false); + Error err; + if (!binary_tokens.is_empty()) { + err = parser.parse_binary(binary_tokens, path); + } else { + err = parser.parse(source, path, false); + } if (err) { if (EngineDebugger::is_active()) { GDScriptLanguage::get_singleton()->debug_break_parse(_get_debug_path(), parser.get_errors().front()->get().line, "Parser Error: " + parser.get_errors().front()->get().message); @@ -1050,6 +1056,19 @@ Error GDScript::load_source_code(const String &p_path) { return OK; } +void GDScript::set_binary_tokens_source(const Vector &p_binary_tokens) { + binary_tokens = p_binary_tokens; +} + +const Vector &GDScript::get_binary_tokens_source() const { + return binary_tokens; +} + +Vector GDScript::get_as_binary_tokens() const { + GDScriptTokenizerBuffer tokenizer; + return tokenizer.parse_code_string(source); +} + const HashMap &GDScript::debug_get_member_functions() const { return member_functions; } @@ -2805,6 +2824,7 @@ Ref ResourceFormatLoaderGDScript::load(const String &p_path, const Str void ResourceFormatLoaderGDScript::get_recognized_extensions(List *p_extensions) const { p_extensions->push_back("gd"); + p_extensions->push_back("gdc"); } bool ResourceFormatLoaderGDScript::handles_type(const String &p_type) const { @@ -2813,7 +2833,7 @@ bool ResourceFormatLoaderGDScript::handles_type(const String &p_type) const { String ResourceFormatLoaderGDScript::get_resource_type(const String &p_path) const { String el = p_path.get_extension().to_lower(); - if (el == "gd") { + if (el == "gd" || el == "gdc") { return "GDScript"; } return ""; -- cgit v1.2.3 From 72e5f8c31e9237a4581cf250aa9943a4290f4017 Mon Sep 17 00:00:00 2001 From: George Marques Date: Fri, 26 Jan 2024 14:49:31 -0300 Subject: GDScript: Enable compression on export Besides the regular option to export GDScript as binary tokens, this also includes a compression option on top of it. The binary format needs to encode some information which generally makes it bigger than the source text. This option reduces that difference by using Zstandard compression on the buffer. --- modules/gdscript/gdscript.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/gdscript/gdscript.cpp') diff --git a/modules/gdscript/gdscript.cpp b/modules/gdscript/gdscript.cpp index 551adcb320..7a576d5292 100644 --- a/modules/gdscript/gdscript.cpp +++ b/modules/gdscript/gdscript.cpp @@ -1066,7 +1066,7 @@ const Vector &GDScript::get_binary_tokens_source() const { Vector GDScript::get_as_binary_tokens() const { GDScriptTokenizerBuffer tokenizer; - return tokenizer.parse_code_string(source); + return tokenizer.parse_code_string(source, GDScriptTokenizerBuffer::COMPRESS_NONE); } const HashMap &GDScript::debug_get_member_functions() const { -- cgit v1.2.3