From b4d0a09f15c60c88bbf516d2f6dcdb451dcad9c7 Mon Sep 17 00:00:00 2001 From: George Marques Date: Mon, 22 Jan 2024 11:31:55 -0300 Subject: GDScript: Reintroduce binary tokenization on export This adds back a function available in 3.x: exporting the GDScript files in a binary form by converting the tokens recognized by the tokenizer into a data format. It is enabled by default on export but can be manually disabled. The format helps with loading times since, the tokens are easily reconstructed, and with hiding the source code, since recovering it would require a specialized tool. Code comments are not stored in this format. The `--test` command can also include a `--use-binary-tokens` flag which will run the GDScript tests with the binary format instead of the regular source code by converting them in-memory before the test runs. --- modules/gdscript/tests/gdscript_test_runner.cpp | 96 +++++++++++++++++++++---- 1 file changed, 82 insertions(+), 14 deletions(-) (limited to 'modules/gdscript/tests/gdscript_test_runner.cpp') diff --git a/modules/gdscript/tests/gdscript_test_runner.cpp b/modules/gdscript/tests/gdscript_test_runner.cpp index 4d93a6fc18..880289d2a8 100644 --- a/modules/gdscript/tests/gdscript_test_runner.cpp +++ b/modules/gdscript/tests/gdscript_test_runner.cpp @@ -34,6 +34,7 @@ #include "../gdscript_analyzer.h" #include "../gdscript_compiler.h" #include "../gdscript_parser.h" +#include "../gdscript_tokenizer_buffer.h" #include "core/config/project_settings.h" #include "core/core_globals.h" @@ -131,10 +132,11 @@ void finish_language() { StringName GDScriptTestRunner::test_function_name; -GDScriptTestRunner::GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames) { +GDScriptTestRunner::GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames, bool p_use_binary_tokens) { test_function_name = StaticCString::create("test"); do_init_languages = p_init_language; print_filenames = p_print_filenames; + binary_tokens = p_use_binary_tokens; source_dir = p_source_dir; if (!source_dir.ends_with("/")) { @@ -277,6 +279,9 @@ bool GDScriptTestRunner::make_tests_for_dir(const String &p_dir) { if (next.ends_with(".notest.gd")) { next = dir->get_next(); continue; + } else if (binary_tokens && next.ends_with(".textonly.gd")) { + next = dir->get_next(); + continue; } else if (next.get_extension().to_lower() == "gd") { #ifndef DEBUG_ENABLED // On release builds, skip tests marked as debug only. @@ -299,6 +304,9 @@ bool GDScriptTestRunner::make_tests_for_dir(const String &p_dir) { ERR_FAIL_V_MSG(false, "Could not find output file for " + next); } GDScriptTest test(current_dir.path_join(next), current_dir.path_join(out_file), source_dir); + if (binary_tokens) { + test.set_tokenizer_mode(GDScriptTest::TOKENIZER_BUFFER); + } tests.push_back(test); } } @@ -321,24 +329,65 @@ bool GDScriptTestRunner::make_tests() { return make_tests_for_dir(dir->get_current_dir()); } -bool GDScriptTestRunner::generate_class_index() { +static bool generate_class_index_recursive(const String &p_dir) { + Error err = OK; + Ref dir(DirAccess::open(p_dir, &err)); + + if (err != OK) { + return false; + } + + String current_dir = dir->get_current_dir(); + + dir->list_dir_begin(); + String next = dir->get_next(); + StringName gdscript_name = GDScriptLanguage::get_singleton()->get_name(); - for (int i = 0; i < tests.size(); i++) { - GDScriptTest test = tests[i]; - String base_type; + while (!next.is_empty()) { + if (dir->current_is_dir()) { + if (next == "." || next == ".." || next == "completion" || next == "lsp") { + next = dir->get_next(); + continue; + } + if (!generate_class_index_recursive(current_dir.path_join(next))) { + return false; + } + } else { + if (!next.ends_with(".gd")) { + next = dir->get_next(); + continue; + } + String base_type; + String source_file = current_dir.path_join(next); + String class_name = GDScriptLanguage::get_singleton()->get_global_class_name(source_file, &base_type); + if (class_name.is_empty()) { + next = dir->get_next(); + continue; + } + ERR_FAIL_COND_V_MSG(ScriptServer::is_global_class(class_name), false, + "Class name '" + class_name + "' from " + source_file + " is already used in " + ScriptServer::get_global_class_path(class_name)); - String class_name = GDScriptLanguage::get_singleton()->get_global_class_name(test.get_source_file(), &base_type); - if (class_name.is_empty()) { - continue; + ScriptServer::add_global_class(class_name, base_type, gdscript_name, source_file); } - ERR_FAIL_COND_V_MSG(ScriptServer::is_global_class(class_name), false, - "Class name '" + class_name + "' from " + test.get_source_file() + " is already used in " + ScriptServer::get_global_class_path(class_name)); - ScriptServer::add_global_class(class_name, base_type, gdscript_name, test.get_source_file()); + next = dir->get_next(); } + + dir->list_dir_end(); + return true; } +bool GDScriptTestRunner::generate_class_index() { + Error err = OK; + Ref dir(DirAccess::open(source_dir, &err)); + + ERR_FAIL_COND_V_MSG(err != OK, false, "Could not open specified test directory."); + + source_dir = dir->get_current_dir() + "/"; // Make it absolute path. + return generate_class_index_recursive(dir->get_current_dir()); +} + GDScriptTest::GDScriptTest(const String &p_source_path, const String &p_output_path, const String &p_base_dir) { source_file = p_source_path; output_file = p_output_path; @@ -484,7 +533,15 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) { Ref script; script.instantiate(); script->set_path(source_file); - err = script->load_source_code(source_file); + if (tokenizer_mode == TOKENIZER_TEXT) { + err = script->load_source_code(source_file); + } else { + String code = FileAccess::get_file_as_string(source_file, &err); + if (!err) { + Vector buffer = GDScriptTokenizerBuffer::parse_code_string(code); + script->set_binary_tokens_source(buffer); + } + } if (err != OK) { enable_stdout(); result.status = GDTEST_LOAD_ERROR; @@ -494,7 +551,11 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) { // Test parsing. GDScriptParser parser; - err = parser.parse(script->get_source_code(), source_file, false); + if (tokenizer_mode == TOKENIZER_TEXT) { + err = parser.parse(script->get_source_code(), source_file, false); + } else { + err = parser.parse_binary(script->get_binary_tokens_source(), source_file); + } if (err != OK) { enable_stdout(); result.status = GDTEST_PARSER_ERROR; @@ -583,7 +644,14 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) { add_print_handler(&_print_handler); add_error_handler(&_error_handler); - script->reload(); + err = script->reload(); + if (err) { + enable_stdout(); + result.status = GDTEST_LOAD_ERROR; + result.output = ""; + result.passed = false; + ERR_FAIL_V_MSG(result, "\nCould not reload script: '" + source_file + "'"); + } // Create object instance for test. Object *obj = ClassDB::instantiate(script->get_native()->get_name()); -- cgit v1.2.3 From 72e5f8c31e9237a4581cf250aa9943a4290f4017 Mon Sep 17 00:00:00 2001 From: George Marques Date: Fri, 26 Jan 2024 14:49:31 -0300 Subject: GDScript: Enable compression on export Besides the regular option to export GDScript as binary tokens, this also includes a compression option on top of it. The binary format needs to encode some information which generally makes it bigger than the source text. This option reduces that difference by using Zstandard compression on the buffer. --- modules/gdscript/tests/gdscript_test_runner.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'modules/gdscript/tests/gdscript_test_runner.cpp') diff --git a/modules/gdscript/tests/gdscript_test_runner.cpp b/modules/gdscript/tests/gdscript_test_runner.cpp index 880289d2a8..a0329eb8d2 100644 --- a/modules/gdscript/tests/gdscript_test_runner.cpp +++ b/modules/gdscript/tests/gdscript_test_runner.cpp @@ -538,7 +538,7 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) { } else { String code = FileAccess::get_file_as_string(source_file, &err); if (!err) { - Vector buffer = GDScriptTokenizerBuffer::parse_code_string(code); + Vector buffer = GDScriptTokenizerBuffer::parse_code_string(code, GDScriptTokenizerBuffer::COMPRESS_ZSTD); script->set_binary_tokens_source(buffer); } } -- cgit v1.2.3