GDScript: Reintroduce binary tokenization on export

This adds back a function available in 3.x: exporting the GDScript
files in a binary form by converting the tokens recognized by the
tokenizer into a data format.

It is enabled by default on export but can be manually disabled. The
format helps with loading times since, the tokens are easily
reconstructed, and with hiding the source code, since recovering it
would require a specialized tool. Code comments are not stored in this
format.

The `--test` command can also include a `--use-binary-tokens` flag
which will run the GDScript tests with the binary format instead of the
regular source code by converting them in-memory before the test runs.

8 files changed, 156 insertions, 22 deletions

diff --git a/modules/gdscript/tests/gdscript_test_runner.cpp b/modules/gdscript/tests/gdscript_test_runner.cpp
index 4d93a6fc18..880289d2a8 100644
--- a/modules/gdscript/tests/gdscript_test_runner.cpp
+++ b/modules/gdscript/tests/gdscript_test_runner.cpp
@@ -34,6 +34,7 @@
 #include "../gdscript_analyzer.h"
 #include "../gdscript_compiler.h"
 #include "../gdscript_parser.h"
+#include "../gdscript_tokenizer_buffer.h"
 
 #include "core/config/project_settings.h"
 #include "core/core_globals.h"
@@ -131,10 +132,11 @@ void finish_language() {
 
 StringName GDScriptTestRunner::test_function_name;
 
-GDScriptTestRunner::GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames) {
+GDScriptTestRunner::GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames, bool p_use_binary_tokens) {
 	test_function_name = StaticCString::create("test");
 	do_init_languages = p_init_language;
 	print_filenames = p_print_filenames;
+	binary_tokens = p_use_binary_tokens;
 
 	source_dir = p_source_dir;
 	if (!source_dir.ends_with("/")) {
@@ -277,6 +279,9 @@ bool GDScriptTestRunner::make_tests_for_dir(const String &p_dir) {
 			if (next.ends_with(".notest.gd")) {
 				next = dir->get_next();
 				continue;
+			} else if (binary_tokens && next.ends_with(".textonly.gd")) {
+				next = dir->get_next();
+				continue;
 			} else if (next.get_extension().to_lower() == "gd") {
 #ifndef DEBUG_ENABLED
 				// On release builds, skip tests marked as debug only.
@@ -299,6 +304,9 @@ bool GDScriptTestRunner::make_tests_for_dir(const String &p_dir) {
 					ERR_FAIL_V_MSG(false, "Could not find output file for " + next);
 				}
 				GDScriptTest test(current_dir.path_join(next), current_dir.path_join(out_file), source_dir);
+				if (binary_tokens) {
+					test.set_tokenizer_mode(GDScriptTest::TOKENIZER_BUFFER);
+				}
 				tests.push_back(test);
 			}
 		}
@@ -321,24 +329,65 @@ bool GDScriptTestRunner::make_tests() {
 	return make_tests_for_dir(dir->get_current_dir());
 }
 
-bool GDScriptTestRunner::generate_class_index() {
+static bool generate_class_index_recursive(const String &p_dir) {
+	Error err = OK;
+	Ref<DirAccess> dir(DirAccess::open(p_dir, &err));
+
+	if (err != OK) {
+		return false;
+	}
+
+	String current_dir = dir->get_current_dir();
+
+	dir->list_dir_begin();
+	String next = dir->get_next();
+
 	StringName gdscript_name = GDScriptLanguage::get_singleton()->get_name();
-	for (int i = 0; i < tests.size(); i++) {
-		GDScriptTest test = tests[i];
-		String base_type;
+	while (!next.is_empty()) {
+		if (dir->current_is_dir()) {
+			if (next == "." || next == ".." || next == "completion" || next == "lsp") {
+				next = dir->get_next();
+				continue;
+			}
+			if (!generate_class_index_recursive(current_dir.path_join(next))) {
+				return false;
+			}
+		} else {
+			if (!next.ends_with(".gd")) {
+				next = dir->get_next();
+				continue;
+			}
+			String base_type;
+			String source_file = current_dir.path_join(next);
+			String class_name = GDScriptLanguage::get_singleton()->get_global_class_name(source_file, &base_type);
+			if (class_name.is_empty()) {
+				next = dir->get_next();
+				continue;
+			}
+			ERR_FAIL_COND_V_MSG(ScriptServer::is_global_class(class_name), false,
+					"Class name '" + class_name + "' from " + source_file + " is already used in " + ScriptServer::get_global_class_path(class_name));
 
-		String class_name = GDScriptLanguage::get_singleton()->get_global_class_name(test.get_source_file(), &base_type);
-		if (class_name.is_empty()) {
-			continue;
+			ScriptServer::add_global_class(class_name, base_type, gdscript_name, source_file);
 		}
-		ERR_FAIL_COND_V_MSG(ScriptServer::is_global_class(class_name), false,
-				"Class name '" + class_name + "' from " + test.get_source_file() + " is already used in " + ScriptServer::get_global_class_path(class_name));
 
-		ScriptServer::add_global_class(class_name, base_type, gdscript_name, test.get_source_file());
+		next = dir->get_next();
 	}
+
+	dir->list_dir_end();
+
 	return true;
 }
 
+bool GDScriptTestRunner::generate_class_index() {
+	Error err = OK;
+	Ref<DirAccess> dir(DirAccess::open(source_dir, &err));
+
+	ERR_FAIL_COND_V_MSG(err != OK, false, "Could not open specified test directory.");
+
+	source_dir = dir->get_current_dir() + "/"; // Make it absolute path.
+	return generate_class_index_recursive(dir->get_current_dir());
+}
+
 GDScriptTest::GDScriptTest(const String &p_source_path, const String &p_output_path, const String &p_base_dir) {
 	source_file = p_source_path;
 	output_file = p_output_path;
@@ -484,7 +533,15 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) {
 	Ref<GDScript> script;
 	script.instantiate();
 	script->set_path(source_file);
-	err = script->load_source_code(source_file);
+	if (tokenizer_mode == TOKENIZER_TEXT) {
+		err = script->load_source_code(source_file);
+	} else {
+		String code = FileAccess::get_file_as_string(source_file, &err);
+		if (!err) {
+			Vector<uint8_t> buffer = GDScriptTokenizerBuffer::parse_code_string(code);
+			script->set_binary_tokens_source(buffer);
+		}
+	}
 	if (err != OK) {
 		enable_stdout();
 		result.status = GDTEST_LOAD_ERROR;
@@ -494,7 +551,11 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) {
 
 	// Test parsing.
 	GDScriptParser parser;
-	err = parser.parse(script->get_source_code(), source_file, false);
+	if (tokenizer_mode == TOKENIZER_TEXT) {
+		err = parser.parse(script->get_source_code(), source_file, false);
+	} else {
+		err = parser.parse_binary(script->get_binary_tokens_source(), source_file);
+	}
 	if (err != OK) {
 		enable_stdout();
 		result.status = GDTEST_PARSER_ERROR;
@@ -583,7 +644,14 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) {
 	add_print_handler(&_print_handler);
 	add_error_handler(&_error_handler);
 
-	script->reload();
+	err = script->reload();
+	if (err) {
+		enable_stdout();
+		result.status = GDTEST_LOAD_ERROR;
+		result.output = "";
+		result.passed = false;
+		ERR_FAIL_V_MSG(result, "\nCould not reload script: '" + source_file + "'");
+	}
 
 	// Create object instance for test.
 	Object *obj = ClassDB::instantiate(script->get_native()->get_name());
diff --git a/modules/gdscript/tests/gdscript_test_runner.h b/modules/gdscript/tests/gdscript_test_runner.h
index b1190604ad..57e3ac86f9 100644
--- a/modules/gdscript/tests/gdscript_test_runner.h
+++ b/modules/gdscript/tests/gdscript_test_runner.h
@@ -62,6 +62,11 @@ public:
 		bool passed;
 	};
 
+	enum TokenizerMode {
+		TOKENIZER_TEXT,
+		TOKENIZER_BUFFER,
+	};
+
 private:
 	struct ErrorHandlerData {
 		TestResult *result = nullptr;
@@ -79,6 +84,8 @@ private:
 	PrintHandlerList _print_handler;
 	ErrorHandlerList _error_handler;
 
+	TokenizerMode tokenizer_mode = TOKENIZER_TEXT;
+
 	void enable_stdout();
 	void disable_stdout();
 	bool check_output(const String &p_output) const;
@@ -96,6 +103,9 @@ public:
 	const String get_source_relative_filepath() const { return source_file.trim_prefix(base_dir); }
 	const String &get_output_file() const { return output_file; }
 
+	void set_tokenizer_mode(TokenizerMode p_tokenizer_mode) { tokenizer_mode = p_tokenizer_mode; }
+	TokenizerMode get_tokenizer_mode() const { return tokenizer_mode; }
+
 	GDScriptTest(const String &p_source_path, const String &p_output_path, const String &p_base_dir);
 	GDScriptTest() :
 			GDScriptTest(String(), String(), String()) {} // Needed to use in Vector.
@@ -108,6 +118,7 @@ class GDScriptTestRunner {
 	bool is_generating = false;
 	bool do_init_languages = false;
 	bool print_filenames; // Whether filenames should be printed when generated/running tests
+	bool binary_tokens; // Test with buffer tokenizer.
 
 	bool make_tests();
 	bool make_tests_for_dir(const String &p_dir);
@@ -120,7 +131,7 @@ public:
 	int run_tests();
 	bool generate_outputs();
 
-	GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames = false);
+	GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames = false, bool p_use_binary_tokens = false);
 	~GDScriptTestRunner();
 };
 
diff --git a/modules/gdscript/tests/gdscript_test_runner_suite.h b/modules/gdscript/tests/gdscript_test_runner_suite.h
index 5fd7d942d2..5acf436e42 100644
--- a/modules/gdscript/tests/gdscript_test_runner_suite.h
+++ b/modules/gdscript/tests/gdscript_test_runner_suite.h
@@ -38,12 +38,10 @@
 namespace GDScriptTests {
 
 TEST_SUITE("[Modules][GDScript]") {
-	// GDScript 2.0 is still under heavy construction.
-	// Allow the tests to fail, but do not ignore errors during development.
-	// Update the scripts and expected output as needed.
 	TEST_CASE("Script compilation and runtime") {
 		bool print_filenames = OS::get_singleton()->get_cmdline_args().find("--print-filenames") != nullptr;
-		GDScriptTestRunner runner("modules/gdscript/tests/scripts", true, print_filenames);
+		bool use_binary_tokens = OS::get_singleton()->get_cmdline_args().find("--use-binary-tokens") != nullptr;
+		GDScriptTestRunner runner("modules/gdscript/tests/scripts", true, print_filenames, use_binary_tokens);
 		int fail_count = runner.run_tests();
 		INFO("Make sure `*.out` files have expected results.");
 		REQUIRE_MESSAGE(fail_count == 0, "All GDScript tests should pass.");
diff --git a/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.gd b/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.textonly.gd
index 9ad77f1432..9ad77f1432 100644
--- a/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.gd
+++ b/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.textonly.gd
diff --git a/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.out b/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.textonly.out
index 31bed2dbc7..31bed2dbc7 100644
--- a/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.out
+++ b/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.textonly.out
diff --git a/modules/gdscript/tests/scripts/parser/features/multiline_if.gd b/modules/gdscript/tests/scripts/parser/features/multiline_if.gd
index 86152f4543..7b82d9b1da 100644
--- a/modules/gdscript/tests/scripts/parser/features/multiline_if.gd
+++ b/modules/gdscript/tests/scripts/parser/features/multiline_if.gd
@@ -9,6 +9,7 @@ func test():
 
 	# Alternatively, backslashes can be used.
 	if 1 == 1 \
+	\
 		and 2 == 2 and \
 		3 == 3:
 		pass
diff --git a/modules/gdscript/tests/test_gdscript.cpp b/modules/gdscript/tests/test_gdscript.cpp
index 467bedc4b2..e4fab68e06 100644
--- a/modules/gdscript/tests/test_gdscript.cpp
+++ b/modules/gdscript/tests/test_gdscript.cpp
@@ -34,6 +34,7 @@
 #include "../gdscript_compiler.h"
 #include "../gdscript_parser.h"
 #include "../gdscript_tokenizer.h"
+#include "../gdscript_tokenizer_buffer.h"
 
 #include "core/config/project_settings.h"
 #include "core/io/file_access.h"
@@ -50,7 +51,7 @@
 namespace GDScriptTests {
 
 static void test_tokenizer(const String &p_code, const Vector<String> &p_lines) {
-	GDScriptTokenizer tokenizer;
+	GDScriptTokenizerText tokenizer;
 	tokenizer.set_source_code(p_code);
 
 	int tab_size = 4;
@@ -107,6 +108,53 @@ static void test_tokenizer(const String &p_code, const Vector<String> &p_lines)
 	print_line(current.get_name()); // Should be EOF
 }
 
+static void test_tokenizer_buffer(const Vector<uint8_t> &p_buffer, const Vector<String> &p_lines);
+
+static void test_tokenizer_buffer(const String &p_code, const Vector<String> &p_lines) {
+	Vector<uint8_t> binary = GDScriptTokenizerBuffer::parse_code_string(p_code);
+	test_tokenizer_buffer(binary, p_lines);
+}
+
+static void test_tokenizer_buffer(const Vector<uint8_t> &p_buffer, const Vector<String> &p_lines) {
+	GDScriptTokenizerBuffer tokenizer;
+	tokenizer.set_code_buffer(p_buffer);
+
+	int tab_size = 4;
+#ifdef TOOLS_ENABLED
+	if (EditorSettings::get_singleton()) {
+		tab_size = EditorSettings::get_singleton()->get_setting("text_editor/behavior/indent/size");
+	}
+#endif // TOOLS_ENABLED
+	String tab = String(" ").repeat(tab_size);
+
+	GDScriptTokenizer::Token current = tokenizer.scan();
+	while (current.type != GDScriptTokenizer::Token::TK_EOF) {
+		StringBuilder token;
+		token += " --> "; // Padding for line number.
+
+		for (int l = current.start_line; l <= current.end_line && l <= p_lines.size(); l++) {
+			print_line(vformat("%04d %s", l, p_lines[l - 1]).replace("\t", tab));
+		}
+
+		token += current.get_name();
+
+		if (current.type == GDScriptTokenizer::Token::ERROR || current.type == GDScriptTokenizer::Token::LITERAL || current.type == GDScriptTokenizer::Token::IDENTIFIER || current.type == GDScriptTokenizer::Token::ANNOTATION) {
+			token += "(";
+			token += Variant::get_type_name(current.literal.get_type());
+			token += ") ";
+			token += current.literal;
+		}
+
+		print_line(token.as_string());
+
+		print_line("-------------------------------------------------------");
+
+		current = tokenizer.scan();
+	}
+
+	print_line(current.get_name()); // Should be EOF
+}
+
 static void test_parser(const String &p_code, const String &p_script_path, const Vector<String> &p_lines) {
 	GDScriptParser parser;
 	Error err = parser.parse(p_code, p_script_path, false);
@@ -119,7 +167,7 @@ static void test_parser(const String &p_code, const String &p_script_path, const
 	}
 
 	GDScriptAnalyzer analyzer(&parser);
-	analyzer.analyze();
+	err = analyzer.analyze();
 
 	if (err != OK) {
 		const List<GDScriptParser::ParserError> &errors = parser.get_errors();
@@ -212,7 +260,7 @@ void test(TestType p_type) {
 	}
 
 	String test = cmdlargs.back()->get();
-	if (!test.ends_with(".gd")) {
+	if (!test.ends_with(".gd") && !test.ends_with(".gdc")) {
 		print_line("This test expects a path to a GDScript file as its last parameter. Got: " + test);
 		return;
 	}
@@ -255,6 +303,13 @@ void test(TestType p_type) {
 		case TEST_TOKENIZER:
 			test_tokenizer(code, lines);
 			break;
+		case TEST_TOKENIZER_BUFFER:
+			if (test.ends_with(".gdc")) {
+				test_tokenizer_buffer(buf, lines);
+			} else {
+				test_tokenizer_buffer(code, lines);
+			}
+			break;
 		case TEST_PARSER:
 			test_parser(code, test, lines);
 			break;
diff --git a/modules/gdscript/tests/test_gdscript.h b/modules/gdscript/tests/test_gdscript.h
index b39dfe2b5a..32f278d5ce 100644
--- a/modules/gdscript/tests/test_gdscript.h
+++ b/modules/gdscript/tests/test_gdscript.h
@@ -39,6 +39,7 @@ namespace GDScriptTests {
 
 enum TestType {
 	TEST_TOKENIZER,
+	TEST_TOKENIZER_BUFFER,
 	TEST_PARSER,
 	TEST_COMPILER,
 	TEST_BYTECODE,