8 files changed, 156 insertions, 22 deletions

diff --git a/modules/gdscript/tests/gdscript_test_runner.cpp b/modules/gdscript/tests/gdscript_test_runner.cpp
index 4d93a6fc18..880289d2a8 100644
--- a/modules/gdscript/tests/gdscript_test_runner.cpp
+++ b/modules/gdscript/tests/gdscript_test_runner.cpp
@@ -34,6 +34,7 @@
 #include "../gdscript_analyzer.h"
 #include "../gdscript_compiler.h"
 #include "../gdscript_parser.h"
+#include "../gdscript_tokenizer_buffer.h"
 
 #include "core/config/project_settings.h"
 #include "core/core_globals.h"
@@ -131,10 +132,11 @@ void finish_language() {
 
 StringName GDScriptTestRunner::test_function_name;
 
-GDScriptTestRunner::GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames) {
+GDScriptTestRunner::GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames, bool p_use_binary_tokens) {
 	test_function_name = StaticCString::create("test");
 	do_init_languages = p_init_language;
 	print_filenames = p_print_filenames;
+	binary_tokens = p_use_binary_tokens;
 
 	source_dir = p_source_dir;
 	if (!source_dir.ends_with("/")) {
@@ -277,6 +279,9 @@ bool GDScriptTestRunner::make_tests_for_dir(const String &p_dir) {
 			if (next.ends_with(".notest.gd")) {
 				next = dir->get_next();
 				continue;
+			} else if (binary_tokens && next.ends_with(".textonly.gd")) {
+				next = dir->get_next();
+				continue;
 			} else if (next.get_extension().to_lower() == "gd") {
 #ifndef DEBUG_ENABLED
 				// On release builds, skip tests marked as debug only.
@@ -299,6 +304,9 @@ bool GDScriptTestRunner::make_tests_for_dir(const String &p_dir) {
 					ERR_FAIL_V_MSG(false, "Could not find output file for " + next);
 				}
 				GDScriptTest test(current_dir.path_join(next), current_dir.path_join(out_file), source_dir);
+				if (binary_tokens) {
+					test.set_tokenizer_mode(GDScriptTest::TOKENIZER_BUFFER);
+				}
 				tests.push_back(test);
 			}
 		}
@@ -321,24 +329,65 @@ bool GDScriptTestRunner::make_tests() {
 	return make_tests_for_dir(dir->get_current_dir());
 }
 
-bool GDScriptTestRunner::generate_class_index() {
+static bool generate_class_index_recursive(const String &p_dir) {
+	Error err = OK;
+	Ref<DirAccess> dir(DirAccess::open(p_dir, &err));
+
+	if (err != OK) {
+		return false;
+	}
+
+	String current_dir = dir->get_current_dir();
+
+	dir->list_dir_begin();
+	String next = dir->get_next();
+
 	StringName gdscript_name = GDScriptLanguage::get_singleton()->get_name();
-	for (int i = 0; i < tests.size(); i++) {
-		GDScriptTest test = tests[i];
-		String base_type;
+	while (!next.is_empty()) {
+		if (dir->current_is_dir()) {
+			if (next == "." || next == ".." || next == "completion" || next == "lsp") {
+				next = dir->get_next();
+				continue;
+			}
+			if (!generate_class_index_recursive(current_dir.path_join(next))) {
+				return false;
+			}
+		} else {
+			if (!next.ends_with(".gd")) {
+				next = dir->get_next();
+				continue;
+			}
+			String base_type;
+			String source_file = current_dir.path_join(next);
+			String class_name = GDScriptLanguage::get_singleton()->get_global_class_name(source_file, &base_type);
+			if (class_name.is_empty()) {
+				next = dir->get_next();
+				continue;
+			}
+			ERR_FAIL_COND_V_MSG(ScriptServer::is_global_class(class_name), false,
+					"Class name '" + class_name + "' from " + source_file + " is already used in " + ScriptServer::get_global_class_path(class_name));
 
-		String class_name = GDScriptLanguage::get_singleton()->get_global_class_name(test.get_source_file(), &base_type);
-		if (class_name.is_empty()) {
-			continue;
+			ScriptServer::add_global_class(class_name, base_type, gdscript_name, source_file);
 		}
-		ERR_FAIL_COND_V_MSG(ScriptServer::is_global_class(class_name), false,
-				"Class name '" + class_name + "' from " + test.get_source_file() + " is already used in " + ScriptServer::get_global_class_path(class_name));
 
-		ScriptServer::add_global_class(class_name, base_type, gdscript_name, test.get_source_file());
+		next = dir->get_next();
 	}
+
+	dir->list_dir_end();
+
 	return true;
 }
 
+bool GDScriptTestRunner::generate_class_index() {
+	Error err = OK;
+	Ref<DirAccess> dir(DirAccess::open(source_dir, &err));
+
+	ERR_FAIL_COND_V_MSG(err != OK, false, "Could not open specified test directory.");
+
+	source_dir = dir->get_current_dir() + "/"; // Make it absolute path.
+	return generate_class_index_recursive(dir->get_current_dir());
+}
+
 GDScriptTest::GDScriptTest(const String &p_source_path, const String &p_output_path, const String &p_base_dir) {
 	source_file = p_source_path;
 	output_file = p_output_path;
@@ -484,7 +533,15 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) {
 	Ref<GDScript> script;
 	script.instantiate();
 	script->set_path(source_file);
-	err = script->load_source_code(source_file);
+	if (tokenizer_mode == TOKENIZER_TEXT) {
+		err = script->load_source_code(source_file);
+	} else {
+		String code = FileAccess::get_file_as_string(source_file, &err);
+		if (!err) {
+			Vector<uint8_t> buffer = GDScriptTokenizerBuffer::parse_code_string(code);
+			script->set_binary_tokens_source(buffer);
+		}
+	}
 	if (err != OK) {
 		enable_stdout();
 		result.status = GDTEST_LOAD_ERROR;
@@ -494,7 +551,11 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) {
 
 	// Test parsing.
 	GDScriptParser parser;
-	err = parser.parse(script->get_source_code(), source_file, false);
+	if (tokenizer_mode == TOKENIZER_TEXT) {
+		err = parser.parse(script->get_source_code(), source_file, false);
+	} else {
+		err = parser.parse_binary(script->get_binary_tokens_source(), source_file);
+	}
 	if (err != OK) {
 		enable_stdout();
 		result.status = GDTEST_PARSER_ERROR;
@@ -583,7 +644,14 @@ GDScriptTest::TestResult GDScriptTest::execute_test_code(bool p_is_generating) {
 	add_print_handler(&_print_handler);
 	add_error_handler(&_error_handler);
 
-	script->reload();
+	err = script->reload();
+	if (err) {
+		enable_stdout();
+		result.status = GDTEST_LOAD_ERROR;
+		result.output = "";
+		result.passed = false;
+		ERR_FAIL_V_MSG(result, "\nCould not reload script: '" + source_file + "'");
+	}
 
 	// Create object instance for test.
 	Object *obj = ClassDB::instantiate(script->get_native()->get_name());
diff --git a/modules/gdscript/tests/gdscript_test_runner.h b/modules/gdscript/tests/gdscript_test_runner.h
index b1190604ad..57e3ac86f9 100644
--- a/modules/gdscript/tests/gdscript_test_runner.h
+++ b/modules/gdscript/tests/gdscript_test_runner.h
@@ -62,6 +62,11 @@ public:
 		bool passed;
 	};
 
+	enum TokenizerMode {
+		TOKENIZER_TEXT,
+		TOKENIZER_BUFFER,
+	};
+
 private:
 	struct ErrorHandlerData {
 		TestResult *result = nullptr;
@@ -79,6 +84,8 @@ private:
 	PrintHandlerList _print_handler;
 	ErrorHandlerList _error_handler;
 
+	TokenizerMode tokenizer_mode = TOKENIZER_TEXT;
+
 	void enable_stdout();
 	void disable_stdout();
 	bool check_output(const String &p_output) const;
@@ -96,6 +103,9 @@ public:
 	const String get_source_relative_filepath() const { return source_file.trim_prefix(base_dir); }
 	const String &get_output_file() const { return output_file; }
 
+	void set_tokenizer_mode(TokenizerMode p_tokenizer_mode) { tokenizer_mode = p_tokenizer_mode; }
+	TokenizerMode get_tokenizer_mode() const { return tokenizer_mode; }
+
 	GDScriptTest(const String &p_source_path, const String &p_output_path, const String &p_base_dir);
 	GDScriptTest() :
 			GDScriptTest(String(), String(), String()) {} // Needed to use in Vector.
@@ -108,6 +118,7 @@ class GDScriptTestRunner {
 	bool is_generating = false;
 	bool do_init_languages = false;
 	bool print_filenames; // Whether filenames should be printed when generated/running tests
+	bool binary_tokens; // Test with buffer tokenizer.
 
 	bool make_tests();
 	bool make_tests_for_dir(const String &p_dir);
@@ -120,7 +131,7 @@ public:
 	int run_tests();
 	bool generate_outputs();
 
-	GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames = false);
+	GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames = false, bool p_use_binary_tokens = false);
 	~GDScriptTestRunner();
 };
 
diff --git a/modules/gdscript/tests/gdscript_test_runner_suite.h b/modules/gdscript/tests/gdscript_test_runner_suite.h
index 5fd7d942d2..5acf436e42 100644
--- a/modules/gdscript/tests/gdscript_test_runner_suite.h
+++ b/modules/gdscript/tests/gdscript_test_runner_suite.h
@@ -38,12 +38,10 @@
 namespace GDScriptTests {
 
 TEST_SUITE("[Modules][GDScript]") {
-	// GDScript 2.0 is still under heavy construction.
-	// Allow the tests to fail, but do not ignore errors during development.
-	// Update the scripts and expected output as needed.
 	TEST_CASE("Script compilation and runtime") {
 		bool print_filenames = OS::get_singleton()->get_cmdline_args().find("--print-filenames") != nullptr;
-		GDScriptTestRunner runner("modules/gdscript/tests/scripts", true, print_filenames);
+		bool use_binary_tokens = OS::get_singleton()->get_cmdline_args().find("--use-binary-tokens") != nullptr;
+		GDScriptTestRunner runner("modules/gdscript/tests/scripts", true, print_filenames, use_binary_tokens);
 		int fail_count = runner.run_tests();
 		INFO("Make sure `*.out` files have expected results.");
 		REQUIRE_MESSAGE(fail_count == 0, "All GDScript tests should pass.");
diff --git a/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.gd b/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.textonly.gd
index 9ad77f1432..9ad77f1432 100644
--- a/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.gd
+++ b/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.textonly.gd
diff --git a/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.out b/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.textonly.out
index 31bed2dbc7..31bed2dbc7 100644
--- a/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.out
+++ b/modules/gdscript/tests/scripts/parser/errors/mixing_tabs_spaces.textonly.out
diff --git a/modules/gdscript/tests/scripts/parser/features/multiline_if.gd b/modules/gdscript/tests/scripts/parser/features/multiline_if.gd
index 86152f4543..7b82d9b1da 100644
--- a/modules/gdscript/tests/scripts/parser/features/multiline_if.gd
+++ b/modules/gdscript/tests/scripts/parser/features/multiline_if.gd
@@ -9,6 +9,7 @@ func test():
 
 	# Alternatively, backslashes can be used.
 	if 1 == 1 \
+	\
 		and 2 == 2 and \
 		3 == 3:
 		pass
diff --git a/modules/gdscript/tests/test_gdscript.cpp b/modules/gdscript/tests/test_gdscript.cpp
index 467bedc4b2..e4fab68e06 100644
--- a/modules/gdscript/tests/test_gdscript.cpp
+++ b/modules/gdscript/tests/test_gdscript.cpp
@@ -34,6 +34,7 @@
 #include "../gdscript_compiler.h"
 #include "../gdscript_parser.h"
 #include "../gdscript_tokenizer.h"
+#include "../gdscript_tokenizer_buffer.h"
 
 #include "core/config/project_settings.h"
 #include "core/io/file_access.h"
@@ -50,7 +51,7 @@
 namespace GDScriptTests {
 
 static void test_tokenizer(const String &p_code, const Vector<String> &p_lines) {
-	GDScriptTokenizer tokenizer;
+	GDScriptTokenizerText tokenizer;
 	tokenizer.set_source_code(p_code);
 
 	int tab_size = 4;
@@ -107,6 +108,53 @@ static void test_tokenizer(const String &p_code, const Vector<String> &p_lines)
 	print_line(current.get_name()); // Should be EOF
 }
 
+static void test_tokenizer_buffer(const Vector<uint8_t> &p_buffer, const Vector<String> &p_lines);
+
+static void test_tokenizer_buffer(const String &p_code, const Vector<String> &p_lines) {
+	Vector<uint8_t> binary = GDScriptTokenizerBuffer::parse_code_string(p_code);
+	test_tokenizer_buffer(binary, p_lines);
+}
+
+static void test_tokenizer_buffer(const Vector<uint8_t> &p_buffer, const Vector<String> &p_lines) {
+	GDScriptTokenizerBuffer tokenizer;
+	tokenizer.set_code_buffer(p_buffer);
+
+	int tab_size = 4;
+#ifdef TOOLS_ENABLED
+	if (EditorSettings::get_singleton()) {
+		tab_size = EditorSettings::get_singleton()->get_setting("text_editor/behavior/indent/size");
+	}
+#endif // TOOLS_ENABLED
+	String tab = String(" ").repeat(tab_size);
+
+	GDScriptTokenizer::Token current = tokenizer.scan();
+	while (current.type != GDScriptTokenizer::Token::TK_EOF) {
+		StringBuilder token;
+		token += " --> "; // Padding for line number.
+
+		for (int l = current.start_line; l <= current.end_line && l <= p_lines.size(); l++) {
+			print_line(vformat("%04d %s", l, p_lines[l - 1]).replace("\t", tab));
+		}
+
+		token += current.get_name();
+
+		if (current.type == GDScriptTokenizer::Token::ERROR || current.type == GDScriptTokenizer::Token::LITERAL || current.type == GDScriptTokenizer::Token::IDENTIFIER || current.type == GDScriptTokenizer::Token::ANNOTATION) {
+			token += "(";
+			token += Variant::get_type_name(current.literal.get_type());
+			token += ") ";
+			token += current.literal;
+		}
+
+		print_line(token.as_string());
+
+		print_line("-------------------------------------------------------");
+
+		current = tokenizer.scan();
+	}
+
+	print_line(current.get_name()); // Should be EOF
+}
+
 static void test_parser(const String &p_code, const String &p_script_path, const Vector<String> &p_lines) {
 	GDScriptParser parser;
 	Error err = parser.parse(p_code, p_script_path, false);
@@ -119,7 +167,7 @@ static void test_parser(const String &p_code, const String &p_script_path, const
 	}
 
 	GDScriptAnalyzer analyzer(&parser);
-	analyzer.analyze();
+	err = analyzer.analyze();
 
 	if (err != OK) {
 		const List<GDScriptParser::ParserError> &errors = parser.get_errors();
@@ -212,7 +260,7 @@ void test(TestType p_type) {
 	}
 
 	String test = cmdlargs.back()->get();
-	if (!test.ends_with(".gd")) {
+	if (!test.ends_with(".gd") && !test.ends_with(".gdc")) {
 		print_line("This test expects a path to a GDScript file as its last parameter. Got: " + test);
 		return;
 	}
@@ -255,6 +303,13 @@ void test(TestType p_type) {
 		case TEST_TOKENIZER:
 			test_tokenizer(code, lines);
 			break;
+		case TEST_TOKENIZER_BUFFER:
+			if (test.ends_with(".gdc")) {
+				test_tokenizer_buffer(buf, lines);
+			} else {
+				test_tokenizer_buffer(code, lines);
+			}
+			break;
 		case TEST_PARSER:
 			test_parser(code, test, lines);
 			break;
diff --git a/modules/gdscript/tests/test_gdscript.h b/modules/gdscript/tests/test_gdscript.h
index b39dfe2b5a..32f278d5ce 100644
--- a/modules/gdscript/tests/test_gdscript.h
+++ b/modules/gdscript/tests/test_gdscript.h
@@ -39,6 +39,7 @@ namespace GDScriptTests {
 
 enum TestType {
 	TEST_TOKENIZER,
+	TEST_TOKENIZER_BUFFER,
 	TEST_PARSER,
 	TEST_COMPILER,
 	TEST_BYTECODE,