GDScript: Reintroduce binary tokenization on export

This adds back a function available in 3.x: exporting the GDScript
files in a binary form by converting the tokens recognized by the
tokenizer into a data format.

It is enabled by default on export but can be manually disabled. The
format helps with loading times since, the tokens are easily
reconstructed, and with hiding the source code, since recovering it
would require a specialized tool. Code comments are not stored in this
format.

The `--test` command can also include a `--use-binary-tokens` flag
which will run the GDScript tests with the binary format instead of the
regular source code by converting them in-memory before the test runs.

1 files changed, 12 insertions, 1 deletions

diff --git a/modules/gdscript/tests/gdscript_test_runner.h b/modules/gdscript/tests/gdscript_test_runner.h
index b1190604ad..57e3ac86f9 100644
--- a/modules/gdscript/tests/gdscript_test_runner.h
+++ b/modules/gdscript/tests/gdscript_test_runner.h
@@ -62,6 +62,11 @@ public:
 		bool passed;
 	};
 
+	enum TokenizerMode {
+		TOKENIZER_TEXT,
+		TOKENIZER_BUFFER,
+	};
+
 private:
 	struct ErrorHandlerData {
 		TestResult *result = nullptr;
@@ -79,6 +84,8 @@ private:
 	PrintHandlerList _print_handler;
 	ErrorHandlerList _error_handler;
 
+	TokenizerMode tokenizer_mode = TOKENIZER_TEXT;
+
 	void enable_stdout();
 	void disable_stdout();
 	bool check_output(const String &p_output) const;
@@ -96,6 +103,9 @@ public:
 	const String get_source_relative_filepath() const { return source_file.trim_prefix(base_dir); }
 	const String &get_output_file() const { return output_file; }
 
+	void set_tokenizer_mode(TokenizerMode p_tokenizer_mode) { tokenizer_mode = p_tokenizer_mode; }
+	TokenizerMode get_tokenizer_mode() const { return tokenizer_mode; }
+
 	GDScriptTest(const String &p_source_path, const String &p_output_path, const String &p_base_dir);
 	GDScriptTest() :
 			GDScriptTest(String(), String(), String()) {} // Needed to use in Vector.
@@ -108,6 +118,7 @@ class GDScriptTestRunner {
 	bool is_generating = false;
 	bool do_init_languages = false;
 	bool print_filenames; // Whether filenames should be printed when generated/running tests
+	bool binary_tokens; // Test with buffer tokenizer.
 
 	bool make_tests();
 	bool make_tests_for_dir(const String &p_dir);
@@ -120,7 +131,7 @@ public:
 	int run_tests();
 	bool generate_outputs();
 
-	GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames = false);
+	GDScriptTestRunner(const String &p_source_dir, bool p_init_language, bool p_print_filenames = false, bool p_use_binary_tokens = false);
 	~GDScriptTestRunner();
 };