1 files changed, 40 insertions, 18 deletions

diff --git a/modules/gdscript/gdscript_tokenizer.h b/modules/gdscript/gdscript_tokenizer.h
index a64aaf6820..5d76375173 100644
--- a/modules/gdscript/gdscript_tokenizer.h
+++ b/modules/gdscript/gdscript_tokenizer.h
@@ -181,14 +181,13 @@ public:
 		bool can_precede_bin_op() const;
 		bool is_identifier() const;
 		bool is_node_name() const;
-		StringName get_identifier() const { return source; }
+		StringName get_identifier() const { return literal; }
 
 		Token(Type p_type) {
 			type = p_type;
 		}
 
-		Token() {
-		}
+		Token() {}
 	};
 
 #ifdef TOOLS_ENABLED
@@ -203,12 +202,26 @@ public:
 			new_line = p_new_line;
 		}
 	};
-	const HashMap<int, CommentData> &get_comments() const {
-		return comments;
-	}
+	virtual const HashMap<int, CommentData> &get_comments() const = 0;
 #endif // TOOLS_ENABLED
 
-private:
+	static String get_token_name(Token::Type p_token_type);
+
+	virtual int get_cursor_line() const = 0;
+	virtual int get_cursor_column() const = 0;
+	virtual void set_cursor_position(int p_line, int p_column) = 0;
+	virtual void set_multiline_mode(bool p_state) = 0;
+	virtual bool is_past_cursor() const = 0;
+	virtual void push_expression_indented_block() = 0; // For lambdas, or blocks inside expressions.
+	virtual void pop_expression_indented_block() = 0; // For lambdas, or blocks inside expressions.
+	virtual bool is_text() = 0;
+
+	virtual Token scan() = 0;
+
+	virtual ~GDScriptTokenizer() {}
+};
+
+class GDScriptTokenizerText : public GDScriptTokenizer {
 	String source;
 	const char32_t *_source = nullptr;
 	const char32_t *_current = nullptr;
@@ -235,6 +248,7 @@ private:
 	char32_t indent_char = '\0';
 	int position = 0;
 	int length = 0;
+	Vector<int> continuation_lines;
 #ifdef DEBUG_ENABLED
 	Vector<String> keyword_list;
 #endif // DEBUG_ENABLED
@@ -275,20 +289,28 @@ private:
 	Token annotation();
 
 public:
-	Token scan();
-
 	void set_source_code(const String &p_source_code);
 
-	int get_cursor_line() const;
-	int get_cursor_column() const;
-	void set_cursor_position(int p_line, int p_column);
-	void set_multiline_mode(bool p_state);
-	bool is_past_cursor() const;
-	static String get_token_name(Token::Type p_token_type);
-	void push_expression_indented_block(); // For lambdas, or blocks inside expressions.
-	void pop_expression_indented_block(); // For lambdas, or blocks inside expressions.
+	const Vector<int> &get_continuation_lines() const { return continuation_lines; }
+
+	virtual int get_cursor_line() const override;
+	virtual int get_cursor_column() const override;
+	virtual void set_cursor_position(int p_line, int p_column) override;
+	virtual void set_multiline_mode(bool p_state) override;
+	virtual bool is_past_cursor() const override;
+	virtual void push_expression_indented_block() override; // For lambdas, or blocks inside expressions.
+	virtual void pop_expression_indented_block() override; // For lambdas, or blocks inside expressions.
+	virtual bool is_text() override { return true; }
+
+#ifdef TOOLS_ENABLED
+	virtual const HashMap<int, CommentData> &get_comments() const override {
+		return comments;
+	}
+#endif // TOOLS_ENABLED
+
+	virtual Token scan() override;
 
-	GDScriptTokenizer();
+	GDScriptTokenizerText();
 };
 
 #endif // GDSCRIPT_TOKENIZER_H