Improvements to GDScript identifier tokenization

1 files changed, 19 insertions, 13 deletions

diff --git a/modules/gdscript/gdscript_tokenizer.cpp b/modules/gdscript/gdscript_tokenizer.cpp
index a45a73a8d5..9455a4c0a6 100644
--- a/modules/gdscript/gdscript_tokenizer.cpp
+++ b/modules/gdscript/gdscript_tokenizer.cpp
@@ -559,6 +559,24 @@ GDScriptTokenizer::Token GDScriptTokenizer::potential_identifier() {
 		return make_identifier(name);
 	}
 
+	if (!only_ascii) {
+		// Kept here in case the order with push_error matters.
+		Token id = make_identifier(name);
+
+#ifdef DEBUG_ENABLED
+		// Additional checks for identifiers but only in debug and if it's available in TextServer.
+		if (TS->has_feature(TextServer::FEATURE_UNICODE_SECURITY)) {
+			int64_t confusable = TS->is_confusable(name, keyword_list);
+			if (confusable >= 0) {
+				push_error(vformat(R"(Identifier "%s" is visually similar to the GDScript keyword "%s" and thus not allowed.)", name, keyword_list[confusable]));
+			}
+		}
+#endif // DEBUG_ENABLED
+
+		// Cannot be a keyword, as keywords are ASCII only.
+		return id;
+	}
+
 	// Define some helper macros for the switch case.
 #define KEYWORD_GROUP_CASE(char) \
 	break;                       \
@@ -594,19 +612,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::potential_identifier() {
 	}
 
 	// Not a keyword, so must be an identifier.
-	Token id = make_identifier(name);
-
-#ifdef DEBUG_ENABLED
-	// Additional checks for identifiers but only in debug and if it's available in TextServer.
-	if (!only_ascii && TS->has_feature(TextServer::FEATURE_UNICODE_SECURITY)) {
-		int64_t confusable = TS->is_confusable(name, keyword_list);
-		if (confusable >= 0) {
-			push_error(vformat(R"(Identifier "%s" is visually similar to the GDScript keyword "%s" and thus not allowed.)", name, keyword_list[confusable]));
-		}
-	}
-#endif // DEBUG_ENABLED
-
-	return id;
+	return make_identifier(name);
 
 #undef KEYWORD_GROUP_CASE
 #undef KEYWORD