Clean up Shader Preprocessor

* Moved preprocessor to Shader and ShaderInclude
* Clean up RenderingServer side
* Preprocessor is separate from parser now, but it emits tokens with include location hints.
* Improved ShaderEditor validation code
* Added include file code completion
* Added notification for all files affected by a broken include.

1 files changed, 43 insertions, 164 deletions

diff --git a/servers/rendering/shader_language.cpp b/servers/rendering/shader_language.cpp
index f4de32d14b..d2b80fb277 100644
--- a/servers/rendering/shader_language.cpp
+++ b/servers/rendering/shader_language.cpp
@@ -32,8 +32,8 @@
 
 #include "core/os/os.h"
 #include "core/string/print_string.h"
+#include "core/templates/local_vector.h"
 #include "servers/rendering_server.h"
-#include "shader_preprocessor.h"
 
 #define HAS_WARNING(flag) (warning_flags & flag)
 
@@ -575,6 +575,37 @@ ShaderLanguage::Token ShaderLanguage::_get_token() {
 
 				return _make_token(TK_OP_MOD);
 			} break;
+			case '@': {
+				if (GETCHAR(0) == '@' && GETCHAR(1) == '>') {
+					char_idx += 2;
+
+					LocalVector<char32_t> incp;
+					while (GETCHAR(0) != '\n') {
+						incp.push_back(GETCHAR(0));
+						char_idx++;
+					}
+					incp.push_back(0); // Zero end it.
+					String include_path(incp.ptr());
+					include_positions.write[include_positions.size() - 1].line = tk_line;
+					FilePosition fp;
+					fp.file = include_path;
+					fp.line = 0;
+					tk_line = 0;
+					include_positions.push_back(fp);
+
+				} else if (GETCHAR(0) == '@' && GETCHAR(1) == '<') {
+					if (include_positions.size() == 1) {
+						return _make_token(TK_ERROR, "Invalid include exit hint @@< without matching enter hint.");
+					}
+					char_idx += 2;
+
+					include_positions.resize(include_positions.size() - 1); // Pop back.
+					tk_line = include_positions[include_positions.size() - 1].line; // Restore line.
+
+				} else {
+					return _make_token(TK_ERROR, "Invalid include enter/exit hint token (@@> and @@<)");
+				}
+			} break;
 			default: {
 				char_idx--; //go back one, since we have no idea what this is
 
@@ -1123,6 +1154,9 @@ void ShaderLanguage::clear() {
 	completion_base = TYPE_VOID;
 	completion_base_array = false;
 
+	include_positions.clear();
+	include_positions.push_back(FilePosition());
+
 #ifdef DEBUG_ENABLED
 	keyword_completion_context = CF_GLOBAL_SPACE;
 	used_constants.clear();
@@ -4119,10 +4153,6 @@ void ShaderLanguage::get_keyword_list(List<String> *r_keywords) {
 	}
 }
 
-void ShaderLanguage::get_preprocessor_keyword_list(List<String> *r_keywords, bool p_include_shader_keywords) {
-	ShaderPreprocessor::get_keyword_list(r_keywords, p_include_shader_keywords);
-}
-
 bool ShaderLanguage::is_control_flow_keyword(String p_keyword) {
 	return p_keyword == "break" ||
 			p_keyword == "case" ||
@@ -7682,27 +7712,6 @@ Error ShaderLanguage::_validate_precision(DataType p_type, DataPrecision p_preci
 	return OK;
 }
 
-Error ShaderLanguage::_preprocess_shader(const String &p_code, String &r_result, int *r_completion_type) {
-	Error error = OK;
-
-	ShaderPreprocessor processor(p_code);
-	processor.preprocess(r_result);
-
-	ShaderPreprocessor::State *state = processor.get_state();
-	if (!state->error.is_empty()) {
-		error_line = state->error_line;
-		error_set = true;
-		error_str = state->error;
-		error = FAILED;
-	}
-
-	if (r_completion_type != nullptr) {
-		*r_completion_type = (int)state->completion_type;
-	}
-
-	return error;
-}
-
 Error ShaderLanguage::_parse_shader(const HashMap<StringName, FunctionInfo> &p_functions, const Vector<ModeInfo> &p_render_modes, const HashSet<String> &p_shader_types, bool p_is_include) {
 	Token tk;
 	TkPos prev_pos;
@@ -9500,97 +9509,6 @@ String ShaderLanguage::get_shader_type(const String &p_code) {
 	return String();
 }
 
-void ShaderLanguage::get_shader_dependencies(const String &p_code, HashSet<Ref<ShaderInclude>> *r_dependencies) {
-	bool reading_inc = false;
-	String cur_identifier;
-
-	for (int i = _get_first_ident_pos(p_code); i < p_code.length(); i++) {
-		if (p_code[i] == ';') {
-			continue;
-
-		} else if (p_code[i] <= 32) {
-			if (cur_identifier == "#include") {
-				reading_inc = true;
-				cur_identifier = String();
-			} else {
-				if (reading_inc) {
-					String path = cur_identifier;
-					if (path.begins_with("\"") && path.ends_with("\"")) {
-						path = path.substr(1, path.length() - 2);
-						if (!path.begins_with("res://")) {
-							path = path.insert(0, "res://");
-						}
-						Ref<ShaderInclude> inc = ResourceLoader::load(path);
-						if (inc.is_valid()) {
-							r_dependencies->insert(inc);
-						}
-					}
-					reading_inc = false;
-				}
-			}
-		} else {
-			cur_identifier += String::chr(p_code[i]);
-		}
-	}
-}
-
-String ShaderLanguage::get_shader_type_and_dependencies(const String &p_code, HashSet<Ref<ShaderInclude>> *r_dependencies) {
-	bool read_type = true;
-	bool reading_type = false;
-	bool reading_inc = false;
-	String type;
-
-	String cur_identifier;
-
-	for (int i = _get_first_ident_pos(p_code); i < p_code.length(); i++) {
-		if (p_code[i] == ';') {
-			continue;
-
-		} else if (p_code[i] <= 32) {
-			if (!cur_identifier.is_empty()) {
-				if (read_type) {
-					if (!reading_type) {
-						if (cur_identifier == "shader_type") {
-							reading_type = true;
-							cur_identifier = String();
-						}
-					} else {
-						type = cur_identifier;
-						read_type = false;
-						cur_identifier = String();
-					}
-				} else if (cur_identifier == "#include") {
-					reading_inc = true;
-					cur_identifier = String();
-				} else {
-					if (reading_inc) {
-						String path = cur_identifier;
-						if (path.begins_with("\"") && path.ends_with("\"")) {
-							path = path.substr(1, path.length() - 2);
-							if (!path.begins_with("res://")) {
-								path = path.insert(0, "res://");
-							}
-							Ref<ShaderInclude> inc = ResourceLoader::load(path);
-							if (inc.is_valid()) {
-								r_dependencies->insert(inc);
-							}
-						}
-						reading_inc = false;
-					}
-				}
-			}
-		} else {
-			cur_identifier += String::chr(p_code[i]);
-		}
-	}
-
-	if (reading_type) {
-		return type;
-	}
-
-	return String();
-}
-
 #ifdef DEBUG_ENABLED
 void ShaderLanguage::_check_warning_accums() {
 	for (const KeyValue<ShaderWarning::Code, HashMap<StringName, HashMap<StringName, Usage>> *> &E : warnings_check_map2) {
@@ -9630,21 +9548,15 @@ uint32_t ShaderLanguage::get_warning_flags() const {
 Error ShaderLanguage::compile(const String &p_code, const ShaderCompileInfo &p_info) {
 	clear();
 
-	Error err = _preprocess_shader(p_code, code);
-	if (err != OK) {
-		return err;
-	}
-
-	// Clear after preprocessing. Because preprocess uses the resource loader, it means if this instance is held in a singleton, it can have a changed state after.
-	clear();
-
+	code = p_code;
 	global_var_get_type_func = p_info.global_variable_type_func;
+
 	varying_function_names = p_info.varying_function_names;
 
 	nodes = nullptr;
 
 	shader = alloc_node<ShaderNode>();
-	err = _parse_shader(p_info.functions, p_info.render_modes, p_info.shader_types, p_info.is_include);
+	Error err = _parse_shader(p_info.functions, p_info.render_modes, p_info.shader_types, p_info.is_include);
 
 #ifdef DEBUG_ENABLED
 	if (check_warnings) {
@@ -9661,44 +9573,7 @@ Error ShaderLanguage::compile(const String &p_code, const ShaderCompileInfo &p_i
 Error ShaderLanguage::complete(const String &p_code, const ShaderCompileInfo &p_info, List<ScriptLanguage::CodeCompletionOption> *r_options, String &r_call_hint) {
 	clear();
 
-	int preprocessor_completion_type;
-	Error error = _preprocess_shader(p_code, code, &preprocessor_completion_type);
-
-	switch (preprocessor_completion_type) {
-		case ShaderPreprocessor::COMPLETION_TYPE_DIRECTIVE: {
-			static List<String> options;
-
-			if (options.is_empty()) {
-				ShaderPreprocessor::get_keyword_list(&options, true);
-			}
-
-			for (const String &E : options) {
-				ScriptLanguage::CodeCompletionOption option(E, ScriptLanguage::CODE_COMPLETION_KIND_PLAIN_TEXT);
-				r_options->push_back(option);
-			}
-
-			return OK;
-		} break;
-		case ShaderPreprocessor::COMPLETION_TYPE_PRAGMA: {
-			static List<String> options;
-
-			if (options.is_empty()) {
-				ShaderPreprocessor::get_pragma_list(&options);
-			}
-
-			for (const String &E : options) {
-				ScriptLanguage::CodeCompletionOption option(E, ScriptLanguage::CODE_COMPLETION_KIND_PLAIN_TEXT);
-				r_options->push_back(option);
-			}
-
-			return OK;
-		} break;
-	}
-
-	if (error != OK) {
-		return error;
-	}
-
+	code = p_code;
 	varying_function_names = p_info.varying_function_names;
 
 	nodes = nullptr;
@@ -10231,6 +10106,10 @@ String ShaderLanguage::get_error_text() {
 	return error_str;
 }
 
+Vector<ShaderLanguage::FilePosition> ShaderLanguage::get_include_positions() {
+	return include_positions;
+}
+
 int ShaderLanguage::get_error_line() {
 	return error_line;
 }