Merge pull request #72855 from RandomShaper/cache_help

Cache classes editor help (a.k.a. faster editor startup)

4 files changed, 429 insertions, 6 deletions

diff --git a/core/doc_data.h b/core/doc_data.h
index c503a4e0d6..d064818cd5 100644
--- a/core/doc_data.h
+++ b/core/doc_data.h
@@ -78,6 +78,27 @@ public:
 
 			return doc;
 		}
+		static Dictionary to_dict(const ArgumentDoc &p_doc) {
+			Dictionary dict;
+
+			if (!p_doc.name.is_empty()) {
+				dict["name"] = p_doc.name;
+			}
+
+			if (!p_doc.type.is_empty()) {
+				dict["type"] = p_doc.type;
+			}
+
+			if (!p_doc.enumeration.is_empty()) {
+				dict["enumeration"] = p_doc.enumeration;
+			}
+
+			if (!p_doc.default_value.is_empty()) {
+				dict["default_value"] = p_doc.default_value;
+			}
+
+			return dict;
+		}
 	};
 
 	struct MethodDoc {
@@ -167,6 +188,51 @@ public:
 
 			return doc;
 		}
+		static Dictionary to_dict(const MethodDoc &p_doc) {
+			Dictionary dict;
+
+			if (!p_doc.name.is_empty()) {
+				dict["name"] = p_doc.name;
+			}
+
+			if (!p_doc.return_type.is_empty()) {
+				dict["return_type"] = p_doc.return_type;
+			}
+
+			if (!p_doc.return_enum.is_empty()) {
+				dict["return_enum"] = p_doc.return_enum;
+			}
+
+			if (!p_doc.qualifiers.is_empty()) {
+				dict["qualifiers"] = p_doc.qualifiers;
+			}
+
+			if (!p_doc.description.is_empty()) {
+				dict["description"] = p_doc.description;
+			}
+
+			dict["is_deprecated"] = p_doc.is_deprecated;
+
+			dict["is_experimental"] = p_doc.is_experimental;
+
+			if (!p_doc.arguments.is_empty()) {
+				Array arguments;
+				for (int i = 0; i < p_doc.arguments.size(); i++) {
+					arguments.push_back(ArgumentDoc::to_dict(p_doc.arguments[i]));
+				}
+				dict["arguments"] = arguments;
+			}
+
+			if (!p_doc.errors_returned.is_empty()) {
+				Array errors_returned;
+				for (int i = 0; i < p_doc.errors_returned.size(); i++) {
+					errors_returned.push_back(p_doc.errors_returned[i]);
+				}
+				dict["errors_returned"] = errors_returned;
+			}
+
+			return dict;
+		}
 	};
 
 	struct ConstantDoc {
@@ -218,6 +284,35 @@ public:
 
 			return doc;
 		}
+		static Dictionary to_dict(const ConstantDoc &p_doc) {
+			Dictionary dict;
+
+			if (!p_doc.name.is_empty()) {
+				dict["name"] = p_doc.name;
+			}
+
+			if (!p_doc.value.is_empty()) {
+				dict["value"] = p_doc.value;
+			}
+
+			dict["is_value_valid"] = p_doc.is_value_valid;
+
+			if (!p_doc.enumeration.is_empty()) {
+				dict["enumeration"] = p_doc.enumeration;
+			}
+
+			dict["is_bitfield"] = p_doc.is_bitfield;
+
+			if (!p_doc.description.is_empty()) {
+				dict["description"] = p_doc.description;
+			}
+
+			dict["is_deprecated"] = p_doc.is_deprecated;
+
+			dict["is_experimental"] = p_doc.is_experimental;
+
+			return dict;
+		}
 	};
 
 	struct EnumDoc {
@@ -250,6 +345,29 @@ public:
 
 			return doc;
 		}
+		static Dictionary to_dict(const EnumDoc &p_doc) {
+			Dictionary dict;
+
+			if (!p_doc.name.is_empty()) {
+				dict["name"] = p_doc.name;
+			}
+
+			dict["is_bitfield"] = p_doc.is_bitfield;
+
+			if (!p_doc.description.is_empty()) {
+				dict["description"] = p_doc.description;
+			}
+
+			if (!p_doc.values.is_empty()) {
+				Array values;
+				for (int i = 0; i < p_doc.values.size(); i++) {
+					values.push_back(ConstantDoc::to_dict(p_doc.values[i]));
+				}
+				dict["values"] = values;
+			}
+
+			return dict;
+		}
 	};
 
 	struct PropertyDoc {
@@ -315,6 +433,49 @@ public:
 
 			return doc;
 		}
+		static Dictionary to_dict(const PropertyDoc &p_doc) {
+			Dictionary dict;
+
+			if (!p_doc.name.is_empty()) {
+				dict["name"] = p_doc.name;
+			}
+
+			if (!p_doc.type.is_empty()) {
+				dict["type"] = p_doc.type;
+			}
+
+			if (!p_doc.enumeration.is_empty()) {
+				dict["enumeration"] = p_doc.enumeration;
+			}
+
+			if (!p_doc.description.is_empty()) {
+				dict["description"] = p_doc.description;
+			}
+
+			if (!p_doc.setter.is_empty()) {
+				dict["setter"] = p_doc.setter;
+			}
+
+			if (!p_doc.getter.is_empty()) {
+				dict["getter"] = p_doc.getter;
+			}
+
+			if (!p_doc.default_value.is_empty()) {
+				dict["default_value"] = p_doc.default_value;
+			}
+
+			dict["overridden"] = p_doc.overridden;
+
+			if (!p_doc.overrides.is_empty()) {
+				dict["overrides"] = p_doc.overrides;
+			}
+
+			dict["is_deprecated"] = p_doc.is_deprecated;
+
+			dict["is_experimental"] = p_doc.is_experimental;
+
+			return dict;
+		}
 	};
 
 	struct ThemeItemDoc {
@@ -355,6 +516,31 @@ public:
 
 			return doc;
 		}
+		static Dictionary to_dict(const ThemeItemDoc &p_doc) {
+			Dictionary dict;
+
+			if (!p_doc.name.is_empty()) {
+				dict["name"] = p_doc.name;
+			}
+
+			if (!p_doc.type.is_empty()) {
+				dict["type"] = p_doc.type;
+			}
+
+			if (!p_doc.data_type.is_empty()) {
+				dict["data_type"] = p_doc.data_type;
+			}
+
+			if (!p_doc.description.is_empty()) {
+				dict["description"] = p_doc.description;
+			}
+
+			if (!p_doc.default_value.is_empty()) {
+				dict["default_value"] = p_doc.default_value;
+			}
+
+			return dict;
+		}
 	};
 
 	struct TutorialDoc {
@@ -373,6 +559,19 @@ public:
 
 			return doc;
 		}
+		static Dictionary to_dict(const TutorialDoc &p_doc) {
+			Dictionary dict;
+
+			if (!p_doc.link.is_empty()) {
+				dict["link"] = p_doc.link;
+			}
+
+			if (!p_doc.title.is_empty()) {
+				dict["title"] = p_doc.title;
+			}
+
+			return dict;
+		}
 	};
 
 	struct ClassDoc {
@@ -514,6 +713,117 @@ public:
 
 			return doc;
 		}
+		static Dictionary to_dict(const ClassDoc &p_doc) {
+			Dictionary dict;
+
+			if (!p_doc.name.is_empty()) {
+				dict["name"] = p_doc.name;
+			}
+
+			if (!p_doc.inherits.is_empty()) {
+				dict["inherits"] = p_doc.inherits;
+			}
+
+			if (!p_doc.brief_description.is_empty()) {
+				dict["brief_description"] = p_doc.brief_description;
+			}
+
+			if (!p_doc.description.is_empty()) {
+				dict["description"] = p_doc.description;
+			}
+
+			if (!p_doc.tutorials.is_empty()) {
+				Array tutorials;
+				for (int i = 0; i < p_doc.tutorials.size(); i++) {
+					tutorials.push_back(TutorialDoc::to_dict(p_doc.tutorials[i]));
+				}
+				dict["tutorials"] = tutorials;
+			}
+
+			if (!p_doc.constructors.is_empty()) {
+				Array constructors;
+				for (int i = 0; i < p_doc.constructors.size(); i++) {
+					constructors.push_back(MethodDoc::to_dict(p_doc.constructors[i]));
+				}
+				dict["constructors"] = constructors;
+			}
+
+			if (!p_doc.methods.is_empty()) {
+				Array methods;
+				for (int i = 0; i < p_doc.methods.size(); i++) {
+					methods.push_back(MethodDoc::to_dict(p_doc.methods[i]));
+				}
+				dict["methods"] = methods;
+			}
+
+			if (!p_doc.operators.is_empty()) {
+				Array operators;
+				for (int i = 0; i < p_doc.operators.size(); i++) {
+					operators.push_back(MethodDoc::to_dict(p_doc.operators[i]));
+				}
+				dict["operators"] = operators;
+			}
+
+			if (!p_doc.signals.is_empty()) {
+				Array signals;
+				for (int i = 0; i < p_doc.signals.size(); i++) {
+					signals.push_back(MethodDoc::to_dict(p_doc.signals[i]));
+				}
+				dict["signals"] = signals;
+			}
+
+			if (!p_doc.constants.is_empty()) {
+				Array constants;
+				for (int i = 0; i < p_doc.constants.size(); i++) {
+					constants.push_back(ConstantDoc::to_dict(p_doc.constants[i]));
+				}
+				dict["constants"] = constants;
+			}
+
+			if (!p_doc.enums.is_empty()) {
+				Dictionary enums;
+				for (const KeyValue<String, String> &E : p_doc.enums) {
+					enums[E.key] = E.value;
+				}
+				dict["enums"] = enums;
+			}
+
+			if (!p_doc.properties.is_empty()) {
+				Array properties;
+				for (int i = 0; i < p_doc.properties.size(); i++) {
+					properties.push_back(PropertyDoc::to_dict(p_doc.properties[i]));
+				}
+				dict["properties"] = properties;
+			}
+
+			if (!p_doc.annotations.is_empty()) {
+				Array annotations;
+				for (int i = 0; i < p_doc.annotations.size(); i++) {
+					annotations.push_back(MethodDoc::to_dict(p_doc.annotations[i]));
+				}
+				dict["annotations"] = annotations;
+			}
+
+			if (!p_doc.theme_properties.is_empty()) {
+				Array theme_properties;
+				for (int i = 0; i < p_doc.theme_properties.size(); i++) {
+					theme_properties.push_back(ThemeItemDoc::to_dict(p_doc.theme_properties[i]));
+				}
+				dict["theme_properties"] = theme_properties;
+			}
+
+			dict["is_deprecated"] = p_doc.is_deprecated;
+
+			dict["is_experimental"] = p_doc.is_experimental;
+
+			dict["is_script_doc"] = p_doc.is_script_doc;
+
+			if (!p_doc.script_path.is_empty()) {
+				dict["script_path"] = p_doc.script_path;
+			}
+
+			return dict;
+		}
 	};
 
 	static String get_default_value_string(const Variant &p_value);
diff --git a/editor/editor_help.cpp b/editor/editor_help.cpp
index 61dabb9541..a9dc8dfee6 100644
--- a/editor/editor_help.cpp
+++ b/editor/editor_help.cpp
@@ -33,9 +33,11 @@
 #include "core/core_constants.h"
 #include "core/input/input.h"
 #include "core/os/keyboard.h"
+#include "core/version.h"
 #include "core/version_generated.gen.h"
 #include "doc_data_compressed.gen.h"
 #include "editor/editor_node.h"
+#include "editor/editor_paths.h"
 #include "editor/editor_scale.h"
 #include "editor/editor_settings.h"
 #include "editor/plugins/script_editor_plugin.h"
@@ -45,6 +47,33 @@
 
 DocTools *EditorHelp::doc = nullptr;
 
+class DocCache : public Resource {
+	GDCLASS(DocCache, Resource);
+	RES_BASE_EXTENSION("doc_cache");
+
+	String version_hash;
+	Array classes;
+
+protected:
+	static void _bind_methods() {
+		ClassDB::bind_method(D_METHOD("set_version_hash", "version_hash"), &DocCache::set_version_hash);
+		ClassDB::bind_method(D_METHOD("get_version_hash"), &DocCache::get_version_hash);
+
+		ClassDB::bind_method(D_METHOD("set_classes", "classes"), &DocCache::set_classes);
+		ClassDB::bind_method(D_METHOD("get_classes"), &DocCache::get_classes);
+
+		ADD_PROPERTY(PropertyInfo(Variant::STRING, "version_hash"), "set_version_hash", "get_version_hash");
+		ADD_PROPERTY(PropertyInfo(Variant::DICTIONARY, "classes"), "set_classes", "get_classes");
+	}
+
+public:
+	String get_version_hash() const { return version_hash; }
+	void set_version_hash(const String &p_version_hash) { version_hash = p_version_hash; }
+
+	Array get_classes() const { return classes; }
+	void set_classes(const Array &p_classes) { classes = p_classes; }
+};
+
 void EditorHelp::_update_theme_item_cache() {
 	VBoxContainer::_update_theme_item_cache();
 
@@ -2168,23 +2197,98 @@ void EditorHelp::_wait_for_thread() {
 	}
 }
 
+String EditorHelp::get_cache_full_path() {
+	return EditorPaths::get_singleton()->get_cache_dir().path_join("editor.doc_cache");
+}
+
+static bool first_attempt = true;
+static List<StringName> classes_whitelist;
+
+void EditorHelp::_load_doc_thread(void *p_udata) {
+	DEV_ASSERT(first_attempt);
+	Ref<DocCache> cache_res = ResourceLoader::load(get_cache_full_path());
+	if (cache_res.is_valid() && cache_res->get_version_hash() == String(VERSION_HASH)) {
+		for (int i = 0; i < cache_res->get_classes().size(); i++) {
+			doc->add_doc(DocData::ClassDoc::from_dict(cache_res->get_classes()[i]));
+		}
+		classes_whitelist.clear();
+	} else {
+		// We have to go back to the main thread to start from scratch.
+		first_attempt = false;
+		callable_mp_static(&EditorHelp::generate_doc).call_deferred();
+	}
+}
+
 void EditorHelp::_gen_doc_thread(void *p_udata) {
 	DocTools compdoc;
 	compdoc.load_compressed(_doc_data_compressed, _doc_data_compressed_size, _doc_data_uncompressed_size);
 	doc->merge_from(compdoc); // Ensure all is up to date.
+
+	Ref<DocCache> cache_res;
+	cache_res.instantiate();
+	cache_res->set_version_hash(VERSION_HASH);
+	Array classes;
+	for (const KeyValue<String, DocData::ClassDoc> &E : doc->class_list) {
+		classes.push_back(DocData::ClassDoc::to_dict(E.value));
+	}
+	cache_res->set_classes(classes);
+	Error err = ResourceSaver::save(cache_res, get_cache_full_path(), ResourceSaver::FLAG_COMPRESS);
+	if (err) {
+		ERR_PRINT("Cannot save editor help cache (" + get_cache_full_path() + ").");
+	}
 }
 
 static bool doc_gen_use_threads = true;
 
 void EditorHelp::generate_doc() {
-	doc = memnew(DocTools);
-	// Not doable on threads unfortunately, since it instantiates all sorts of classes to get default values.
-	doc->generate(true);
-
 	if (doc_gen_use_threads) {
-		thread.start(_gen_doc_thread, nullptr);
+		// In case not the first attempt.
+		_wait_for_thread();
+	}
+
+	DEV_ASSERT(first_attempt == (doc == nullptr));
+
+	if (!doc) {
+		// Classes registered after this point should not have documentation generated.
+		ClassDB::get_class_list(&classes_whitelist);
+
+		GDREGISTER_CLASS(DocCache);
+		doc = memnew(DocTools);
+	}
+
+	if (first_attempt && FileAccess::exists(get_cache_full_path())) {
+		if (doc_gen_use_threads) {
+			thread.start(_load_doc_thread, nullptr);
+		} else {
+			_load_doc_thread(nullptr);
+		}
 	} else {
-		_gen_doc_thread(nullptr);
+		print_verbose("Regenerating editor help cache");
+
+		if (!first_attempt) {
+			// Some classes that should not be exposed may have been registered by now. Unexpose them.
+			// Arduous, but happens only when regenerating.
+			List<StringName> current_classes;
+			ClassDB::get_class_list(&current_classes);
+			List<StringName>::Element *W = classes_whitelist.front();
+			for (const StringName &name : current_classes) {
+				if (W && W->get() == name) {
+					W = W->next();
+				} else {
+					ClassDB::classes[name].exposed = false;
+				}
+			}
+		}
+		classes_whitelist.clear();
+
+		// Not doable on threads unfortunately, since it instantiates all sorts of classes to get default values.
+		doc->generate(true);
+
+		if (doc_gen_use_threads) {
+			thread.start(_gen_doc_thread, nullptr);
+		} else {
+			_gen_doc_thread(nullptr);
+		}
 	}
 }
 
diff --git a/editor/editor_help.h b/editor/editor_help.h
index a23dbca213..f2967027a0 100644
--- a/editor/editor_help.h
+++ b/editor/editor_help.h
@@ -193,7 +193,9 @@ class EditorHelp : public VBoxContainer {
 	static Thread thread;
 
 	static void _wait_for_thread();
+	static void _load_doc_thread(void *p_udata);
 	static void _gen_doc_thread(void *p_udata);
+	static void _generate_doc_first_step();
 
 protected:
 	virtual void _update_theme_item_cache() override;
@@ -205,6 +207,7 @@ public:
 	static void generate_doc();
 	static DocTools *get_doc_data();
 	static void cleanup_doc();
+	static String get_cache_full_path();
 
 	void go_to_help(const String &p_help);
 	void go_to_class(const String &p_class, int p_scroll = 0);
diff --git a/main/main.cpp b/main/main.cpp
index fbd0b75e58..3619815825 100644
--- a/main/main.cpp
+++ b/main/main.cpp
@@ -89,6 +89,7 @@
 #include "editor/debugger/editor_debugger_node.h"
 #include "editor/doc_data_class_path.gen.h"
 #include "editor/doc_tools.h"
+#include "editor/editor_help.h"
 #include "editor/editor_node.h"
 #include "editor/editor_paths.h"
 #include "editor/editor_settings.h"
@@ -2576,6 +2577,11 @@ bool Main::start() {
 		err = doc.save_classes(index_path, doc_data_classes);
 		ERR_FAIL_COND_V_MSG(err != OK, false, "Error saving new docs:" + itos(err));
 
+		print_line("Deleting docs cache...");
+		if (FileAccess::exists(EditorHelp::get_cache_full_path())) {
+			DirAccess::remove_file_or_error(EditorHelp::get_cache_full_path());
+		}
+
 		OS::get_singleton()->set_exit_code(EXIT_SUCCESS);
 		return false;
 	}