Support threads in the script debugger

* This implementation adds threads on the side of the client (script debugger).
* Some functions of the debugger are optimized.
* The profile is also now thread safe using atomics.
* The editor can switch between multiple threads when debugging.

This PR adds threaded support for the script language debugger. Every thread has its own thread local data and it will connect to the debugger using multiple thread IDs.
This means that, now, the editor can receive multiple threads entering debug mode at the same time.

3 files changed, 169 insertions, 77 deletions

diff --git a/editor/debugger/editor_debugger_inspector.cpp b/editor/debugger/editor_debugger_inspector.cpp
index e083e1746d..1e9b7c2c60 100644
--- a/editor/debugger/editor_debugger_inspector.cpp
+++ b/editor/debugger/editor_debugger_inspector.cpp
@@ -232,7 +232,7 @@ void EditorDebuggerInspector::add_stack_variable(const Array &p_array) {
 	PropertyHint h = PROPERTY_HINT_NONE;
 	String hs;
 
-	if (v.get_type() == Variant::OBJECT) {
+	if (var.var_type == Variant::OBJECT) {
 		v = Object::cast_to<EncodedObjectAsID>(v)->get_object_id();
 		h = PROPERTY_HINT_OBJECT_ID;
 		hs = "Object";
diff --git a/editor/debugger/script_editor_debugger.cpp b/editor/debugger/script_editor_debugger.cpp
index 8985387043..2c40f0e120 100644
--- a/editor/debugger/script_editor_debugger.cpp
+++ b/editor/debugger/script_editor_debugger.cpp
@@ -71,10 +71,12 @@
 
 using CameraOverride = EditorDebuggerNode::CameraOverride;
 
-void ScriptEditorDebugger::_put_msg(String p_message, Array p_data) {
+void ScriptEditorDebugger::_put_msg(String p_message, Array p_data, uint64_t p_thread_id) {
+	ERR_FAIL_COND(p_thread_id == Thread::UNASSIGNED_ID);
 	if (is_session_active()) {
 		Array msg;
 		msg.push_back(p_message);
+		msg.push_back(p_thread_id);
 		msg.push_back(p_data);
 		peer->put_message(msg);
 	}
@@ -98,31 +100,31 @@ void ScriptEditorDebugger::debug_skip_breakpoints() {
 
 	Array msg;
 	msg.push_back(skip_breakpoints_value);
-	_put_msg("set_skip_breakpoints", msg);
+	_put_msg("set_skip_breakpoints", msg, debugging_thread_id != Thread::UNASSIGNED_ID ? debugging_thread_id : Thread::MAIN_ID);
 }
 
 void ScriptEditorDebugger::debug_next() {
-	ERR_FAIL_COND(!breaked);
+	ERR_FAIL_COND(!is_breaked());
 
-	_put_msg("next", Array());
+	_put_msg("next", Array(), debugging_thread_id);
 	_clear_execution();
 }
 
 void ScriptEditorDebugger::debug_step() {
-	ERR_FAIL_COND(!breaked);
+	ERR_FAIL_COND(!is_breaked());
 
-	_put_msg("step", Array());
+	_put_msg("step", Array(), debugging_thread_id);
 	_clear_execution();
 }
 
 void ScriptEditorDebugger::debug_break() {
-	ERR_FAIL_COND(breaked);
+	ERR_FAIL_COND(is_breaked());
 
 	_put_msg("break", Array());
 }
 
 void ScriptEditorDebugger::debug_continue() {
-	ERR_FAIL_COND(!breaked);
+	ERR_FAIL_COND(!is_breaked());
 
 	// Allow focus stealing only if we actually run this client for security.
 	if (remote_pid && EditorNode::get_singleton()->has_child_process(remote_pid)) {
@@ -130,7 +132,7 @@ void ScriptEditorDebugger::debug_continue() {
 	}
 
 	_clear_execution();
-	_put_msg("continue", Array());
+	_put_msg("continue", Array(), debugging_thread_id);
 	_put_msg("servers:foreground", Array());
 }
 
@@ -299,43 +301,89 @@ Size2 ScriptEditorDebugger::get_minimum_size() const {
 	return ms;
 }
 
-void ScriptEditorDebugger::_parse_message(const String &p_msg, const Array &p_data) {
+void ScriptEditorDebugger::_thread_debug_enter(uint64_t p_thread_id) {
+	ERR_FAIL_COND(!threads_debugged.has(p_thread_id));
+	ThreadDebugged &td = threads_debugged[p_thread_id];
+	_set_reason_text(td.error, MESSAGE_ERROR);
+	emit_signal(SNAME("breaked"), true, td.can_debug, td.error, td.has_stackdump);
+	if (!td.error.is_empty()) {
+		tabs->set_current_tab(0);
+	}
+	inspector->clear_cache(); // Take a chance to force remote objects update.
+	_put_msg("get_stack_dump", Array(), p_thread_id);
+}
+
+void ScriptEditorDebugger::_select_thread(int p_index) {
+	debugging_thread_id = threads->get_item_metadata(threads->get_selected());
+	_thread_debug_enter(debugging_thread_id);
+}
+
+void ScriptEditorDebugger::_parse_message(const String &p_msg, uint64_t p_thread_id, const Array &p_data) {
 	emit_signal(SNAME("debug_data"), p_msg, p_data);
 	if (p_msg == "debug_enter") {
-		_put_msg("get_stack_dump", Array());
-
-		ERR_FAIL_COND(p_data.size() != 3);
-		bool can_continue = p_data[0];
-		String error = p_data[1];
-		bool has_stackdump = p_data[2];
-		breaked = true;
-		can_request_idle_draw = true;
-		can_debug = can_continue;
-		_update_buttons_state();
-		_set_reason_text(error, MESSAGE_ERROR);
-		emit_signal(SNAME("breaked"), true, can_continue, error, has_stackdump);
-		if (is_move_to_foreground()) {
-			DisplayServer::get_singleton()->window_move_to_foreground();
-		}
-		if (!error.is_empty()) {
-			tabs->set_current_tab(0);
+		ERR_FAIL_COND(p_data.size() != 4);
+
+		ThreadDebugged td;
+		td.name = p_data[3];
+		td.error = p_data[1];
+		td.can_debug = p_data[0];
+		td.has_stackdump = p_data[2];
+		td.thread_id = p_thread_id;
+		static uint32_t order_inc = 0;
+		td.debug_order = order_inc++;
+
+		threads_debugged.insert(p_thread_id, td);
+
+		if (threads_debugged.size() == 1) {
+			// First thread that requests debug
+			debugging_thread_id = p_thread_id;
+			_thread_debug_enter(p_thread_id);
+			can_request_idle_draw = true;
+			if (is_move_to_foreground()) {
+				DisplayServer::get_singleton()->window_move_to_foreground();
+			}
+			profiler->set_enabled(false, false);
+			visual_profiler->set_enabled(false);
 		}
-		profiler->set_enabled(false, false);
-		visual_profiler->set_enabled(false);
-		inspector->clear_cache(); // Take a chance to force remote objects update.
+		_update_buttons_state();
 
 	} else if (p_msg == "debug_exit") {
-		breaked = false;
-		can_debug = false;
-		_clear_execution();
-		_update_buttons_state();
-		_set_reason_text(TTR("Execution resumed."), MESSAGE_SUCCESS);
-		emit_signal(SNAME("breaked"), false, false, "", false);
+		threads_debugged.erase(p_thread_id);
+		if (p_thread_id == debugging_thread_id) {
+			_clear_execution();
+			if (threads_debugged.size() == 0) {
+				debugging_thread_id = Thread::UNASSIGNED_ID;
+			} else {
+				// Find next thread to debug.
+				uint32_t min_order = 0xFFFFFFFF;
+				uint64_t next_thread = Thread::UNASSIGNED_ID;
+				for (KeyValue<uint64_t, ThreadDebugged> T : threads_debugged) {
+					if (T.value.debug_order < min_order) {
+						min_order = T.value.debug_order;
+						next_thread = T.key;
+					}
+				}
+
+				debugging_thread_id = next_thread;
+			}
 
-		profiler->set_enabled(true, false);
-		profiler->disable_seeking();
+			if (debugging_thread_id == Thread::UNASSIGNED_ID) {
+				// Nothing else to debug.
+				profiler->set_enabled(true, false);
+				profiler->disable_seeking();
 
-		visual_profiler->set_enabled(true);
+				visual_profiler->set_enabled(true);
+
+				_set_reason_text(TTR("Execution resumed."), MESSAGE_SUCCESS);
+				emit_signal(SNAME("breaked"), false, false, "", false);
+
+				_update_buttons_state();
+			} else {
+				_thread_debug_enter(debugging_thread_id);
+			}
+		} else {
+			_update_buttons_state();
+		}
 
 	} else if (p_msg == "set_pid") {
 		ERR_FAIL_COND(p_data.size() < 1);
@@ -379,7 +427,6 @@ void ScriptEditorDebugger::_parse_message(const String &p_msg, const Array &p_da
 
 		vmem_total->set_tooltip_text(TTR("Bytes:") + " " + itos(total));
 		vmem_total->set_text(String::humanize_size(total));
-
 	} else if (p_msg == "servers:drawn") {
 		can_request_idle_draw = true;
 	} else if (p_msg == "stack_dump") {
@@ -414,11 +461,9 @@ void ScriptEditorDebugger::_parse_message(const String &p_msg, const Array &p_da
 		inspector->clear_stack_variables();
 		ERR_FAIL_COND(p_data.size() != 1);
 		emit_signal(SNAME("stack_frame_vars"), p_data[0]);
-
 	} else if (p_msg == "stack_frame_var") {
 		inspector->add_stack_variable(p_data);
 		emit_signal(SNAME("stack_frame_var"), p_data);
-
 	} else if (p_msg == "output") {
 		ERR_FAIL_COND(p_data.size() != 2);
 
@@ -458,7 +503,6 @@ void ScriptEditorDebugger::_parse_message(const String &p_msg, const Array &p_da
 			frame_data.write[i] = p_data[i];
 		}
 		performance_profiler->add_profile_frame(frame_data);
-
 	} else if (p_msg == "visual:profile_frame") {
 		ServersDebugger::VisualProfilerFrame frame;
 		frame.deserialize(p_data);
@@ -477,7 +521,6 @@ void ScriptEditorDebugger::_parse_message(const String &p_msg, const Array &p_da
 			}
 		}
 		visual_profiler->add_frame_metric(metric);
-
 	} else if (p_msg == "error") {
 		DebuggerMarshalls::OutputError oe;
 		ERR_FAIL_COND_MSG(oe.deserialize(p_data) == false, "Failed to deserialize error message");
@@ -625,13 +668,11 @@ void ScriptEditorDebugger::_parse_message(const String &p_msg, const Array &p_da
 		} else {
 			error_count++;
 		}
-
 	} else if (p_msg == "servers:function_signature") {
 		// Cache a profiler signature.
 		ServersDebugger::ScriptFunctionSignature sig;
 		sig.deserialize(p_data);
 		profiler_signature[sig.id] = sig.name;
-
 	} else if (p_msg == "servers:profile_frame" || p_msg == "servers:profile_total") {
 		EditorProfiler::Metric metric;
 		ServersDebugger::ServersProfilerFrame frame;
@@ -744,11 +785,9 @@ void ScriptEditorDebugger::_parse_message(const String &p_msg, const Array &p_da
 		} else {
 			profiler->add_frame_metric(metric, true);
 		}
-
 	} else if (p_msg == "request_quit") {
 		emit_signal(SNAME("stop_requested"));
 		_stop_and_notify();
-
 	} else if (p_msg == "performance:profile_names") {
 		Vector<StringName> monitors;
 		monitors.resize(p_data.size());
@@ -757,13 +796,11 @@ void ScriptEditorDebugger::_parse_message(const String &p_msg, const Array &p_da
 			monitors.set(i, p_data[i]);
 		}
 		performance_profiler->update_monitors(monitors);
-
 	} else if (p_msg == "filesystem:update_file") {
 		ERR_FAIL_COND(p_data.size() < 1);
 		if (EditorFileSystem::get_singleton()) {
 			EditorFileSystem::get_singleton()->update_file(p_data[0]);
 		}
-
 	} else {
 		int colon_index = p_msg.find_char(':');
 		ERR_FAIL_COND_MSG(colon_index < 1, "Invalid message received");
@@ -878,7 +915,7 @@ void ScriptEditorDebugger::_notification(int p_what) {
 					msg.push_back(cam->get_far());
 					_put_msg("scene:override_camera_3D:transform", msg);
 				}
-				if (breaked && can_request_idle_draw) {
+				if (is_breaked() && can_request_idle_draw) {
 					_put_msg("servers:draw", Array());
 					can_request_idle_draw = false;
 				}
@@ -888,11 +925,12 @@ void ScriptEditorDebugger::_notification(int p_what) {
 
 			while (peer.is_valid() && peer->has_message()) {
 				Array arr = peer->get_message();
-				if (arr.size() != 2 || arr[0].get_type() != Variant::STRING || arr[1].get_type() != Variant::ARRAY) {
+				if (arr.size() != 3 || arr[0].get_type() != Variant::STRING || arr[1].get_type() != Variant::INT || arr[2].get_type() != Variant::ARRAY) {
 					_stop_and_notify();
 					ERR_FAIL_MSG("Invalid message format received from peer");
 				}
-				_parse_message(arr[0], arr[1]);
+
+				_parse_message(arr[0], arr[1], arr[2]);
 
 				if (OS::get_singleton()->get_ticks_msec() > until) {
 					break;
@@ -959,8 +997,6 @@ void ScriptEditorDebugger::start(Ref<RemoteDebuggerPeer> p_peer) {
 	performance_profiler->reset();
 
 	set_process(true);
-	breaked = false;
-	can_debug = true;
 	camera_override = CameraOverride::OVERRIDE_NONE;
 
 	tabs->set_current_tab(0);
@@ -973,13 +1009,35 @@ void ScriptEditorDebugger::_update_buttons_state() {
 	const bool active = is_session_active();
 	const bool has_editor_tree = active && editor_remote_tree && editor_remote_tree->get_selected();
 	vmem_refresh->set_disabled(!active);
-	step->set_disabled(!active || !breaked || !can_debug);
-	next->set_disabled(!active || !breaked || !can_debug);
-	copy->set_disabled(!active || !breaked);
-	docontinue->set_disabled(!active || !breaked);
-	dobreak->set_disabled(!active || breaked);
+	step->set_disabled(!active || !is_breaked() || !is_debuggable());
+	next->set_disabled(!active || !is_breaked() || !is_debuggable());
+	copy->set_disabled(!active || !is_breaked());
+	docontinue->set_disabled(!active || !is_breaked());
+	dobreak->set_disabled(!active || is_breaked());
 	le_clear->set_disabled(!active);
 	le_set->set_disabled(!has_editor_tree);
+
+	thread_list_updating = true;
+	LocalVector<ThreadDebugged *> threadss;
+	for (KeyValue<uint64_t, ThreadDebugged> &I : threads_debugged) {
+		threadss.push_back(&I.value);
+	}
+
+	threadss.sort_custom<ThreadSort>();
+	threads->clear();
+	int32_t selected_index = -1;
+	for (uint32_t i = 0; i < threadss.size(); i++) {
+		if (debugging_thread_id == threadss[i]->thread_id) {
+			selected_index = i;
+		}
+		threads->add_item(threadss[i]->name);
+		threads->set_item_metadata(threads->get_item_count() - 1, threadss[i]->thread_id);
+	}
+	if (selected_index != -1) {
+		threads->select(selected_index);
+	}
+
+	thread_list_updating = false;
 }
 
 void ScriptEditorDebugger::_stop_and_notify() {
@@ -990,8 +1048,8 @@ void ScriptEditorDebugger::_stop_and_notify() {
 
 void ScriptEditorDebugger::stop() {
 	set_process(false);
-	breaked = false;
-	can_debug = false;
+	threads_debugged.clear();
+	debugging_thread_id = Thread::UNASSIGNED_ID;
 	remote_pid = 0;
 	_clear_execution();
 
@@ -1043,7 +1101,7 @@ void ScriptEditorDebugger::_profiler_activate(bool p_enable, int p_type) {
 }
 
 void ScriptEditorDebugger::_profiler_seeked() {
-	if (breaked) {
+	if (is_breaked()) {
 		return;
 	}
 	debug_break();
@@ -1067,7 +1125,7 @@ void ScriptEditorDebugger::_export_csv() {
 }
 
 String ScriptEditorDebugger::get_var_value(const String &p_var) const {
-	if (!breaked) {
+	if (!is_breaked()) {
 		return String();
 	}
 	return inspector->get_stack_variable(p_var);
@@ -1255,7 +1313,7 @@ bool ScriptEditorDebugger::request_stack_dump(const int &p_frame) {
 
 	Array msg;
 	msg.push_back(p_frame);
-	_put_msg("get_stack_frame_vars", msg);
+	_put_msg("get_stack_frame_vars", msg, debugging_thread_id);
 	return true;
 }
 
@@ -1407,7 +1465,7 @@ void ScriptEditorDebugger::set_breakpoint(const String &p_path, int p_line, bool
 	msg.push_back(p_path);
 	msg.push_back(p_line);
 	msg.push_back(p_enabled);
-	_put_msg("breakpoint", msg);
+	_put_msg("breakpoint", msg, debugging_thread_id != Thread::UNASSIGNED_ID ? debugging_thread_id : Thread::MAIN_ID);
 
 	TreeItem *path_item = breakpoints_tree->search_item_text(p_path);
 	if (path_item == nullptr) {
@@ -1450,7 +1508,7 @@ void ScriptEditorDebugger::set_breakpoint(const String &p_path, int p_line, bool
 }
 
 void ScriptEditorDebugger::reload_scripts() {
-	_put_msg("reload_scripts", Array());
+	_put_msg("reload_scripts", Array(), debugging_thread_id != Thread::UNASSIGNED_ID ? debugging_thread_id : Thread::MAIN_ID);
 }
 
 bool ScriptEditorDebugger::is_skip_breakpoints() {
@@ -1804,15 +1862,26 @@ ScriptEditorDebugger::ScriptEditorDebugger() {
 		sc->set_h_size_flags(SIZE_EXPAND_FILL);
 		parent_sc->add_child(sc);
 
+		VBoxContainer *stack_vb = memnew(VBoxContainer);
+		stack_vb->set_h_size_flags(SIZE_EXPAND_FILL);
+		sc->add_child(stack_vb);
+		HBoxContainer *thread_hb = memnew(HBoxContainer);
+		stack_vb->add_child(thread_hb);
+		thread_hb->add_child(memnew(Label(TTR("Thread:"))));
+		threads = memnew(OptionButton);
+		thread_hb->add_child(threads);
+		threads->set_h_size_flags(SIZE_EXPAND_FILL);
+		threads->connect("item_selected", callable_mp(this, &ScriptEditorDebugger::_select_thread));
+
 		stack_dump = memnew(Tree);
 		stack_dump->set_allow_reselect(true);
 		stack_dump->set_columns(1);
 		stack_dump->set_column_titles_visible(true);
 		stack_dump->set_column_title(0, TTR("Stack Frames"));
-		stack_dump->set_h_size_flags(SIZE_EXPAND_FILL);
 		stack_dump->set_hide_root(true);
+		stack_dump->set_v_size_flags(SIZE_EXPAND_FILL);
 		stack_dump->connect("cell_selected", callable_mp(this, &ScriptEditorDebugger::_stack_dump_frame_selected));
-		sc->add_child(stack_dump);
+		stack_vb->add_child(stack_dump);
 
 		VBoxContainer *inspector_vbox = memnew(VBoxContainer);
 		inspector_vbox->set_h_size_flags(SIZE_EXPAND_FILL);
diff --git a/editor/debugger/script_editor_debugger.h b/editor/debugger/script_editor_debugger.h
index 336a113163..7e9a767273 100644
--- a/editor/debugger/script_editor_debugger.h
+++ b/editor/debugger/script_editor_debugger.h
@@ -138,6 +138,7 @@ private:
 
 	Tree *stack_dump = nullptr;
 	LineEdit *search = nullptr;
+	OptionButton *threads = nullptr;
 	EditorDebuggerInspector *inspector = nullptr;
 	SceneDebuggerTree *scene_tree = nullptr;
 
@@ -152,19 +153,39 @@ private:
 	EditorPerformanceProfiler *performance_profiler = nullptr;
 
 	OS::ProcessID remote_pid = 0;
-	bool breaked = false;
-	bool can_debug = false;
 	bool move_to_foreground = true;
 	bool can_request_idle_draw = false;
 
 	bool live_debug;
 
+	uint64_t debugging_thread_id = Thread::UNASSIGNED_ID;
+
+	struct ThreadDebugged {
+		String name;
+		String error;
+		bool can_debug = false;
+		bool has_stackdump = false;
+		uint32_t debug_order = 0;
+		uint64_t thread_id = Thread::UNASSIGNED_ID; // for order
+	};
+
+	struct ThreadSort {
+		bool operator()(const ThreadDebugged *a, const ThreadDebugged *b) const {
+			return a->debug_order < b->debug_order;
+		}
+	};
+
+	HashMap<uint64_t, ThreadDebugged> threads_debugged;
+	bool thread_list_updating = false;
+
+	void _select_thread(int p_index);
+
 	EditorDebuggerNode::CameraOverride camera_override;
 
 	void _stack_dump_frame_selected();
 
 	void _file_selected(const String &p_file);
-	void _parse_message(const String &p_msg, const Array &p_data);
+	void _parse_message(const String &p_msg, uint64_t p_thread_id, const Array &p_data);
 	void _set_reason_text(const String &p_reason, MessageType p_type);
 	void _update_buttons_state();
 	void _remote_object_selected(ObjectID p_object);
@@ -200,7 +221,7 @@ private:
 	void _item_menu_id_pressed(int p_option);
 	void _tab_changed(int p_tab);
 
-	void _put_msg(String p_message, Array p_data);
+	void _put_msg(String p_message, Array p_data, uint64_t p_thread_id = Thread::MAIN_ID);
 	void _export_csv();
 
 	void _clear_execution();
@@ -213,6 +234,8 @@ private:
 
 	String _format_frame_text(const ScriptLanguage::StackInfo *info);
 
+	void _thread_debug_enter(uint64_t p_thread_id);
+
 protected:
 	void _notification(int p_what);
 	static void _bind_methods();
@@ -238,9 +261,9 @@ public:
 	void debug_step();
 	void debug_break();
 	void debug_continue();
-	bool is_breaked() const { return breaked; }
-	bool is_debuggable() const { return can_debug; }
-	bool is_session_active() { return peer.is_valid() && peer->is_peer_connected(); };
+	bool is_breaked() const { return threads_debugged.size() > 0; }
+	bool is_debuggable() const { return threads_debugged.size() > 0 && threads_debugged[debugging_thread_id].can_debug; }
+	bool is_session_active() { return peer.is_valid() && peer->is_peer_connected(); }
 	int get_remote_pid() const { return remote_pid; }
 
 	bool is_move_to_foreground() const;