1 files changed, 191 insertions, 233 deletions

diff --git a/editor/debugger/editor_file_server.cpp b/editor/debugger/editor_file_server.cpp
index ba5dd9e6e2..df317f8106 100644
--- a/editor/debugger/editor_file_server.cpp
+++ b/editor/debugger/editor_file_server.cpp
@@ -32,273 +32,230 @@
 
 #include "../editor_settings.h"
 #include "core/io/marshalls.h"
+#include "editor/editor_node.h"
+#include "editor/export/editor_export_platform.h"
+
+#define FILESYSTEM_PROTOCOL_VERSION 1
+#define PASSWORD_LENGTH 32
+#define MAX_FILE_BUFFER_SIZE 100 * 1024 * 1024 // 100mb max file buffer size (description of files to update, compressed).
+
+static void _add_file(String f, const uint64_t &p_modified_time, HashMap<String, uint64_t> &files_to_send, HashMap<String, uint64_t> &cached_files) {
+	f = f.replace_first("res://", ""); // remove res://
+	const uint64_t *cached_mt = cached_files.getptr(f);
+	if (cached_mt && *cached_mt == p_modified_time) {
+		// File is good, skip it.
+		cached_files.erase(f); // Erase to mark this file as existing. Remaning files not added to files_to_send will be considered erased here, so they need to be erased in the client too.
+		return;
+	}
+	files_to_send.insert(f, p_modified_time);
+}
 
-//#define DEBUG_PRINT(m_p) print_line(m_p)
-//#define DEBUG_TIME(m_what) printf("MS: %s - %lu\n", m_what, OS::get_singleton()->get_ticks_usec());
+void EditorFileServer::_scan_files_changed(EditorFileSystemDirectory *efd, const Vector<String> &p_tags, HashMap<String, uint64_t> &files_to_send, HashMap<String, uint64_t> &cached_files) {
+	for (int i = 0; i < efd->get_file_count(); i++) {
+		String f = efd->get_file_path(i);
+		if (FileAccess::exists(f + ".import")) {
+			// is imported, determine what to do
+			// Todo the modified times of remapped files should most likely be kept in EditorFileSystem to speed this up in the future.
+			Ref<ConfigFile> cf;
+			cf.instantiate();
+			Error err = cf->load(f + ".import");
+
+			ERR_CONTINUE(err != OK);
+			{
+				uint64_t mt = FileAccess::get_modified_time(f + ".import");
+				_add_file(f + ".import", mt, files_to_send, cached_files);
+			}
 
-#define DEBUG_PRINT(m_what)
-#define DEBUG_TIME(m_what)
+			if (!cf->has_section("remap")) {
+				continue;
+			}
 
-void EditorFileServer::_close_client(ClientData *cd) {
-	cd->connection->disconnect_from_host();
-	{
-		MutexLock lock(cd->efs->wait_mutex);
-		cd->efs->to_wait.insert(cd->thread);
+			List<String> remaps;
+			cf->get_section_keys("remap", &remaps);
+
+			for (const String &remap : remaps) {
+				if (remap == "path") {
+					String remapped_path = cf->get_value("remap", remap);
+					uint64_t mt = FileAccess::get_modified_time(remapped_path);
+					_add_file(remapped_path, mt, files_to_send, cached_files);
+				} else if (remap.begins_with("path.")) {
+					String feature = remap.get_slice(".", 1);
+					if (p_tags.find(feature) != -1) {
+						String remapped_path = cf->get_value("remap", remap);
+						uint64_t mt = FileAccess::get_modified_time(remapped_path);
+						_add_file(remapped_path, mt, files_to_send, cached_files);
+					}
+				}
+			}
+		} else {
+			uint64_t mt = efd->get_file_modified_time(i);
+			_add_file(f, mt, files_to_send, cached_files);
+		}
 	}
-	while (cd->files.size()) {
-		cd->files.remove(cd->files.begin());
+
+	for (int i = 0; i < efd->get_subdir_count(); i++) {
+		_scan_files_changed(efd->get_subdir(i), p_tags, files_to_send, cached_files);
 	}
-	memdelete(cd);
 }
 
-void EditorFileServer::_subthread_start(void *s) {
-	ClientData *cd = static_cast<ClientData *>(s);
+static void _add_custom_file(const String f, HashMap<String, uint64_t> &files_to_send, HashMap<String, uint64_t> &cached_files) {
+	if (!FileAccess::exists(f)) {
+		return;
+	}
+	_add_file(f, FileAccess::get_modified_time(f), files_to_send, cached_files);
+}
 
-	cd->connection->set_no_delay(true);
-	uint8_t buf4[8];
-	Error err = cd->connection->get_data(buf4, 4);
-	if (err != OK) {
-		_close_client(cd);
-		ERR_FAIL_COND(err != OK);
+void EditorFileServer::poll() {
+	if (!active) {
+		return;
 	}
 
-	int passlen = decode_uint32(buf4);
-
-	if (passlen > 512) {
-		_close_client(cd);
-		ERR_FAIL_COND(passlen > 512);
-	} else if (passlen > 0) {
-		Vector<char> passutf8;
-		passutf8.resize(passlen + 1);
-		err = cd->connection->get_data((uint8_t *)passutf8.ptr(), passlen);
-		if (err != OK) {
-			_close_client(cd);
-			ERR_FAIL_COND(err != OK);
-		}
-		passutf8.write[passlen] = 0;
-		String s2;
-		s2.parse_utf8(passutf8.ptr());
-		if (s2 != cd->efs->password) {
-			encode_uint32(ERR_INVALID_DATA, buf4);
-			cd->connection->put_data(buf4, 4);
-			OS::get_singleton()->delay_usec(1000000);
-			_close_client(cd);
-			ERR_PRINT("CLIENT PASSWORD MISMATCH");
-			ERR_FAIL();
-		}
-	} else {
-		if (!cd->efs->password.is_empty()) {
-			encode_uint32(ERR_INVALID_DATA, buf4);
-			cd->connection->put_data(buf4, 4);
-			OS::get_singleton()->delay_usec(1000000);
-			_close_client(cd);
-			ERR_PRINT("CLIENT PASSWORD MISMATCH (should be empty!)");
-			ERR_FAIL();
-		}
+	if (!server->is_connection_available()) {
+		return;
 	}
 
-	encode_uint32(OK, buf4);
-	cd->connection->put_data(buf4, 4);
+	Ref<StreamPeerTCP> tcp_peer = server->take_connection();
+	ERR_FAIL_COND(tcp_peer.is_null());
+
+	// Got a connection!
+	EditorProgress pr("updating_remote_file_system", TTR("Updating assets on target device:"), 105);
+
+	pr.step(TTR("Syncinc headers"), 0, true);
+	print_verbose("EFS: Connecting taken!");
+	char header[4];
+	Error err = tcp_peer->get_data((uint8_t *)&header, 4);
+	ERR_FAIL_COND(err != OK);
+	ERR_FAIL_COND(header[0] != 'G');
+	ERR_FAIL_COND(header[1] != 'R');
+	ERR_FAIL_COND(header[2] != 'F');
+	ERR_FAIL_COND(header[3] != 'S');
+
+	uint32_t protocol_version = tcp_peer->get_u32();
+	ERR_FAIL_COND(protocol_version != FILESYSTEM_PROTOCOL_VERSION);
+
+	char cpassword[PASSWORD_LENGTH + 1];
+	err = tcp_peer->get_data((uint8_t *)cpassword, PASSWORD_LENGTH);
+	cpassword[PASSWORD_LENGTH] = 0;
+	ERR_FAIL_COND(err != OK);
+	print_verbose("EFS: Got password: " + String(cpassword));
+	ERR_FAIL_COND_MSG(password != cpassword, "Client disconnected because password mismatch.");
+
+	uint32_t tag_count = tcp_peer->get_u32();
+	print_verbose("EFS: Getting tags: " + itos(tag_count));
+
+	ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED);
+	Vector<String> tags;
+	for (uint32_t i = 0; i < tag_count; i++) {
+		String tag = tcp_peer->get_utf8_string();
+		print_verbose("EFS: tag #" + itos(i) + ": " + tag);
+		ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED);
+		tags.push_back(tag);
+	}
 
-	while (!cd->quit) {
-		//wait for ID
-		err = cd->connection->get_data(buf4, 4);
-		DEBUG_TIME("get_data")
+	uint32_t file_buffer_decompressed_size = tcp_peer->get_32();
+	HashMap<String, uint64_t> cached_files;
 
-		if (err != OK) {
-			_close_client(cd);
-			ERR_FAIL_COND(err != OK);
-		}
-		int id = decode_uint32(buf4);
+	if (file_buffer_decompressed_size > 0) {
+		pr.step(TTR("Getting remote file system"), 1, true);
 
-		//wait for command
-		err = cd->connection->get_data(buf4, 4);
-		if (err != OK) {
-			_close_client(cd);
-			ERR_FAIL_COND(err != OK);
-		}
-		int cmd = decode_uint32(buf4);
-
-		switch (cmd) {
-			case FileAccessNetwork::COMMAND_FILE_EXISTS:
-			case FileAccessNetwork::COMMAND_GET_MODTIME:
-			case FileAccessNetwork::COMMAND_OPEN_FILE: {
-				DEBUG_TIME("open_file")
-				err = cd->connection->get_data(buf4, 4);
-				if (err != OK) {
-					_close_client(cd);
-					ERR_FAIL_COND(err != OK);
-				}
+		// Got files cached by client.
+		uint32_t file_buffer_size = tcp_peer->get_32();
+		print_verbose("EFS: Getting file buffer: compressed - " + String::humanize_size(file_buffer_size) + " decompressed: " + String::humanize_size(file_buffer_decompressed_size));
 
-				int namelen = decode_uint32(buf4);
-				Vector<char> fileutf8;
-				fileutf8.resize(namelen + 1);
-				err = cd->connection->get_data((uint8_t *)fileutf8.ptr(), namelen);
-				if (err != OK) {
-					_close_client(cd);
-					ERR_FAIL_COND(err != OK);
-				}
-				fileutf8.write[namelen] = 0;
-				String s2;
-				s2.parse_utf8(fileutf8.ptr());
+		ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED);
+		ERR_FAIL_COND(file_buffer_size > MAX_FILE_BUFFER_SIZE);
+		LocalVector<uint8_t> file_buffer;
+		file_buffer.resize(file_buffer_size);
+		LocalVector<uint8_t> file_buffer_decompressed;
+		file_buffer_decompressed.resize(file_buffer_decompressed_size);
 
-				if (cmd == FileAccessNetwork::COMMAND_FILE_EXISTS) {
-					print_verbose("FILE EXISTS: " + s2);
-				}
-				if (cmd == FileAccessNetwork::COMMAND_GET_MODTIME) {
-					print_verbose("MOD TIME: " + s2);
-				}
-				if (cmd == FileAccessNetwork::COMMAND_OPEN_FILE) {
-					print_verbose("OPEN: " + s2);
-				}
+		err = tcp_peer->get_data(file_buffer.ptr(), file_buffer_size);
 
-				if (!s2.begins_with("res://")) {
-					_close_client(cd);
-					ERR_FAIL_COND(!s2.begins_with("res://"));
-				}
-				ERR_CONTINUE(cd->files.has(id));
-
-				if (cmd == FileAccessNetwork::COMMAND_FILE_EXISTS) {
-					encode_uint32(id, buf4);
-					cd->connection->put_data(buf4, 4);
-					encode_uint32(FileAccessNetwork::RESPONSE_FILE_EXISTS, buf4);
-					cd->connection->put_data(buf4, 4);
-					encode_uint32(FileAccess::exists(s2), buf4);
-					cd->connection->put_data(buf4, 4);
-					DEBUG_TIME("open_file_end")
-					break;
-				}
+		pr.step(TTR("Decompressing remote file system"), 2, true);
 
-				if (cmd == FileAccessNetwork::COMMAND_GET_MODTIME) {
-					encode_uint32(id, buf4);
-					cd->connection->put_data(buf4, 4);
-					encode_uint32(FileAccessNetwork::RESPONSE_GET_MODTIME, buf4);
-					cd->connection->put_data(buf4, 4);
-					encode_uint64(FileAccess::get_modified_time(s2), buf4);
-					cd->connection->put_data(buf4, 8);
-					DEBUG_TIME("open_file_end")
-					break;
-				}
-
-				Ref<FileAccess> fa = FileAccess::open(s2, FileAccess::READ);
-				if (fa.is_null()) {
-					//not found, continue
-					encode_uint32(id, buf4);
-					cd->connection->put_data(buf4, 4);
-					encode_uint32(FileAccessNetwork::RESPONSE_OPEN, buf4);
-					cd->connection->put_data(buf4, 4);
-					encode_uint32(ERR_FILE_NOT_FOUND, buf4);
-					cd->connection->put_data(buf4, 4);
-					DEBUG_TIME("open_file_end")
-					break;
-				}
+		ERR_FAIL_COND(err != OK);
+		// Decompress the text with all the files
+		Compression::decompress(file_buffer_decompressed.ptr(), file_buffer_decompressed.size(), file_buffer.ptr(), file_buffer.size(), Compression::MODE_ZSTD);
+		String files_text = String::utf8((const char *)file_buffer_decompressed.ptr(), file_buffer_decompressed.size());
+		Vector<String> files = files_text.split("\n");
+
+		print_verbose("EFS: Total cached files received: " + itos(files.size()));
+		for (int i = 0; i < files.size(); i++) {
+			if (files[i].get_slice_count("::") != 2) {
+				continue;
+			}
+			String file = files[i].get_slice("::", 0);
+			uint64_t modified_time = files[i].get_slice("::", 1).to_int();
 
-				encode_uint32(id, buf4);
-				cd->connection->put_data(buf4, 4);
-				encode_uint32(FileAccessNetwork::RESPONSE_OPEN, buf4);
-				cd->connection->put_data(buf4, 4);
-				encode_uint32(OK, buf4);
-				cd->connection->put_data(buf4, 4);
-				encode_uint64(fa->get_length(), buf4);
-				cd->connection->put_data(buf4, 8);
-
-				cd->files[id] = fa;
-				DEBUG_TIME("open_file_end")
-
-			} break;
-			case FileAccessNetwork::COMMAND_READ_BLOCK: {
-				err = cd->connection->get_data(buf4, 8);
-				if (err != OK) {
-					_close_client(cd);
-					ERR_FAIL_COND(err != OK);
-				}
+			cached_files.insert(file, modified_time);
+		}
+	} else {
+		// Client does not have any files stored.
+	}
 
-				ERR_CONTINUE(!cd->files.has(id));
+	pr.step(TTR("Scanning for local changes"), 3, true);
 
-				uint64_t offset = decode_uint64(buf4);
+	print_verbose("EFS: Scanning changes:");
 
-				err = cd->connection->get_data(buf4, 4);
-				if (err != OK) {
-					_close_client(cd);
-					ERR_FAIL_COND(err != OK);
-				}
+	HashMap<String, uint64_t> files_to_send;
+	// Scan files to send.
+	_scan_files_changed(EditorFileSystem::get_singleton()->get_filesystem(), tags, files_to_send, cached_files);
+	// Add forced export files
+	Vector<String> forced_export = EditorExportPlatform::get_forced_export_files();
+	for (int i = 0; i < forced_export.size(); i++) {
+		_add_custom_file(forced_export[i], files_to_send, cached_files);
+	}
 
-				int blocklen = decode_uint32(buf4);
-				ERR_CONTINUE(blocklen > (16 * 1024 * 1024));
-
-				cd->files[id]->seek(offset);
-				Vector<uint8_t> buf;
-				buf.resize(blocklen);
-				uint32_t read = cd->files[id]->get_buffer(buf.ptrw(), blocklen);
-
-				print_verbose("GET BLOCK - offset: " + itos(offset) + ", blocklen: " + itos(blocklen));
-
-				//not found, continue
-				encode_uint32(id, buf4);
-				cd->connection->put_data(buf4, 4);
-				encode_uint32(FileAccessNetwork::RESPONSE_DATA, buf4);
-				cd->connection->put_data(buf4, 4);
-				encode_uint64(offset, buf4);
-				cd->connection->put_data(buf4, 8);
-				encode_uint32(read, buf4);
-				cd->connection->put_data(buf4, 4);
-				cd->connection->put_data(buf.ptr(), read);
-
-			} break;
-			case FileAccessNetwork::COMMAND_CLOSE: {
-				print_verbose("CLOSED");
-				ERR_CONTINUE(!cd->files.has(id));
-				cd->files.erase(id);
-			} break;
+	_add_custom_file("res://project.godot", files_to_send, cached_files);
+	// Check which files were removed and also add them
+	for (KeyValue<String, uint64_t> K : cached_files) {
+		if (!files_to_send.has(K.key)) {
+			files_to_send.insert(K.key, 0); //0 means removed
 		}
 	}
 
-	_close_client(cd);
-}
+	tcp_peer->put_32(files_to_send.size());
 
-void EditorFileServer::_thread_start(void *s) {
-	EditorFileServer *self = static_cast<EditorFileServer *>(s);
-	while (!self->quit) {
-		if (self->cmd == CMD_ACTIVATE) {
-			self->server->listen(self->port);
-			self->active = true;
-			self->cmd = CMD_NONE;
-		} else if (self->cmd == CMD_STOP) {
-			self->server->stop();
-			self->active = false;
-			self->cmd = CMD_NONE;
-		}
+	print_verbose("EFS: Sending list of changed files.");
+	pr.step(TTR("Sending list of changed files:"), 4, true);
 
-		if (self->active) {
-			if (self->server->is_connection_available()) {
-				ClientData *cd = memnew(ClientData);
-				cd->connection = self->server->take_connection();
-				cd->efs = self;
-				cd->quit = false;
-				cd->thread = memnew(Thread);
-				cd->thread->start(_subthread_start, cd);
-			}
-		}
+	// Send list of changed files first, to ensure that if connecting breaks, the client is not found in a broken state.
+	for (KeyValue<String, uint64_t> K : files_to_send) {
+		tcp_peer->put_utf8_string(K.key);
+		tcp_peer->put_64(K.value);
+	}
+
+	print_verbose("EFS: Sending " + itos(files_to_send.size()) + " files.");
+
+	int idx = 0;
+	for (KeyValue<String, uint64_t> K : files_to_send) {
+		pr.step(TTR("Sending file: ") + K.key.get_file(), 5 + idx * 100 / files_to_send.size(), false);
+		idx++;
 
-		self->wait_mutex.lock();
-		while (self->to_wait.size()) {
-			Thread *w = *self->to_wait.begin();
-			self->to_wait.erase(w);
-			self->wait_mutex.unlock();
-			w->wait_to_finish();
-			self->wait_mutex.lock();
+		if (K.value == 0 || !FileAccess::exists("res://" + K.key)) { // File was removed
+			continue;
 		}
-		self->wait_mutex.unlock();
 
-		OS::get_singleton()->delay_usec(100000);
+		Vector<uint8_t> array = FileAccess::_get_file_as_bytes("res://" + K.key);
+		tcp_peer->put_64(array.size());
+		tcp_peer->put_data(array.ptr(), array.size());
+		ERR_FAIL_COND(tcp_peer->get_status() != StreamPeerTCP::STATUS_CONNECTED);
 	}
+
+	tcp_peer->put_data((const uint8_t *)"GEND", 4); // End marker.
+
+	print_verbose("EFS: Done.");
 }
 
 void EditorFileServer::start() {
-	stop();
+	if (active) {
+		stop();
+	}
 	port = EDITOR_GET("filesystem/file_server/port");
 	password = EDITOR_GET("filesystem/file_server/password");
-	cmd = CMD_ACTIVATE;
+	Error err = server->listen(port);
+	ERR_FAIL_COND_MSG(err != OK, "EditorFileServer: Unable to listen on port " + itos(port));
+	active = true;
 }
 
 bool EditorFileServer::is_active() const {
@@ -306,18 +263,19 @@ bool EditorFileServer::is_active() const {
 }
 
 void EditorFileServer::stop() {
-	cmd = CMD_STOP;
+	if (active) {
+		server->stop();
+		active = false;
+	}
 }
 
 EditorFileServer::EditorFileServer() {
 	server.instantiate();
-	thread.start(_thread_start, this);
 
 	EDITOR_DEF("filesystem/file_server/port", 6010);
 	EDITOR_DEF("filesystem/file_server/password", "");
 }
 
 EditorFileServer::~EditorFileServer() {
-	quit = true;
-	thread.wait_to_finish();
+	stop();
 }