From 5d6e8538065050d5f5579ec03cfa9e241811e062 Mon Sep 17 00:00:00 2001 From: George Marques Date: Fri, 1 May 2020 19:14:56 -0300 Subject: New GDScript tokenizer and parser Sometimes to fix something you have to break it first. This get GDScript mostly working with the new tokenizer and parser but a lot of things isn't working yet. It compiles and it's usable, and that should be enough for now. Don't worry: other huge commits will come after this. --- modules/gdscript/gdscript_editor.cpp | 3142 ++-------------------------------- 1 file changed, 104 insertions(+), 3038 deletions(-) (limited to 'modules/gdscript/gdscript_editor.cpp') diff --git a/modules/gdscript/gdscript_editor.cpp b/modules/gdscript/gdscript_editor.cpp index 3a5db3687b..5cbd8afc07 100644 --- a/modules/gdscript/gdscript_editor.cpp +++ b/modules/gdscript/gdscript_editor.cpp @@ -34,6 +34,8 @@ #include "core/global_constants.h" #include "core/os/file_access.h" #include "gdscript_compiler.h" +#include "gdscript_parser.h" +#include "gdscript_tokenizer.h" #ifdef TOOLS_ENABLED #include "editor/editor_file_system.h" @@ -114,50 +116,47 @@ void GDScriptLanguage::make_template(const String &p_class_name, const String &p p_script->set_source_code(_template); } +static void get_function_names_recursively(const GDScriptParser::ClassNode *p_class, const String &p_prefix, Map &r_funcs) { + for (int i = 0; i < p_class->members.size(); i++) { + if (p_class->members[i].type == GDScriptParser::ClassNode::Member::FUNCTION) { + const GDScriptParser::FunctionNode *function = p_class->members[i].function; + r_funcs[function->start_line] = p_prefix.empty() ? String(function->identifier->name) : p_prefix + "." + String(function->identifier->name); + } else if (p_class->members[i].type == GDScriptParser::ClassNode::Member::CLASS) { + String new_prefix = p_class->members[i].m_class->identifier->name; + get_function_names_recursively(p_class->members[i].m_class, p_prefix.empty() ? new_prefix : p_prefix + "." + new_prefix, r_funcs); + } + } +} + bool GDScriptLanguage::validate(const String &p_script, int &r_line_error, int &r_col_error, String &r_test_error, const String &p_path, List *r_functions, List *r_warnings, Set *r_safe_lines) const { GDScriptParser parser; - Error err = parser.parse(p_script, p_path.get_base_dir(), true, p_path, false, r_safe_lines); + Error err = parser.parse(p_script, p_path, false); #ifdef DEBUG_ENABLED - if (r_warnings) { - for (const List::Element *E = parser.get_warnings().front(); E; E = E->next()) { - const GDScriptWarning &warn = E->get(); - ScriptLanguage::Warning w; - w.line = warn.line; - w.code = (int)warn.code; - w.string_code = GDScriptWarning::get_name_from_code(warn.code); - w.message = warn.get_message(); - r_warnings->push_back(w); - } - } + // FIXME: Warnings. + // if (r_warnings) { + // for (const List::Element *E = parser.get_warnings().front(); E; E = E->next()) { + // const GDScriptWarning &warn = E->get(); + // ScriptLanguage::Warning w; + // w.line = warn.line; + // w.code = (int)warn.code; + // w.string_code = GDScriptWarning::get_name_from_code(warn.code); + // w.message = warn.get_message(); + // r_warnings->push_back(w); + // } + // } #endif if (err) { - r_line_error = parser.get_error_line(); - r_col_error = parser.get_error_column(); - r_test_error = parser.get_error(); + GDScriptParser::ParserError parse_error = parser.get_errors().front()->get(); + r_line_error = parse_error.line; + r_col_error = parse_error.column; + r_test_error = parse_error.message; return false; } else { - const GDScriptParser::Node *root = parser.get_parse_tree(); - ERR_FAIL_COND_V(root->type != GDScriptParser::Node::TYPE_CLASS, false); - - const GDScriptParser::ClassNode *cl = static_cast(root); + const GDScriptParser::ClassNode *cl = parser.get_tree(); Map funcs; - for (int i = 0; i < cl->functions.size(); i++) { - funcs[cl->functions[i]->line] = cl->functions[i]->name; - } - for (int i = 0; i < cl->static_functions.size(); i++) { - funcs[cl->static_functions[i]->line] = cl->static_functions[i]->name; - } - - for (int i = 0; i < cl->subclasses.size(); i++) { - for (int j = 0; j < cl->subclasses[i]->functions.size(); j++) { - funcs[cl->subclasses[i]->functions[j]->line] = String(cl->subclasses[i]->name) + "." + cl->subclasses[i]->functions[j]->name; - } - for (int j = 0; j < cl->subclasses[i]->static_functions.size(); j++) { - funcs[cl->subclasses[i]->static_functions[j]->line] = String(cl->subclasses[i]->name) + "." + cl->subclasses[i]->static_functions[j]->name; - } - } + get_function_names_recursively(cl, "", funcs); for (Map::Element *E = funcs.front(); E; E = E->next()) { r_functions->push_back(E->get() + ":" + itos(E->key())); @@ -176,20 +175,26 @@ bool GDScriptLanguage::supports_builtin_mode() const { } int GDScriptLanguage::find_function(const String &p_function, const String &p_code) const { - GDScriptTokenizerText tokenizer; - tokenizer.set_code(p_code); + GDScriptTokenizer tokenizer; + tokenizer.set_source_code(p_code); int indent = 0; - while (tokenizer.get_token() != GDScriptTokenizer::TK_EOF && tokenizer.get_token() != GDScriptTokenizer::TK_ERROR) { - if (tokenizer.get_token() == GDScriptTokenizer::TK_NEWLINE) { - indent = tokenizer.get_token_line_indent(); + GDScriptTokenizer::Token current = tokenizer.scan(); + while (current.type != GDScriptTokenizer::Token::TK_EOF && current.type != GDScriptTokenizer::Token::ERROR) { + if (current.type == GDScriptTokenizer::Token::INDENT) { + indent++; + } else if (current.type == GDScriptTokenizer::Token::DEDENT) { + indent--; } - if (indent == 0 && tokenizer.get_token() == GDScriptTokenizer::TK_PR_FUNCTION && tokenizer.get_token(1) == GDScriptTokenizer::TK_IDENTIFIER) { - String identifier = tokenizer.get_token_identifier(1); - if (identifier == p_function) { - return tokenizer.get_token_line(); + if (indent == 0 && current.type == GDScriptTokenizer::Token::FUNC) { + current = tokenizer.scan(); + if (current.is_identifier()) { + String identifier = current.get_identifier(); + if (identifier == p_function) { + return current.start_line; + } } } - tokenizer.advance(); + current = tokenizer.scan(); } return -1; } @@ -395,16 +400,6 @@ void GDScriptLanguage::get_public_functions(List *p_functions) const mi.return_val = PropertyInfo(Variant::OBJECT, "", PROPERTY_HINT_RESOURCE_TYPE, "Resource"); p_functions->push_back(mi); } - { - MethodInfo mi; - mi.name = "yield"; - mi.arguments.push_back(PropertyInfo(Variant::OBJECT, "object")); - mi.arguments.push_back(PropertyInfo(Variant::STRING, "signal")); - mi.default_arguments.push_back(Variant()); - mi.default_arguments.push_back(String()); - mi.return_val = PropertyInfo(Variant::OBJECT, "", PROPERTY_HINT_RESOURCE_TYPE, "GDScriptFunctionState"); - p_functions->push_back(mi); - } { MethodInfo mi; mi.name = "assert"; @@ -467,3027 +462,98 @@ String GDScriptLanguage::make_function(const String &p_class, const String &p_na //////// COMPLETION ////////// -#if defined(DEBUG_METHODS_ENABLED) && defined(TOOLS_ENABLED) - -struct GDScriptCompletionContext { - const GDScriptParser::ClassNode *_class = nullptr; - const GDScriptParser::FunctionNode *function = nullptr; - const GDScriptParser::BlockNode *block = nullptr; - Object *base = nullptr; - String base_path; - int line = 0; - uint32_t depth = 0; - - GDScriptCompletionContext() {} -}; - -struct GDScriptCompletionIdentifier { - GDScriptParser::DataType type; - String enumeration; - Variant value; - const GDScriptParser::Node *assigned_expression = nullptr; - - GDScriptCompletionIdentifier() {} -}; - -static void _get_directory_contents(EditorFileSystemDirectory *p_dir, Map &r_list) { - const String quote_style = EDITOR_DEF("text_editor/completion/use_single_quotes", false) ? "'" : "\""; - - for (int i = 0; i < p_dir->get_file_count(); i++) { - ScriptCodeCompletionOption option(p_dir->get_file_path(i), ScriptCodeCompletionOption::KIND_FILE_PATH); - option.insert_text = quote_style + option.display + quote_style; - r_list.insert(option.display, option); - } - - for (int i = 0; i < p_dir->get_subdir_count(); i++) { - _get_directory_contents(p_dir->get_subdir(i), r_list); - } -} - -static String _get_visual_datatype(const PropertyInfo &p_info, bool p_isarg = true) { - if (p_info.usage & PROPERTY_USAGE_CLASS_IS_ENUM) { - String enum_name = p_info.class_name; - if (enum_name.find(".") == -1) { - return enum_name; - } - return enum_name.get_slice(".", 1); - } - - String n = p_info.name; - int idx = n.find(":"); - if (idx != -1) { - return n.substr(idx + 1, n.length()); - } - - if (p_info.type == Variant::OBJECT) { - if (p_info.hint == PROPERTY_HINT_RESOURCE_TYPE) { - return p_info.hint_string; - } else { - return p_info.class_name.operator String(); - } - } - if (p_info.type == Variant::NIL) { - if (p_isarg || (p_info.usage & PROPERTY_USAGE_NIL_IS_VARIANT)) { - return "Variant"; - } else { - return "void"; - } - } - - return Variant::get_type_name(p_info.type); -} - -static GDScriptCompletionIdentifier _type_from_variant(const Variant &p_value) { - GDScriptCompletionIdentifier ci; - ci.value = p_value; - ci.type.is_constant = true; - ci.type.has_type = true; - ci.type.kind = GDScriptParser::DataType::BUILTIN; - ci.type.builtin_type = p_value.get_type(); - - if (ci.type.builtin_type == Variant::OBJECT) { - Object *obj = p_value.operator Object *(); - if (!obj) { - return ci; - } - ci.type.native_type = obj->get_class_name(); - Ref