diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 76a1f6f001..3fd2ec9285 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -112,7 +112,6 @@ endforeach (FILE_TO_LINK) set(FORMAT_FILES formats/access_log.json formats/alb_log.json - formats/autodeploy_log.json formats/block_log.json formats/candlepin_log.json formats/choose_repo_log.json @@ -121,6 +120,7 @@ set(FORMAT_FILES formats/elb_log.json formats/engine_log.json formats/error_log.json + formats/esx_syslog_log.json formats/fsck_hfs_log.json formats/glog_log.json formats/haproxy_log.json @@ -144,6 +144,8 @@ set(FORMAT_FILES formats/vdsm_log.json formats/vmk_log.json formats/vmw_log.json + formats/vmw_vc_svc_log.json + formats/vmw_py_log.json formats/xmlrpc_log.json) set(FORMAT_FILE_PATHS ${FORMAT_FILES}) diff --git a/src/document.sections.cc b/src/document.sections.cc index 3ba7154b8b..53ff63df6f 100644 --- a/src/document.sections.cc +++ b/src/document.sections.cc @@ -243,30 +243,49 @@ class structure_walker { } case DT_LCURLY: case DT_LSQUARE: - case DT_LPAREN: + case DT_LPAREN: { this->flush_values(); + // this->append_child_node(term); this->sw_depth += 1; - if (!this->sw_interval_state.back().is_start) { - this->sw_interval_state.back().is_start - = el.e_capture.c_begin; - this->sw_interval_state.back().is_line_number - = this->sw_line_number; - } + this->sw_interval_state.back().is_start + = el.e_capture.c_begin; + this->sw_interval_state.back().is_line_number + = this->sw_line_number; this->sw_interval_state.resize(this->sw_depth + 1); this->sw_hier_nodes.push_back( std::make_unique()); break; + } case DT_RCURLY: case DT_RSQUARE: case DT_RPAREN: { auto term = this->flush_values(); if (this->sw_depth > 0) { - this->sw_depth -= 1; this->append_child_node(term); + this->sw_depth -= 1; this->sw_interval_state.pop_back(); this->sw_hier_stage = std::move(this->sw_hier_nodes.back()); this->sw_hier_nodes.pop_back(); + if (this->sw_interval_state.back().is_start) { + pcre_context::capture_t obj_cap = { + static_cast(this->sw_interval_state.back() + .is_start.value()), + el.e_capture.c_end, + }; + + auto sf = pi.get_string_fragment(&obj_cap); + if (!sf.find('\n')) { + this->sw_hier_stage->hn_named_children.clear(); + this->sw_hier_stage->hn_children.clear(); + while (!this->sw_intervals.empty() + && this->sw_intervals.back().start + > obj_cap.c_begin) + { + this->sw_intervals.pop_back(); + } + } + } } this->sw_values.emplace_back(el); break; @@ -383,21 +402,24 @@ class structure_walker { void append_child_node(nonstd::optional terminator) { auto& ivstate = this->sw_interval_state.back(); - if (!ivstate.is_start || !terminator) { + if (!ivstate.is_start || !terminator || this->sw_depth == 0) { + ivstate.is_start = nonstd::nullopt; + ivstate.is_line_number = 0; + ivstate.is_name.clear(); return; } + const auto& pi = this->sw_scanner.get_input().get_string(); + auto new_node = this->sw_hier_stage != nullptr + ? std::move(this->sw_hier_stage) + : std::make_unique(); + auto iv_start = ivstate.is_start.value(); + auto iv_stop = static_cast(terminator.value().c_end); auto* top_node = this->sw_hier_nodes.back().get(); auto new_key = ivstate.is_name.empty() ? lnav::document::section_key_t{top_node->hn_children.size()} : lnav::document::section_key_t{ivstate.is_name}; - this->sw_intervals.emplace_back( - ivstate.is_start.value(), - static_cast(terminator.value().c_end), - new_key); - auto new_node = this->sw_hier_stage != nullptr - ? std::move(this->sw_hier_stage) - : std::make_unique(); + this->sw_intervals.emplace_back(iv_start, iv_stop, new_key); auto* retval = new_node.get(); new_node->hn_parent = top_node; new_node->hn_start = this->sw_intervals.back().start; diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index 123438c672..7a9358fa58 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -2226,7 +2226,7 @@ logfile_sub_source::text_crumbs_for_line(int line, vis_line_t(line_from_top + line_number)); }; }); - if (curr_node && curr_node.value()->hn_parent->is_named_only()) + if (curr_node && !curr_node.value()->hn_parent->is_named_only()) { auto node = lnav::document::hier_node::lookup_path( meta->m_sections_root.get(), path); diff --git a/src/readline_possibilities.cc b/src/readline_possibilities.cc index 7d830139c7..3998b6aa4f 100644 --- a/src/readline_possibilities.cc +++ b/src/readline_possibilities.cc @@ -414,11 +414,11 @@ add_config_possibilities() const std::string& path, void* mem) { if (jph.jph_children) { - if (!jph.jph_regex.p_named_count) { + if (!jph.jph_regex->p_named_count) { rc->add_possibility(ln_mode_t::COMMAND, "config-option", path); } - for (auto named_iter = jph.jph_regex.named_begin(); - named_iter != jph.jph_regex.named_end(); + for (auto named_iter = jph.jph_regex->named_begin(); + named_iter != jph.jph_regex->named_end(); ++named_iter) { if (visited.count(named_iter->pnc_name) == 0) { diff --git a/src/yajlpp/yajlpp.cc b/src/yajlpp/yajlpp.cc index 773678fd9b..2f137f5893 100644 --- a/src/yajlpp/yajlpp.cc +++ b/src/yajlpp/yajlpp.cc @@ -46,7 +46,8 @@ json_path_handler_base::json_path_handler_base(const std::string& property) : jph_property(property.back() == '#' ? property.substr(0, property.size() - 1) : property), - jph_regex(pcrepp::quote(property), PCRE_ANCHORED), + jph_regex( + std::make_shared(pcrepp::quote(property), PCRE_ANCHORED)), jph_is_array(property.back() == '#') { memset(&this->jph_callbacks, 0, sizeof(this->jph_callbacks)); @@ -61,7 +62,8 @@ scrub_pattern(const std::string& pattern) } json_path_handler_base::json_path_handler_base(const pcrepp& property) - : jph_property(scrub_pattern(property.p_pattern)), jph_regex(property), + : jph_property(scrub_pattern(property.p_pattern)), + jph_regex(std::make_shared(property)), jph_is_array(property.p_pattern.back() == '#'), jph_is_pattern_property(true) { @@ -70,12 +72,21 @@ json_path_handler_base::json_path_handler_base(const pcrepp& property) json_path_handler_base::json_path_handler_base(std::string property, const pcrepp& property_re) - : jph_property(std::move(property)), jph_regex(property_re), + : jph_property(std::move(property)), + jph_regex(std::make_shared(property_re)), jph_is_array(property_re.p_pattern.find('#') != std::string::npos) { memset(&this->jph_callbacks, 0, sizeof(this->jph_callbacks)); } +json_path_handler_base::json_path_handler_base( + std::string property, const std::shared_ptr& property_re) + : jph_property(std::move(property)), jph_regex(property_re), + jph_is_array(property_re->p_pattern.find('#') != std::string::npos) +{ + memset(&this->jph_callbacks, 0, sizeof(this->jph_callbacks)); +} + yajl_gen_status json_path_handler_base::gen(yajlpp_gen_context& ygc, yajl_gen handle) const { @@ -125,7 +136,7 @@ json_path_handler_base::gen(yajlpp_gen_context& ygc, yajl_gen handle) const pcre_context_static<30> pc; pcre_input pi(full_path); - this->jph_regex.match(pc, pi); + this->jph_regex->match(pc, pi); ygc.ygc_obj_stack.push(this->jph_obj_provider( {{pc, pi}, yajlpp_provider_context::nindex}, ygc.ygc_obj_stack.top())); @@ -192,7 +203,7 @@ json_path_handler_base::gen_schema(yajlpp_gen_context& ygc) const } if (this->jph_is_pattern_property) { ygc.ygc_path.emplace_back(fmt::format( - FMT_STRING("<{}>"), this->jph_regex.name_for_capture(0))); + FMT_STRING("<{}>"), this->jph_regex->name_for_capture(0))); } else { ygc.ygc_path.emplace_back(this->jph_property); } @@ -202,7 +213,7 @@ json_path_handler_base::gen_schema(yajlpp_gen_context& ygc) const fmt::join(ygc.ygc_path, "/"))); schema.gen("type"); if (this->jph_is_array) { - if (this->jph_regex.p_pattern.find("#?") + if (this->jph_regex->p_pattern.find("#?") == std::string::npos) { schema.gen("array"); } else { @@ -238,7 +249,7 @@ json_path_handler_base::gen_schema(yajlpp_gen_context& ygc) const if (this->jph_is_pattern_property) { ygc.ygc_path.emplace_back(fmt::format( - FMT_STRING("<{}>"), this->jph_regex.name_for_capture(0))); + FMT_STRING("<{}>"), this->jph_regex->name_for_capture(0))); } else { ygc.ygc_path.emplace_back(this->jph_property); } @@ -254,7 +265,7 @@ json_path_handler_base::gen_schema(yajlpp_gen_context& ygc) const schema.gen("type"); if (this->jph_is_array) { - if (this->jph_regex.p_pattern.find("#?") == std::string::npos) { + if (this->jph_regex->p_pattern.find("#?") == std::string::npos) { schema.gen("array"); } else { yajlpp_array type_array(ygc.ygc_handle); @@ -381,9 +392,8 @@ json_path_handler_base::walk( full_path += "/"; } json_path_container dummy{ - json_path_handler(this->jph_property), + json_path_handler(this->jph_property, this->jph_regex), }; - dummy.jpc_children[0].jph_callbacks = this->jph_callbacks; yajlpp_parse_context ypc(POSS_SRC, &dummy); void* child_root = root; @@ -393,7 +403,7 @@ json_path_handler_base::walk( std::string full_path = lpath + "/"; pcre_input pi(full_path); - if (!this->jph_regex.match(ypc.ypc_pcre_context, pi)) { + if (!this->jph_regex->match(ypc.ypc_pcre_context, pi)) { ensure(false); } child_root = this->jph_obj_provider( @@ -592,7 +602,7 @@ yajlpp_parse_context::update_callbacks(const json_path_container* orig_handlers, pi.reset(&this->ypc_path[1 + child_start], 0, this->ypc_path.size() - 2 - child_start); - if (jph.jph_regex.match(this->ypc_pcre_context, pi)) { + if (jph.jph_regex->match(this->ypc_pcre_context, pi)) { pcre_context::capture_t* cap = this->ypc_pcre_context.all(); if (jph.jph_obj_provider) { @@ -1365,6 +1375,7 @@ dump_schema_to(const json_path_container& jpc, ygc.gen_schema(); } + string_fragment yajlpp_gen::to_string_fragment() { diff --git a/src/yajlpp/yajlpp.hh b/src/yajlpp/yajlpp.hh index 710e35ab13..1b34185ef2 100644 --- a/src/yajlpp/yajlpp.hh +++ b/src/yajlpp/yajlpp.hh @@ -162,10 +162,10 @@ struct json_path_handler_base { json_path_handler_base(std::string property, const pcrepp& property_re); - bool is_array() const - { - return this->jph_is_array; - } + json_path_handler_base(std::string property, + const std::shared_ptr& property_re); + + bool is_array() const { return this->jph_is_array; } nonstd::optional to_enum_value(const string_fragment& sf) const; const char* to_enum_string(int value) const; @@ -192,7 +192,7 @@ struct json_path_handler_base { std::vector get_types() const; std::string jph_property; - pcrepp jph_regex; + std::shared_ptr jph_regex; yajl_callbacks jph_callbacks{}; std::function diff --git a/src/yajlpp/yajlpp_def.hh b/src/yajlpp/yajlpp_def.hh index cc6ca3f906..e2097b9ecd 100644 --- a/src/yajlpp/yajlpp_def.hh +++ b/src/yajlpp/yajlpp_def.hh @@ -114,7 +114,15 @@ struct json_path_handler : public json_path_handler_base { } json_path_handler(const std::string& path, const pcrepp& re) - : json_path_handler_base(path, re){}; + : json_path_handler_base(path, re) + { + } + + json_path_handler(const std::string& path, + const std::shared_ptr& re) + : json_path_handler_base(path, re) + { + } json_path_handler& add_cb(int (*null_func)(yajlpp_parse_context*)) { diff --git a/test/document.sections.tests.cc b/test/document.sections.tests.cc index bdb4f7a3a9..dd341e4022 100644 --- a/test/document.sections.tests.cc +++ b/test/document.sections.tests.cc @@ -82,3 +82,53 @@ TEST_CASE("lnav::document::sections::basics") } }); } + +TEST_CASE("lnav::document::sections::empty") +{ + static const std::string INPUT + = R"(SOCKET 1 (10) creating new listening socket on port -1)"; + + auto meta = lnav::document::discover_structure(INPUT); + + meta.m_sections_tree.visit_all([](const auto& intv) { + auto ser = intv.value.match( + [](const std::string& name) { return name; }, + [](const size_t index) { return fmt::format("{}", index); }); + printf("interval %d:%d %s\n", intv.start, intv.stop, ser.c_str()); + }); + lnav::document::hier_node::depth_first( + meta.m_sections_root.get(), [](const auto* node) { + printf("node %p %d\n", node, node->hn_start); + for (const auto& pair : node->hn_named_children) { + printf(" child: %p %s\n", pair.second, pair.first.c_str()); + } + }); +} + +TEST_CASE("lnav::document::sections::sql") +{ + static const std::string INPUT + = R"(2022-06-03T22:05:58.186Z verbose -[35642] [Originator@6876 sub=Default] [VdbStatement]Executing SQL: +--> INSERT INTO PM_CLUSTER_DRAFT_VALIDATION_STATE +--> (draft_id, errors, hosts) VALUES (?::integer, ?::jsonb, ARRAY[]::text[]) +--> ON CONFLICT (draft_id) DO UPDATE +--> SET errors = EXCLUDED.errors, hosts = EXCLUDED.hosts +--> +)"; + + auto meta = lnav::document::discover_structure(INPUT); + + meta.m_sections_tree.visit_all([](const auto& intv) { + auto ser = intv.value.match( + [](const std::string& name) { return name; }, + [](const size_t index) { return fmt::format("{}", index); }); + printf("interval %d:%d %s\n", intv.start, intv.stop, ser.c_str()); + }); + lnav::document::hier_node::depth_first( + meta.m_sections_root.get(), [](const auto* node) { + printf("node %p %d\n", node, node->hn_start); + for (const auto& pair : node->hn_named_children) { + printf(" child: %p %s\n", pair.second, pair.first.c_str()); + } + }); +}