diff --git a/NEWS.md b/NEWS.md index f591f93b..aa77919a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,15 @@ Features: * Files that contain a mixture of log messages from separate services (e.g. docker logs) can now be automatically de-multiplexed into separate files that lnav can digest. +* The `log_opid` column on log vtables can now by `UPDATE`d + so that you can manually set an opid on log messages that + don't have one. Setting an opid allows messages to show + up in the gantt chart view. + +Interface Changes: +* In the Gantt chart view, pressing `ENTER` will focus on + the preview pane so you can scroll through messages + with the selected Op ID. Bug Fixes: * Log messages in formats with custom timestamp formats were diff --git a/src/all_logs_vtab.cc b/src/all_logs_vtab.cc index 69acf495..f85002a9 100644 --- a/src/all_logs_vtab.cc +++ b/src/all_logs_vtab.cc @@ -83,7 +83,7 @@ all_logs_vtab::extract(logfile* lf, this->vi_attrs.clear(); sub_values.lvv_sbr = line.clone(); - format->annotate(line_number, this->vi_attrs, sub_values, false); + format->annotate(lf, line_number, this->vi_attrs, sub_values, false); auto body = find_string_attr_range(this->vi_attrs, &SA_BODY); if (body.lr_start == -1) { @@ -111,6 +111,7 @@ all_logs_vtab::extract(logfile* lf, this->alv_values_meta, json_string(gen).to_string_fragment().to_string()); values.lvv_opid_value = std::move(sub_values.lvv_opid_value); + values.lvv_opid_provenance = sub_values.lvv_opid_provenance; } bool diff --git a/src/base/intern_string.hh b/src/base/intern_string.hh index f5c319bc..df60ee9b 100644 --- a/src/base/intern_string.hh +++ b/src/base/intern_string.hh @@ -45,6 +45,8 @@ #include "scn/util/string_view.h" #include "strnatcmp.h" +unsigned long hash_str(const char* str, size_t len); + struct string_fragment { using iterator = const char*; @@ -637,6 +639,11 @@ struct string_fragment { std::string to_string_with_case_style(case_style style) const; + unsigned long hash() const + { + return hash_str(this->data(), this->length()); + } + const char* sf_string; int sf_begin; int sf_end; @@ -805,8 +812,6 @@ private: const intern_string* ist_interned_string; }; -unsigned long hash_str(const char* str, size_t len); - namespace fmt { template<> struct formatter : formatter { diff --git a/src/base/time_util.hh b/src/base/time_util.hh index 1e5b7982..1b82be37 100644 --- a/src/base/time_util.hh +++ b/src/base/time_util.hh @@ -255,6 +255,16 @@ struct time_range { struct timeval tr_begin; struct timeval tr_end; + bool valid() const { return this->tr_end.tv_sec == 0; } + + void invalidate() + { + this->tr_begin.tv_sec = INT_MAX; + this->tr_begin.tv_usec = 0; + this->tr_end.tv_sec = 0; + this->tr_end.tv_usec = 0; + } + bool operator<(const time_range& rhs) const { return this->tr_begin < rhs.tr_begin; diff --git a/src/bookmarks.cc b/src/bookmarks.cc index 043574e0..c94739d2 100644 --- a/src/bookmarks.cc +++ b/src/bookmarks.cc @@ -68,20 +68,23 @@ bookmark_metadata::empty(bookmark_metadata::categories props) const { switch (props) { case categories::any: - return this->bm_name.empty() && this->bm_comment.empty() - && this->bm_tags.empty() + return this->bm_name.empty() && this->bm_opid.empty() + && this->bm_comment.empty() && this->bm_tags.empty() && this->bm_annotations.la_pairs.empty(); case categories::partition: return this->bm_name.empty(); case categories::notes: return this->bm_comment.empty() && this->bm_tags.empty() && this->bm_annotations.la_pairs.empty(); + case categories::opid: + return this->bm_opid.empty(); } } void bookmark_metadata::clear() { + this->bm_opid.clear(); this->bm_comment.clear(); this->bm_tags.clear(); this->bm_annotations.la_pairs.clear(); diff --git a/src/bookmarks.hh b/src/bookmarks.hh index 99b0b714..63e1a854 100644 --- a/src/bookmarks.hh +++ b/src/bookmarks.hh @@ -51,6 +51,7 @@ struct bookmark_metadata { any = 0, partition = 0x01, notes = 0x02, + opid = 0x04, }; bool has(categories props) const @@ -71,10 +72,15 @@ struct bookmark_metadata { return true; } + if (props == categories::opid && !this->bm_opid.empty()) { + return true; + } + return false; } std::string bm_name; + std::string bm_opid; std::string bm_comment; logmsg_annotations bm_annotations; std::vector bm_tags; @@ -177,8 +183,7 @@ public: static type_iterator type_end() { return get_all_types().end(); } - static std::optional find_type( - const std::string& name); + static std::optional find_type(const std::string& name); static std::vector& get_all_types(); diff --git a/src/field_overlay_source.cc b/src/field_overlay_source.cc index 2791307a..1689107d 100644 --- a/src/field_overlay_source.cc +++ b/src/field_overlay_source.cc @@ -536,6 +536,14 @@ field_overlay_source::build_meta_line(const listview_curses& lv, const auto& line_meta = *(line_meta_opt.value()); size_t filename_width = this->fos_lss.get_filename_offset(); + if (!line_meta.bm_opid.empty()) { + auto al = attr_line_t() + .append(" Op ID: "_table_header) + .append(lnav::roles::identifier(line_meta.bm_opid)); + + dst.emplace_back(al); + } + if (!line_meta.bm_comment.empty()) { const auto* lead = line_meta.bm_tags.empty() ? " \u2514 " : " \u251c "; md2attr_line mdal; diff --git a/src/formats/glog_log.json b/src/formats/glog_log.json index 09fcff4d..29f0b3f9 100644 --- a/src/formats/glog_log.json +++ b/src/formats/glog_log.json @@ -20,7 +20,6 @@ "critical": "C", "fatal": "F" }, - "opid-field": "thread", "value": { "thread": { "kind": "integer", diff --git a/src/formats/zookeeper_log.json b/src/formats/zookeeper_log.json index a3b74a52..94d3f59a 100644 --- a/src/formats/zookeeper_log.json +++ b/src/formats/zookeeper_log.json @@ -33,6 +33,20 @@ "zk_notifications": { "level": "info", "pattern": "Notification: my state:(?\\w+); (?.*)" + }, + "zk_commit_session_id": { + "level": "info", + "pattern": "Committing global session (?0x[a-zA-Z0-9]+)" + } + }, + "tags": { + "zk-election-started": { + "description": "Tag for the start of a new election", + "pattern": "New election. " + }, + "zk-election-finished": { + "description": "Tag for the completion of an election", + "pattern": "\\w+ - LEADER ELECTION TOOK - " } }, "sample": [ diff --git a/src/gantt_source.cc b/src/gantt_source.cc index 42c735a0..aa5c94f9 100644 --- a/src/gantt_source.cc +++ b/src/gantt_source.cc @@ -340,25 +340,74 @@ gantt_header_overlay::list_header_for_overlay(const listview_curses& lv, gantt_source::gantt_source(textview_curses& log_view, logfile_sub_source& lss, + textview_curses& preview_view, plain_text_source& preview_source, gantt_status_source& preview_status_source) - : gs_log_view(log_view), gs_lss(lss), gs_preview_source(preview_source), + : gs_log_view(log_view), gs_lss(lss), gs_preview_view(preview_view), + gs_preview_source(preview_source), gs_preview_status_source(preview_status_source) { this->tss_supports_filtering = true; } +bool +gantt_source::list_input_handle_key(listview_curses& lv, int ch) +{ + switch (ch) { + case 'q': + case KEY_ESCAPE: { + if (this->gs_preview_focused) { + this->gs_preview_focused = false; + this->gs_preview_view.set_height(5_vl); + } + break; + } + case '\n': + case '\r': + case KEY_ENTER: { + this->gs_preview_focused = !this->gs_preview_focused; + if (this->gs_preview_focused) { + auto height = this->tss_view->get_dimensions().first; + + if (height > 5) { + this->gs_preview_view.set_height(height - 3_vl); + } + } else { + this->gs_preview_view.set_height(5_vl); + } + return true; + } + } + if (this->gs_preview_focused) { + return this->gs_preview_view.handle_key(ch); + } + + return false; +} + +bool +gantt_source::text_handle_mouse(textview_curses& tc, + const listview_curses::display_line_content_t&, + mouse_event& me) +{ + if (me.is_double_click_in(mouse_button_t::BUTTON_LEFT, line_range{0, -1})) { + this->list_input_handle_key(tc, '\r'); + } + + return false; +} + std::pair gantt_source::get_time_bounds_for(int line) { - static const int CONTEXT_LINES = 5; - const auto& low_row - = this->gs_time_order[std::max(0, line - CONTEXT_LINES)].get(); + = this->gs_time_order[std::max(0_vl, this->tss_view->get_top())].get(); const auto& sel_row = this->gs_time_order[line].get(); const auto& high_row - = this->gs_time_order[std::min(line + CONTEXT_LINES, - (int) this->gs_time_order.size() - 1)] + = this + ->gs_time_order[std::min( + this->tss_view->get_bottom(), + vis_line_t((int) this->gs_time_order.size() - 1))] .get(); auto high_tv_sec = std::max(sel_row.or_value.otr_range.tr_end.tv_sec, high_row.or_value.otr_range.tr_begin.tv_sec); @@ -813,7 +862,7 @@ gantt_source::text_line_width(textview_curses& curses) void gantt_source::text_selection_changed(textview_curses& tc) { - static const size_t MAX_PREVIEW_LINES = 5; + static const size_t MAX_PREVIEW_LINES = 200; auto sel = tc.get_selection(); @@ -847,7 +896,7 @@ gantt_source::text_selection_changed(textview_curses& tc) auto preview_content = attr_line_t(); auto msgs_remaining = size_t{MAX_PREVIEW_LINES}; auto win = this->gs_lss.window_at(low_vl.value(), high_vl); - auto id_hash = hash_str(row.or_name.data(), row.or_name.length()); + auto id_hash = row.or_name.hash(); for (const auto& msg_line : win) { if (!msg_line.get_logline().match_opid_hash(id_hash)) { continue; @@ -860,12 +909,14 @@ gantt_source::text_selection_changed(textview_curses& tc) auto opid_sf = lvv.lvv_opid_value.value(); if (opid_sf == row.or_name) { - std::vector rows_al(1); + std::vector rows_al(msg_line.get_line_count()); this->gs_log_view.listview_value_for_rows( this->gs_log_view, msg_line.get_vis_line(), rows_al); - preview_content.append(rows_al[0]).append("\n"); + for (const auto& row_al : rows_al) { + preview_content.append(row_al).append("\n"); + } msgs_remaining -= 1; if (msgs_remaining == 0) { break; @@ -873,12 +924,8 @@ gantt_source::text_selection_changed(textview_curses& tc) } } - while (msgs_remaining > 0) { - preview_content.append("\u2800\n"); - msgs_remaining -= 1; - } - this->gs_preview_source.replace_with(preview_content); + this->gs_preview_view.set_top(0_vl); this->gs_preview_status_source.get_description().set_value( " ID %.*s", id_sf.length(), id_sf.data()); auto err_count = level_stats.lls_error_count; diff --git a/src/gantt_source.hh b/src/gantt_source.hh index b9a6e020..01bf4a24 100644 --- a/src/gantt_source.hh +++ b/src/gantt_source.hh @@ -39,13 +39,22 @@ class gantt_source : public text_sub_source - , public text_time_translator { + , public list_input_delegate + , public text_time_translator + , public text_delegate { public: explicit gantt_source(textview_curses& log_view, logfile_sub_source& lss, + textview_curses& preview_view, plain_text_source& preview_source, gantt_status_source& preview_status_source); + bool list_input_handle_key(listview_curses& lv, int ch) override; + + bool text_handle_mouse(textview_curses& tc, + const listview_curses::display_line_content_t&, + mouse_event& me) override; + size_t text_line_count() override; size_t text_line_width(textview_curses& curses) override; @@ -72,8 +81,7 @@ public: void text_crumbs_for_line(int line, std::vector& crumbs) override; - std::optional row_for_time( - struct timeval time_bucket) override; + std::optional row_for_time(struct timeval time_bucket) override; std::optional time_for_row(vis_line_t row) override; void rebuild_indexes(); @@ -82,6 +90,7 @@ public: textview_curses& gs_log_view; logfile_sub_source& gs_lss; + textview_curses& gs_preview_view; plain_text_source& gs_preview_source; gantt_status_source& gs_preview_status_source; ArenaAlloc::Alloc gs_allocator{64 * 1024}; @@ -163,6 +172,7 @@ public: size_t gs_filtered_count{0}; std::array gs_filter_hits{}; exec_context* gs_exec_context; + bool gs_preview_focused{false}; }; class gantt_header_overlay : public text_overlay_menu { diff --git a/src/hotkeys.cc b/src/hotkeys.cc index 742998f4..b2cdd443 100644 --- a/src/hotkeys.cc +++ b/src/hotkeys.cc @@ -48,58 +48,6 @@ using namespace lnav::roles::literals; -class logline_helper { -public: - explicit logline_helper(logfile_sub_source& lss) : lh_sub_source(lss) {} - - logline& move_to_msg_start() - { - content_line_t cl = this->lh_sub_source.at(this->lh_current_line); - std::shared_ptr lf = this->lh_sub_source.find(cl); - auto ll = lf->begin() + cl; - while (!ll->is_message()) { - --ll; - --this->lh_current_line; - } - - return (*lf)[cl]; - } - - logline& current_line() const - { - content_line_t cl = this->lh_sub_source.at(this->lh_current_line); - std::shared_ptr lf = this->lh_sub_source.find(cl); - - return (*lf)[cl]; - } - - void annotate() - { - this->lh_string_attrs.clear(); - this->lh_line_values.clear(); - content_line_t cl = this->lh_sub_source.at(this->lh_current_line); - auto lf = this->lh_sub_source.find(cl); - auto ll = lf->begin() + cl; - auto format = lf->get_format(); - lf->read_full_message(ll, this->lh_line_values.lvv_sbr); - this->lh_line_values.lvv_sbr.erase_ansi(); - format->annotate( - cl, this->lh_string_attrs, this->lh_line_values, false); - } - - std::string to_string(const struct line_range& lr) const - { - const char* start = this->lh_line_values.lvv_sbr.get_data(); - - return {&start[lr.lr_start], (size_t) lr.length()}; - } - - logfile_sub_source& lh_sub_source; - vis_line_t lh_current_line; - string_attrs_t lh_string_attrs; - logline_value_vector lh_line_values; -}; - static int key_sql_callback(exec_context& ec, sqlite3_stmt* stmt) { @@ -654,14 +602,10 @@ DELETE FROM lnav_user_notifications WHERE id = 'org.lnav.mouse-support' case 'o': case 'O': if (lss != nullptr) { - logline_helper start_helper(*lss); - - start_helper.lh_current_line = tc->get_selection(); - auto& start_line = start_helper.move_to_msg_start(); - start_helper.annotate(); - + auto start_win = lss->window_at(tc->get_selection()); + auto start_win_iter = start_win.begin(); const auto& opid_opt - = start_helper.lh_line_values.lvv_opid_value; + = start_win_iter->get_values().lvv_opid_value; if (!opid_opt) { alerter::singleton().chime( "Log message does not contain an opid"); @@ -670,36 +614,33 @@ DELETE FROM lnav_user_notifications WHERE id = 'org.lnav.mouse-support' "Log message does not contain an opid") .to_attr_line()); } else { + const auto& start_line = start_win_iter->get_logline(); unsigned int opid_hash = start_line.get_opid(); - logline_helper next_helper(*lss); + auto next_win + = lss->window_to_end(start_win_iter->get_vis_line()); + auto next_win_iter = next_win.begin(); bool found = false; - next_helper.lh_current_line = start_helper.lh_current_line; - while (true) { if (ch == 'o') { - if (++next_helper.lh_current_line - >= tc->get_inner_height()) - { + ++next_win_iter; + if (next_win_iter == next_win.end()) { break; } } else { - if (--next_helper.lh_current_line <= 0) { + if (next_win_iter->get_vis_line() == 0) { break; } + --next_win_iter; } - logline& next_line = next_helper.current_line(); - if (!next_line.is_message()) { - continue; - } - if (next_line.get_opid() != opid_hash) { + const auto& next_line = next_win_iter->get_logline(); + if (!next_line.match_opid_hash(opid_hash)) { continue; } - next_helper.annotate(); const auto& next_opid_opt - = next_helper.lh_line_values.lvv_opid_value; - if (next_opid_opt - && opid_opt.value() != next_opid_opt.value()) + = next_win_iter->get_values().lvv_opid_value; + if (!next_opid_opt + || opid_opt.value() != next_opid_opt.value()) { continue; } @@ -708,7 +649,7 @@ DELETE FROM lnav_user_notifications WHERE id = 'org.lnav.mouse-support' } if (found) { lnav_data.ld_rl_view->set_value(""); - tc->set_selection(next_helper.lh_current_line); + tc->set_selection(next_win_iter->get_vis_line()); } else { lnav_data.ld_rl_view->set_attr_value( lnav::console::user_message::error( diff --git a/src/listview_curses.cc b/src/listview_curses.cc index 108ec17e..1a2bf8f4 100644 --- a/src/listview_curses.cc +++ b/src/listview_curses.cc @@ -792,7 +792,21 @@ listview_curses::shift_selection(shift_amount_t sa) } } } - this->set_selection(new_selection); + + this->set_selection_without_context(new_selection); + auto rows_avail = this->rows_available(this->lv_top, RD_DOWN); + if (height > rows_avail) { + rows_avail = height; + } + if (this->lv_selection > 0 && this->lv_selection <= this->lv_top) { + this->set_top(this->lv_selection - 1_vl); + } else if (rows_avail > this->lv_tail_space + && (this->lv_selection > (this->lv_top + (rows_avail - 1_vl) + - this->lv_tail_space))) + { + this->set_top(this->lv_selection + this->lv_tail_space + - (rows_avail - 1_vl)); + } } else { this->shift_top(vis_line_t{offset}); } @@ -1095,9 +1109,21 @@ listview_curses::set_selection_without_context(vis_line_t sel) void listview_curses::set_selection(vis_line_t sel) { + auto dim = this->get_dimensions(); + auto diff = std::optional{}; + + if (this->lv_selection >= 0_vl && this->lv_selection > this->lv_top + && this->lv_selection < this->lv_top + dim.first) + { + diff = this->lv_selection - this->lv_top; + } this->set_selection_without_context(sel); - auto dim = this->get_dimensions(); + if (diff) { + auto new_top = std::max(0_vl, this->lv_selection - diff.value()); + this->set_top(new_top); + } + if (this->lv_selection > 0 && this->lv_selection <= this->lv_top) { this->set_top(this->lv_selection - 1_vl); } else if (dim.first > this->lv_tail_space diff --git a/src/lnav.cc b/src/lnav.cc index 33742b66..7a634fd9 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -1373,7 +1373,7 @@ VALUES ('org.lnav.mouse-support', -1, DATETIME('now', '+1 minute'), lnav_data.ld_gantt_details_view.set_title("gantt-details"); lnav_data.ld_gantt_details_view.set_window(lnav_data.ld_window); - lnav_data.ld_gantt_details_view.set_show_scrollbar(false); + lnav_data.ld_gantt_details_view.set_show_scrollbar(true); lnav_data.ld_gantt_details_view.set_height(5_vl); lnav_data.ld_gantt_details_view.set_sub_source( &lnav_data.ld_gantt_details_source); @@ -2846,6 +2846,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' auto gantt_view_source = std::make_shared(lnav_data.ld_views[LNV_LOG], lnav_data.ld_log_source, + lnav_data.ld_gantt_details_view, lnav_data.ld_gantt_details_source, lnav_data.ld_gantt_status_source); gantt_view_source->gs_exec_context = &lnav_data.ld_exec_context; @@ -2854,6 +2855,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' lnav_data.ld_views[LNV_GANTT] .set_sub_source(gantt_view_source.get()) .set_overlay_source(gantt_header_source.get()) + .add_input_delegate(*gantt_view_source) .set_tail_space(4_vl); lnav_data.ld_views[LNV_GANTT].set_selectable(true); diff --git a/src/lnav_config.cc b/src/lnav_config.cc index 50712b01..b8cc43bf 100644 --- a/src/lnav_config.cc +++ b/src/lnav_config.cc @@ -523,6 +523,7 @@ static const struct json_path_container keymap_def_handlers = { if (ypc.ypc_parse_context != nullptr) { retval.kc_cmd.pp_path = ypc.ypc_parse_context->get_full_path(); + retval.kc_cmd.pp_location.sl_source = ypc.ypc_parse_context->ypc_source; retval.kc_cmd.pp_location.sl_line_number @@ -1906,6 +1907,7 @@ reset_config(const std::string& path) yajlpp_provider_context provider_ctx{&md, static_cast(-1)}; jph->jph_regex->capture_from(path_frag).into(md).matches(); + ypc.ypc_obj_stack.pop(); jph->jph_obj_deleter(provider_ctx, ypc.ypc_obj_stack.top()); } } diff --git a/src/log.watch.cc b/src/log.watch.cc index c4e5826c..2eb5e927 100644 --- a/src/log.watch.cc +++ b/src/log.watch.cc @@ -128,7 +128,7 @@ eval_with(logfile& lf, logfile::iterator ll) auto format = lf.get_format(); string_attrs_t sa; auto line_number = std::distance(lf.begin(), ll); - format->annotate(line_number, sa, values); + format->annotate(&lf, line_number, sa, values); for (auto& watch_pair : exprs.e_watch_exprs) { if (!watch_pair.second.cwe_enabled) { diff --git a/src/log_data_helper.cc b/src/log_data_helper.cc index 28288830..bda760da 100644 --- a/src/log_data_helper.cc +++ b/src/log_data_helper.cc @@ -81,7 +81,10 @@ log_data_helper::parse_line(content_line_t line, bool allow_middle) this->ldh_line_values.clear(); this->ldh_file->read_full_message(ll, this->ldh_line_values.lvv_sbr); this->ldh_line_values.lvv_sbr.erase_ansi(); - format->annotate(this->ldh_line_index, sa, this->ldh_line_values); + format->annotate(this->ldh_file.get(), + this->ldh_line_index, + sa, + this->ldh_line_values); body = find_string_attr_range(sa, &SA_BODY); if (body.lr_start == -1) { diff --git a/src/log_data_table.cc b/src/log_data_table.cc index f0e05e8c..e7949cb3 100644 --- a/src/log_data_table.cc +++ b/src/log_data_table.cc @@ -65,7 +65,7 @@ log_data_table::get_columns_int() } lf->read_full_message(lf->begin() + cl_copy, line_values.lvv_sbr); line_values.lvv_sbr.erase_ansi(); - format->annotate(cl_copy, sa, line_values, false); + format->annotate(lf.get(), cl_copy, sa, line_values, false); body = find_string_attr_range(sa, &SA_BODY); if (body.lr_end == -1) { this->ldt_schema_id.clear(); @@ -142,7 +142,7 @@ log_data_table::next(log_cursor& lc, logfile_sub_source& lss) lf->read_full_message(lf_iter, line_values.lvv_sbr); line_values.lvv_sbr.erase_ansi(); - lf->get_format()->annotate(cl, sa, line_values, false); + lf->get_format()->annotate(lf, cl, sa, line_values, false); body = find_string_attr_range(sa, &SA_BODY); if (body.lr_end == -1) { return false; diff --git a/src/log_format.cc b/src/log_format.cc index 8a80d91d..a4c9c005 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -98,6 +98,14 @@ log_op_description::operator|=(const log_op_description& rhs) return *this; } +void +opid_time_range::clear() +{ + this->otr_range.invalidate(); + this->otr_sub_ops.clear(); + this->otr_level_stats = {}; +} + opid_time_range& opid_time_range::operator|=(const opid_time_range& rhs) { @@ -123,21 +131,21 @@ opid_time_range::operator|=(const opid_time_range& rhs) } void -log_level_stats::update_msg_count(log_level_t lvl) +log_level_stats::update_msg_count(log_level_t lvl, int32_t amount) { switch (lvl) { case LEVEL_FATAL: case LEVEL_CRITICAL: case LEVEL_ERROR: - this->lls_error_count += 1; + this->lls_error_count += amount; break; case LEVEL_WARNING: - this->lls_warning_count += 1; + this->lls_warning_count += amount; break; default: break; } - this->lls_total_count += 1; + this->lls_total_count += amount; } void @@ -150,6 +158,24 @@ opid_time_range::close_sub_ops(const string_fragment& subid) } } +log_opid_map::iterator +log_opid_state::insert_op(ArenaAlloc::Alloc& alloc, + const string_fragment& opid, + const struct timeval& log_tv) +{ + auto retval = this->los_opid_ranges.find(opid); + if (retval == this->los_opid_ranges.end()) { + auto opid_copy = opid.to_owned(alloc); + auto otr = opid_time_range{time_range{log_tv, log_tv}}; + auto emplace_res = this->los_opid_ranges.emplace(opid_copy, otr); + retval = emplace_res.first; + } else { + retval->second.otr_range.extend_to(log_tv); + } + + return retval; +} + opid_sub_time_range* log_opid_state::sub_op_in_use(ArenaAlloc::Alloc& alloc, log_opid_map::iterator& op_iter, @@ -723,6 +749,24 @@ log_format::log_scanf(uint32_t line_number, return retval; } +void +log_format::annotate(logfile* lf, + uint64_t line_number, + string_attrs_t& sa, + logline_value_vector& values, + bool annotate_module) const +{ + if (lf != nullptr && !values.lvv_opid_value) { + const auto& bm = lf->get_bookmark_metadata(); + auto bm_iter = bm.find(line_number); + if (bm_iter != bm.end() && !bm_iter->second.bm_opid.empty()) { + values.lvv_opid_value = bm_iter->second.bm_opid; + values.lvv_opid_provenance + = logline_value_vector::opid_provenance::user; + } + } +} + void log_format::check_for_new_year(std::vector& dst, exttm etm, @@ -1249,19 +1293,12 @@ external_log_format::scan(logfile& lf, if (jlu.jlu_opid_frag) { this->jlf_line_values.lvv_opid_value = jlu.jlu_opid_frag->to_string(); - auto opid_iter = sbc.sbc_opids.los_opid_ranges.find( - jlu.jlu_opid_frag.value()); - if (opid_iter == sbc.sbc_opids.los_opid_ranges.end()) { - auto otr = opid_time_range{ - time_range{ll.get_timeval(), ll.get_timeval()}, - }; - auto emplace_res = sbc.sbc_opids.los_opid_ranges.emplace( - jlu.jlu_opid_frag.value(), otr); - opid_iter = emplace_res.first; - } else { - opid_iter->second.otr_range.extend_to(ll.get_timeval()); - } - + this->jlf_line_values.lvv_opid_provenance + = logline_value_vector::opid_provenance::file; + auto opid_iter + = sbc.sbc_opids.insert_op(sbc.sbc_allocator, + jlu.jlu_opid_frag.value(), + ll.get_timeval()); opid_iter->second.otr_level_stats.update_msg_count( ll.get_msg_level()); @@ -1456,19 +1493,8 @@ external_log_format::scan(logfile& lf, } if (opid_cap && !opid_cap->empty()) { - auto opid_iter - = sbc.sbc_opids.los_opid_ranges.find(opid_cap.value()); - - if (opid_iter == sbc.sbc_opids.los_opid_ranges.end()) { - auto opid_copy = opid_cap->to_owned(sbc.sbc_allocator); - auto otr = opid_time_range{time_range{log_tv, log_tv}}; - auto emplace_res - = sbc.sbc_opids.los_opid_ranges.emplace(opid_copy, otr); - opid_iter = emplace_res.first; - } else { - opid_iter->second.otr_range.extend_to(log_tv); - } - + auto opid_iter = sbc.sbc_opids.insert_op( + sbc.sbc_allocator, opid_cap.value(), log_tv); auto& otr = opid_iter->second; otr.otr_level_stats.update_msg_count(level); @@ -1493,7 +1519,7 @@ external_log_format::scan(logfile& lf, } this->update_op_description( *this->lf_opid_description_def, otr.otr_description, fpat, md); - opid = hash_str(opid_cap->data(), opid_cap->length()); + opid = opid_cap->hash(); } if (mod_cap) { @@ -1697,7 +1723,8 @@ external_log_format::module_scan(string_fragment body_cap, } void -external_log_format::annotate(uint64_t line_number, +external_log_format::annotate(logfile* lf, + uint64_t line_number, string_attrs_t& sa, logline_value_vector& values, bool annotate_module) const @@ -1730,7 +1757,11 @@ external_log_format::annotate(uint64_t line_number, -this->jlf_cached_sub_range.lr_start); } } + values.lvv_opid_value = this->jlf_line_values.lvv_opid_value; + values.lvv_opid_provenance + = this->jlf_line_values.lvv_opid_provenance; } + log_format::annotate(lf, line_number, sa, values, annotate_module); return; } @@ -1785,6 +1816,8 @@ external_log_format::annotate(uint64_t line_number, sa.emplace_back(to_line_range(opid_cap.value()), logline::L_OPID.value()); values.lvv_opid_value = opid_cap->to_string(); + values.lvv_opid_provenance + = logline_value_vector::opid_provenance::file; } } @@ -1838,7 +1871,7 @@ external_log_format::annotate(uint64_t line_number, = line.narrow(body_cap->sf_begin, body_cap->length()); auto pre_mod_values_size = values.lvv_values.size(); auto pre_mod_sa_size = sa.size(); - mf.mf_mod_format->annotate(line_number, sa, values, false); + mf.mf_mod_format->annotate(lf, line_number, sa, values, false); for (size_t lpc = pre_mod_values_size; lpc < values.lvv_values.size(); lpc++) @@ -1861,6 +1894,8 @@ external_log_format::annotate(uint64_t line_number, } sa.emplace_back(lr, SA_BODY.value()); } + + log_format::annotate(lf, line_number, sa, values, annotate_module); } void @@ -2044,6 +2079,8 @@ rewrite_json_field(yajlpp_parse_context* ypc, if (jlu->jlu_format->elf_opid_field == field_name) { auto frag = string_fragment::from_bytes(str, len); jlu->jlu_format->jlf_line_values.lvv_opid_value = frag.to_string(); + jlu->jlu_format->jlf_line_values.lvv_opid_provenance + = logline_value_vector::opid_provenance::file; } if (jlu->jlu_format->lf_timestamp_field == field_name) { char time_buf[64]; @@ -2235,6 +2272,8 @@ external_log_format::get_subline(const logline& ll, if (jlu.jlu_opid_frag) { this->jlf_line_values.lvv_opid_value = jlu.jlu_opid_frag->to_string(); + this->jlf_line_values.lvv_opid_provenance + = logline_value_vector::opid_provenance::file; } int sub_offset = this->jlf_line_format_init_count; @@ -3884,14 +3923,14 @@ public: auto format = lf->get_format(); return lf->read_line(lf_iter) - .map([this, format, cl](auto line) { + .map([this, format, cl, lf](auto line) { logline_value_vector values; struct line_range mod_name_range; intern_string_t mod_name; this->vi_attrs.clear(); values.lvv_sbr = line.clone(); - format->annotate(cl, this->vi_attrs, values, false); + format->annotate(lf, cl, this->vi_attrs, values, false); this->elt_container_body = find_string_attr_range(this->vi_attrs, &SA_BODY); if (!this->elt_container_body.is_valid()) { @@ -3940,11 +3979,11 @@ public: this->elt_container_body.length()); values.lvv_values.clear(); this->elt_module_format.mf_mod_format->annotate( - line_number, this->vi_attrs, values, false); + lf, line_number, this->vi_attrs, values, false); values.lvv_sbr.widen(narrow_res); } else { this->vi_attrs.clear(); - format->annotate(line_number, this->vi_attrs, values, false); + format->annotate(lf, line_number, this->vi_attrs, values, false); } } diff --git a/src/log_format.hh b/src/log_format.hh index dff2a2a5..94cc2469 100644 --- a/src/log_format.hh +++ b/src/log_format.hh @@ -275,18 +275,26 @@ public: }; struct logline_value_vector { + enum class opid_provenance { + none, + file, + user, + }; + void clear() { this->lvv_values.clear(); this->lvv_sbr.disown(); this->lvv_opid_value = std::nullopt; + this->lvv_opid_provenance = opid_provenance::none; } - logline_value_vector() {} + logline_value_vector() = default; logline_value_vector(const logline_value_vector& other) : lvv_sbr(other.lvv_sbr.clone()), lvv_values(other.lvv_values), - lvv_opid_value(other.lvv_opid_value) + lvv_opid_value(other.lvv_opid_value), + lvv_opid_provenance(other.lvv_opid_provenance) { } @@ -295,6 +303,7 @@ struct logline_value_vector { this->lvv_sbr = other.lvv_sbr.clone(); this->lvv_values = other.lvv_values; this->lvv_opid_value = other.lvv_opid_value; + this->lvv_opid_provenance = other.lvv_opid_provenance; return *this; } @@ -302,6 +311,7 @@ struct logline_value_vector { shared_buffer_ref lvv_sbr; std::vector lvv_values; std::optional lvv_opid_value; + opid_provenance lvv_opid_provenance{opid_provenance::none}; }; struct logline_value_stats { @@ -442,12 +452,11 @@ public: */ virtual void scrub(std::string& line) {} - virtual void annotate(uint64_t line_number, + virtual void annotate(logfile* lf, + uint64_t line_number, string_attrs_t& sa, logline_value_vector& values, - bool annotate_module = true) const - { - } + bool annotate_module = true) const; virtual void rewrite(exec_context& ec, shared_buffer_ref& line, diff --git a/src/log_format_ext.hh b/src/log_format_ext.hh index b885bb69..1d3f76be 100644 --- a/src/log_format_ext.hh +++ b/src/log_format_ext.hh @@ -151,7 +151,8 @@ public: bool scan_for_partial(shared_buffer_ref& sbr, size_t& len_out) const override; - void annotate(uint64_t line_number, + void annotate(logfile* lf, + uint64_t line_number, string_attrs_t& sa, logline_value_vector& values, bool annotate_module = true) const override; diff --git a/src/log_format_fwd.hh b/src/log_format_fwd.hh index e5725fef..b0f0152d 100644 --- a/src/log_format_fwd.hh +++ b/src/log_format_fwd.hh @@ -54,7 +54,7 @@ struct log_level_stats { uint32_t lls_total_count{0}; log_level_stats& operator|=(const log_level_stats& rhs); - void update_msg_count(log_level_t lvl); + void update_msg_count(log_level_t lvl, int32_t amount = 1); }; struct log_op_description { @@ -83,6 +83,8 @@ struct opid_time_range { log_op_description otr_description; std::vector otr_sub_ops; + void clear(); + void close_sub_ops(const string_fragment& subid); opid_time_range& operator|=(const opid_time_range& rhs); @@ -102,6 +104,10 @@ struct log_opid_state { log_opid_map los_opid_ranges; sub_opid_map los_sub_in_use; + log_opid_map::iterator insert_op(ArenaAlloc::Alloc& alloc, + const string_fragment& opid, + const struct timeval& tv); + opid_sub_time_range* sub_op_in_use(ArenaAlloc::Alloc& alloc, log_opid_map::iterator& op_iter, const string_fragment& subid, diff --git a/src/log_format_impls.cc b/src/log_format_impls.cc index 85f0f5b3..46103e72 100644 --- a/src/log_format_impls.cc +++ b/src/log_format_impls.cc @@ -73,13 +73,15 @@ public: return scan_no_match{""}; } - void annotate(uint64_t line_number, + void annotate(logfile* lf, + uint64_t line_number, string_attrs_t& sa, logline_value_vector& values, bool annotate_module) const override { auto lr = line_range{0, 0}; sa.emplace_back(lr, logline::L_TIMESTAMP.value()); + log_format::annotate(lf, line_number, sa, values, annotate_module); } void get_subline(const logline& ll, @@ -255,7 +257,8 @@ public: return scan_no_match{"no patterns matched"}; } - void annotate(uint64_t line_number, + void annotate(logfile* lf, + uint64_t line_number, string_attrs_t& sa, logline_value_vector& values, bool annotate_module) const override @@ -302,6 +305,8 @@ public: lr.lr_start = prefix_len; lr.lr_end = line.length(); sa.emplace_back(lr, SA_BODY.value()); + + log_format::annotate(lf, line_number, sa, values, annotate_module); } std::shared_ptr specialized(int fmt_lock) override @@ -643,18 +648,8 @@ public: } if (opid_cap.is_valid()) { - auto opid_iter = sbc.sbc_opids.los_opid_ranges.find(opid_cap); - - if (opid_iter == sbc.sbc_opids.los_opid_ranges.end()) { - auto opid_copy = opid_cap.to_owned(sbc.sbc_allocator); - auto otr = opid_time_range{time_range{tv, tv}}; - auto emplace_res - = sbc.sbc_opids.los_opid_ranges.emplace(opid_copy, otr); - opid_iter = emplace_res.first; - } else { - opid_iter->second.otr_range.extend_to(tv); - } - + auto opid_iter + = sbc.sbc_opids.insert_op(sbc.sbc_allocator, opid_cap, tv); opid_iter->second.otr_level_stats.update_msg_count(level); auto& otr = opid_iter->second; @@ -854,7 +849,8 @@ public: return scan_no_match{}; } - void annotate(uint64_t line_number, + void annotate(logfile* lf, + uint64_t line_number, string_attrs_t& sa, logline_value_vector& values, bool annotate_module) const override @@ -897,6 +893,8 @@ public: values.lvv_values.back().lv_meta.lvm_user_hidden = fd.fd_root_meta->lvm_user_hidden; } + + log_format::annotate(lf, line_number, sa, values, annotate_module); } const logline_value_stats* stats_for_value( @@ -1523,7 +1521,8 @@ public: return scan_no_match{}; } - void annotate(uint64_t line_number, + void annotate(logfile* lf, + uint64_t line_number, string_attrs_t& sa, logline_value_vector& values, bool annotate_module) const override @@ -1567,6 +1566,7 @@ public: = fd.fd_root_meta->lvm_user_hidden; } } + log_format::annotate(lf, line_number, sa, values, annotate_module); } const logline_value_stats* stats_for_value( @@ -1977,7 +1977,8 @@ public: return retval; } - void annotate(uint64_t line_number, + void annotate(logfile* lf, + uint64_t line_number, string_attrs_t& sa, logline_value_vector& values, bool annotate_module) const override @@ -2078,6 +2079,8 @@ public: return true; }); } + + log_format::annotate(lf, line_number, sa, values, annotate_module); } std::shared_ptr specialized(int fmt_lock) override diff --git a/src/log_search_table.cc b/src/log_search_table.cc index 427aab6c..a80a1c16 100644 --- a/src/log_search_table.cc +++ b/src/log_search_table.cc @@ -47,6 +47,10 @@ log_search_table::log_search_table(std::shared_ptr code, void log_search_table::get_columns_int(std::vector& cols) const { + if (!this->lst_cols.empty()) { + return; + } + column_namer cn{column_namer::language::SQL}; if (this->lst_format != nullptr) { @@ -175,7 +179,7 @@ log_search_table::next(log_cursor& lc, logfile_sub_source& lss) lf->read_full_message(lf_iter, sbr); sbr.erase_ansi(); lf->get_format()->annotate( - cl, this->vi_attrs, this->lst_line_values_cache, false); + lf, cl, this->vi_attrs, this->lst_line_values_cache, false); this->lst_content = this->lst_line_values_cache.lvv_sbr.to_string_fragment(); diff --git a/src/log_vtab_impl.cc b/src/log_vtab_impl.cc index 8a925fcf..fde2390e 100644 --- a/src/log_vtab_impl.cc +++ b/src/log_vtab_impl.cc @@ -226,7 +226,7 @@ log_vtab_impl::extract(logfile* lf, auto format = lf->get_format(); this->vi_attrs.clear(); - format->annotate(line_number, this->vi_attrs, values, false); + format->annotate(lf, line_number, this->vi_attrs, values, false); } bool @@ -321,6 +321,12 @@ struct vtab_cursor { this->log_msg_line = this->log_cursor.lc_curr_line; } + void invalidate() + { + this->line_values.clear(); + this->log_msg_line = -1_vl; + } + sqlite3_vtab_cursor base; struct log_cursor log_cursor; vis_line_t log_msg_line{-1_vl}; @@ -510,7 +516,7 @@ vt_next(sqlite3_vtab_cursor* cur) auto* vt = (log_vtab*) cur->pVtab; auto done = false; - vc->line_values.clear(); + vc->invalidate(); if (!vc->log_cursor.lc_indexed_lines.empty()) { vc->log_cursor.lc_curr_line = vc->log_cursor.lc_indexed_lines.back(); vc->log_cursor.lc_indexed_lines.pop_back(); @@ -562,7 +568,7 @@ vt_next_no_rowid(sqlite3_vtab_cursor* cur) auto* vt = (log_vtab*) cur->pVtab; auto done = false; - vc->line_values.lvv_values.clear(); + vc->invalidate(); do { log_cursor_latest = vc->log_cursor; if (((log_cursor_latest.lc_curr_line % 1024) == 0) @@ -1977,7 +1983,7 @@ vt_update(sqlite3_vtab* tab, int val = sqlite3_value_int( argv[2 + vt->footer_index(log_footer_columns::mark)]); vis_line_t vrowid(rowid); - + const auto msg_info = *vt->lss->window_at(vrowid).begin(); const auto* part_name = sqlite3_value_text( argv[2 + vt->footer_index(log_footer_columns::partition)]); const auto* log_comment = sqlite3_value_text( @@ -1986,6 +1992,8 @@ vt_update(sqlite3_vtab* tab, argc, argv, 2 + vt->footer_index(log_footer_columns::tags)); const auto log_annos = from_sqlite>()( argc, argv, 2 + vt->footer_index(log_footer_columns::annotations)); + const auto log_opid = from_sqlite>()( + argc, argv, 2 + vt->footer_index(log_footer_columns::opid)); bookmark_metadata tmp_bm; if (log_tags) { @@ -2042,7 +2050,11 @@ vt_update(sqlite3_vtab* tab, vt->lss->set_line_meta_changed(); } - if (!has_meta && part_name == nullptr) { + if (!has_meta && part_name == nullptr + && (!log_opid + || msg_info.get_values().lvv_opid_provenance + == logline_value_vector::opid_provenance::file)) + { vt->lss->erase_bookmark_metadata(vrowid); } @@ -2055,6 +2067,23 @@ vt_update(sqlite3_vtab* tab, &textview_curses::BM_PARTITION, vrowid, false); } + if (log_opid) { + auto& lvv = msg_info.get_values(); + if (!lvv.lvv_opid_value + || lvv.lvv_opid_provenance + == logline_value_vector::opid_provenance::user) + { + msg_info.get_file_ptr()->set_logline_opid( + msg_info.get_file_line_number(), log_opid.value()); + vt->lss->set_line_meta_changed(); + } + } else if (msg_info.get_values().lvv_opid_provenance + == logline_value_vector::opid_provenance::user) + { + msg_info.get_file_ptr()->clear_logline_opid( + msg_info.get_file_line_number()); + } + if (has_meta) { auto& line_meta = vt->lss->get_bookmark_metadata(vrowid); @@ -2088,7 +2117,6 @@ vt_update(sqlite3_vtab* tab, { line_meta.bm_annotations.la_pairs.clear(); } - vt->lss->set_line_meta_changed(); } @@ -2096,8 +2124,8 @@ vt_update(sqlite3_vtab* tab, rowid += 1; while ((size_t) rowid < vt->lss->text_line_count()) { vis_line_t vl(rowid); - content_line_t cl = vt->lss->at(vl); - logline* ll = vt->lss->find_line(cl); + auto cl = vt->lss->at(vl); + auto* ll = vt->lss->find_line(cl); if (ll->is_message()) { break; } diff --git a/src/logfile.cc b/src/logfile.cc index 11784601..3fe95e21 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -632,6 +632,39 @@ logfile::rebuild_index(std::optional deadline) static const auto& dts_cfg = injector::get(); + if (!this->lf_invalidated_opids.empty()) { + auto writeOpids = this->lf_opids.writeAccess(); + + for (auto bm_pair : this->lf_bookmark_metadata) { + if (bm_pair.second.bm_opid.empty()) { + continue; + } + + if (!this->lf_invalidated_opids.contains(bm_pair.second.bm_opid)) { + continue; + } + + auto opid_iter + = writeOpids->los_opid_ranges.find(bm_pair.second.bm_opid); + if (opid_iter == writeOpids->los_opid_ranges.end()) { + log_warning("opid not in ranges: %s", + bm_pair.second.bm_opid.c_str()); + continue; + } + + if (bm_pair.first >= this->lf_index.size()) { + log_warning("stale bookmark: %d", bm_pair.first); + continue; + } + + auto& ll = this->lf_index[bm_pair.first]; + opid_iter->second.otr_range.extend_to(ll.get_timeval()); + opid_iter->second.otr_level_stats.update_msg_count( + ll.get_msg_level()); + } + this->lf_invalidated_opids.clear(); + } + if (!this->lf_indexing) { if (this->lf_sort_needed) { this->lf_sort_needed = false; @@ -1466,3 +1499,83 @@ logfile::dump_stats() log_info(" requested_preloads=%lu", buf_stats.s_requested_preloads); log_info(" used_preloads=%lu", buf_stats.s_used_preloads); } + +void +logfile::set_logline_opid(uint32_t line_number, string_fragment opid) +{ + if (line_number >= this->lf_index.size()) { + log_error("invalid line number: %s", line_number); + return; + } + + auto bm_iter = this->lf_bookmark_metadata.find(line_number); + if (bm_iter != this->lf_bookmark_metadata.end()) { + if (bm_iter->second.bm_opid == opid) { + return; + } + } + + auto write_opids = this->lf_opids.writeAccess(); + + if (bm_iter != this->lf_bookmark_metadata.end() + && !bm_iter->second.bm_opid.empty()) + { + auto old_opid_iter = write_opids->los_opid_ranges.find(opid); + if (old_opid_iter != write_opids->los_opid_ranges.end()) { + this->lf_invalidated_opids.insert(old_opid_iter->first); + } + } + + auto& ll = this->lf_index[line_number]; + auto log_tv = ll.get_timeval(); + auto opid_iter = write_opids->insert_op(this->lf_allocator, opid, log_tv); + auto& otr = opid_iter->second; + + otr.otr_level_stats.update_msg_count(ll.get_msg_level()); + ll.set_opid(opid.hash()); + this->lf_bookmark_metadata[line_number].bm_opid = opid.to_string(); +} + +void +logfile::clear_logline_opid(uint32_t line_number) +{ + if (line_number >= this->lf_index.size()) { + return; + } + + auto iter = this->lf_bookmark_metadata.find(line_number); + if (iter == this->lf_bookmark_metadata.end()) { + return; + } + + if (iter->second.bm_opid.empty()) { + return; + } + + auto& ll = this->lf_index[line_number]; + ll.set_opid(0); + auto opid = std::move(iter->second.bm_opid); + auto opid_sf = string_fragment::from_str(opid); + + if (iter->second.empty(bookmark_metadata::categories::any)) { + this->lf_bookmark_metadata.erase(iter); + + auto writeOpids = this->lf_opids.writeAccess(); + + auto otr_iter = writeOpids->los_opid_ranges.find(opid_sf); + if (otr_iter == writeOpids->los_opid_ranges.end()) { + return; + } + + if (otr_iter->second.otr_range.tr_begin != ll.get_timeval() + && otr_iter->second.otr_range.tr_end != ll.get_timeval()) + { + otr_iter->second.otr_level_stats.update_msg_count( + ll.get_msg_level(), -1); + return; + } + + otr_iter->second.clear(); + this->lf_invalidated_opids.insert(opid_sf); + } +} diff --git a/src/logfile.hh b/src/logfile.hh index 1231e027..9aca06a4 100644 --- a/src/logfile.hh +++ b/src/logfile.hh @@ -388,6 +388,10 @@ public: safe_opid_state& get_opids() { return this->lf_opids; } + void set_logline_opid(uint32_t line_number, string_fragment opid); + + void clear_logline_opid(uint32_t line_number); + void quiesce() { this->lf_line_buffer.quiesce(); } void enable_cache() { this->lf_line_buffer.enable_cache(); } @@ -459,6 +463,10 @@ private: bool lf_indexing{true}; bool lf_partial_line{false}; bool lf_zoned_to_local_state{true}; + robin_hood::unordered_set> + lf_invalidated_opids; logline_observer* lf_logline_observer{nullptr}; logfile_observer* lf_logfile_observer{nullptr}; size_t lf_longest_line{0}; diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index 161c84da..e247798b 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -298,7 +298,10 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, sbr.share(this->lss_share_manager, (char*) this->lss_token_value.c_str(), this->lss_token_value.size()); - format->annotate(line, this->lss_token_attrs, this->lss_token_values); + format->annotate(this->lss_token_file.get(), + line, + this->lss_token_attrs, + this->lss_token_values); if (flags & RF_REWRITE) { exec_context ec( &this->lss_token_values, pretty_sql_callback, pretty_pipe_callback); @@ -1666,7 +1669,7 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt, auto format = lf->get_format(); string_attrs_t sa; auto line_number = std::distance(lf->cbegin(), ll); - format->annotate(line_number, sa, values); + format->annotate(lf, line_number, sa, values); sqlite3_reset(stmt); sqlite3_clear_bindings(stmt); @@ -2217,6 +2220,8 @@ logfile_sub_source::meta_grepper::grep_value_for_line(vis_line_t line, } value_out.append("\x1c"); } + value_out.append("\x1c"); + value_out.append(bm.bm_opid); } return !this->lmg_done; @@ -2286,7 +2291,16 @@ logline_window::begin() logline_window::iterator logline_window::end() { - return {this->lw_source, this->lw_end_line}; + auto vl = this->lw_end_line; + while (vl < vis_line_t(this->lw_source.text_line_count())) { + const auto& line = this->lw_source.find_line(this->lw_source.at(vl)); + if (line->is_message()) { + break; + } + ++vl; + } + + return {this->lw_source, vl}; } logline_window::logmsg_info::logmsg_info(logfile_sub_source& lss, vis_line_t vl) @@ -2304,6 +2318,8 @@ logline_window::logmsg_info::logmsg_info(logfile_sub_source& lss, vis_line_t vl) if (line_pair.second->is_message()) { this->li_file = line_pair.first.get(); this->li_logline = line_pair.second; + this->li_line_number + = std::distance(this->li_file->begin(), this->li_logline); break; } else { --vl; @@ -2331,6 +2347,8 @@ logline_window::logmsg_info::next_msg() if (line_pair.second->is_message()) { this->li_file = line_pair.first.get(); this->li_logline = line_pair.second; + this->li_line_number + = std::distance(this->li_file->begin(), this->li_logline); break; } else { ++this->li_line; @@ -2338,6 +2356,79 @@ logline_window::logmsg_info::next_msg() } } +void +logline_window::logmsg_info::prev_msg() +{ + this->li_file = nullptr; + this->li_logline = logfile::iterator{}; + this->li_string_attrs.clear(); + this->li_line_values.clear(); + while (this->li_line > 0) { + --this->li_line; + auto pair_opt = this->li_source.find_line_with_file(this->li_line); + + if (!pair_opt) { + break; + } + + auto line_pair = pair_opt.value(); + if (line_pair.second->is_message()) { + this->li_file = line_pair.first.get(); + this->li_logline = line_pair.second; + this->li_line_number + = std::distance(this->li_file->begin(), this->li_logline); + break; + } + } +} + +std::optional +logline_window::logmsg_info::get_metadata() const +{ + auto line_number = std::distance(this->li_file->begin(), this->li_logline); + auto& bm = this->li_file->get_bookmark_metadata(); + auto bm_iter = bm.find(line_number); + if (bm_iter == bm.end()) { + return std::nullopt; + } + return &bm_iter->second; +} + +logline_window::logmsg_info::metadata_edit_guard::~metadata_edit_guard() +{ + auto line_number = std::distance(this->meg_logmsg_info.li_file->begin(), + this->meg_logmsg_info.li_logline); + auto& bm = this->meg_logmsg_info.li_file->get_bookmark_metadata(); + auto bm_iter = bm.find(line_number); + if (bm_iter != bm.end() + && bm_iter->second.empty(bookmark_metadata::categories::any)) + { + bm.erase(bm_iter); + } +} + +bookmark_metadata& +logline_window::logmsg_info::metadata_edit_guard::operator*() +{ + auto line_number = std::distance(this->meg_logmsg_info.li_file->begin(), + this->meg_logmsg_info.li_logline); + auto& bm = this->meg_logmsg_info.li_file->get_bookmark_metadata(); + return bm[line_number]; +} + +size_t +logline_window::logmsg_info::get_line_count() const +{ + size_t retval = 1; + auto iter = std::next(this->li_logline); + while (iter != this->li_file->end() && iter->is_continued()) { + ++iter; + retval += 1; + } + + return retval; +} + void logline_window::logmsg_info::load_msg() const { @@ -2355,10 +2446,20 @@ logline_window::logmsg_info::load_msg() const scrub_ansi_string(str, &this->li_string_attrs); this->li_line_values.lvv_sbr.get_metadata().m_has_ansi = false; } - format->annotate(std::distance(this->li_file->cbegin(), this->li_logline), + format->annotate(this->li_file, + std::distance(this->li_file->begin(), this->li_logline), this->li_string_attrs, this->li_line_values, false); + + if (!this->li_line_values.lvv_opid_value) { + auto bm_opt = this->get_metadata(); + if (bm_opt && !bm_opt.value()->bm_opid.empty()) { + this->li_line_values.lvv_opid_value = bm_opt.value()->bm_opid; + this->li_line_values.lvv_opid_provenance + = logline_value_vector::opid_provenance::user; + } + } } std::string @@ -2379,6 +2480,14 @@ logline_window::iterator::operator++() return *this; } +logline_window::iterator& +logline_window::iterator::operator--() +{ + this->i_info.prev_msg(); + + return *this; +} + static std::vector timestamp_poss() { @@ -2553,17 +2662,13 @@ logfile_sub_source::text_crumbs_for_line(int line, scrub_ansi_string(al.get_string(), &al.al_attrs); sbr.erase_ansi(); } - format->annotate(file_line_number, al.get_attrs(), values); + format->annotate(lf.get(), file_line_number, al.get_attrs(), values); - auto opid_opt = get_string_attr(al.get_attrs(), logline::L_OPID); - if (opid_opt && !opid_opt.value().saw_string_attr->sa_range.empty()) { - const auto& opid_range = opid_opt.value().saw_string_attr->sa_range; - const auto opid_str - = sbr.to_string_fragment(opid_range.lr_start, opid_range.length()) - .to_string(); + if (values.lvv_opid_value) { crumbs.emplace_back( - opid_str, - attr_line_t().append(lnav::roles::identifier(opid_str)), + values.lvv_opid_value.value(), + attr_line_t().append( + lnav::roles::identifier(values.lvv_opid_value.value())), [this]() -> std::vector { std::vector retval; diff --git a/src/logfile_sub_source.hh b/src/logfile_sub_source.hh index 83e05e3e..730492f2 100644 --- a/src/logfile_sub_source.hh +++ b/src/logfile_sub_source.hh @@ -106,8 +106,7 @@ public: void loc_history_append(vis_line_t top) override; - std::optional loc_history_back( - vis_line_t current_top) override; + std::optional loc_history_back(vis_line_t current_top) override; std::optional loc_history_forward( vis_line_t current_top) override; @@ -135,7 +134,13 @@ public: vis_line_t get_vis_line() const { return this->li_line; } - const logline& get_logline() const { return *this->li_logline; } + size_t get_line_count() const; + + uint32_t get_file_line_number() const { return this->li_line_number; } + + logfile* get_file_ptr() const { return this->li_file; } + + logline& get_logline() const { return *this->li_logline; } const string_attrs_t& get_attrs() const { @@ -149,18 +154,40 @@ public: return this->li_line_values; } + std::optional get_metadata() const; + + struct metadata_edit_guard { + ~metadata_edit_guard(); + + bookmark_metadata& operator*(); + + private: + friend logmsg_info; + + metadata_edit_guard(logmsg_info& li) : meg_logmsg_info(li) {} + logmsg_info& meg_logmsg_info; + }; + + metadata_edit_guard edit_metadata() + { + return metadata_edit_guard(*this); + } + std::string to_string(const struct line_range& lr) const; private: friend iterator; + friend metadata_edit_guard; void next_msg(); + void prev_msg(); void load_msg() const; logfile_sub_source& li_source; vis_line_t li_line; + uint32_t li_line_number; logfile* li_file{nullptr}; - logfile::const_iterator li_logline; + logfile::iterator li_logline; mutable string_attrs_t li_string_attrs; mutable logline_value_vector li_line_values; }; @@ -170,14 +197,22 @@ public: iterator(logfile_sub_source& lss, vis_line_t vl) : i_info(lss, vl) {} iterator& operator++(); + iterator& operator--(); bool operator!=(const iterator& rhs) const { return this->i_info.get_vis_line() != rhs.i_info.get_vis_line(); } + bool operator==(const iterator& rhs) const + { + return this->i_info.get_vis_line() == rhs.i_info.get_vis_line(); + } + const logmsg_info& operator*() const { return this->i_info; } + const logmsg_info* operator->() const { return &this->i_info; } + private: logmsg_info i_info; }; @@ -474,8 +509,7 @@ public: return std::nullopt; } - std::optional find_from_time( - const struct timeval& start) const; + std::optional find_from_time(const struct timeval& start) const; std::optional find_from_time(time_t start) const { @@ -529,6 +563,17 @@ public: return logline_window(*this, start_vl, end_vl); } + logline_window window_at(vis_line_t start_vl) + { + return logline_window(*this, start_vl, start_vl + 1_vl); + } + + logline_window window_to_end(vis_line_t start_vl) + { + return logline_window( + *this, start_vl, vis_line_t(this->text_line_count())); + } + /** * Container for logfile references that keeps of how many lines in the * logfile have been indexed. @@ -605,8 +650,7 @@ public: return retval; } - std::optional find_data( - const std::shared_ptr& lf) + std::optional find_data(const std::shared_ptr& lf) { for (auto& ld : *this) { if (ld->ld_filter_state.lfo_filter_state.tfs_logfile == lf) { diff --git a/src/readline_possibilities.cc b/src/readline_possibilities.cc index 0ba2d8fb..5a4e33e6 100644 --- a/src/readline_possibilities.cc +++ b/src/readline_possibilities.cc @@ -304,7 +304,7 @@ add_filter_expr_possibilities(readline_curses* rlc, lf->read_full_message(ll, values.lvv_sbr); values.lvv_sbr.erase_ansi(); - format->annotate(cl, sa, values); + format->annotate(lf.get(), cl, sa, values); for (auto& lv : values.lvv_values) { if (!lv.lv_meta.lvm_struct_name.empty()) { continue; diff --git a/src/scripts/scripts.am b/src/scripts/scripts.am index 32d0cd82..3d8420da 100644 --- a/src/scripts/scripts.am +++ b/src/scripts/scripts.am @@ -9,10 +9,12 @@ BUILTIN_LNAVSCRIPTS = \ $(srcdir)/scripts/piper-url-handler.lnav \ $(srcdir)/scripts/rename-stdin.lnav \ $(srcdir)/scripts/search-for.lnav \ + $(srcdir)/scripts/zk-set-ops.lnav \ $() BUILTIN_SHSCRIPTS = \ $(srcdir)/scripts/com.vmware.btresolver.py \ $(srcdir)/scripts/dump-pid.sh \ $(srcdir)/scripts/pcap_log-converter.sh \ + $(srcdir)/scripts/zookeeper.sql \ $() diff --git a/src/scripts/zk-set-ops.lnav b/src/scripts/zk-set-ops.lnav new file mode 100644 index 00000000..8d8dc587 --- /dev/null +++ b/src/scripts/zk-set-ops.lnav @@ -0,0 +1,66 @@ +# +# @synopsis: zk-set-ops +# @description: Set Op IDs for zookeeper-related logs +# + +# Drop the table used to store election starts/finishes +;DROP TABLE IF EXISTS zk_elections +# Create a table of the lines where an election is started/finished. +;CREATE TABLE zk_elections AS + SELECT log_line, log_body, log_path, + CASE json_contains(log_tags, '#zk-election-started') + WHEN 1 THEN 'start' + WHEN 0 THEN 'finish' + END AS msg_type + FROM zookeeper_log + WHERE json_contains(log_tags, '#zk-election-started') OR json_contains(log_tags, '#zk-election-finished') + ORDER BY log_line ASC + +# Create a table of the start of an election and either the end +# or another start of an election. We'll use these bounds to +# assign opids to election-related log messages. +;DROP TABLE IF EXISTS zk_election_ops +;CREATE TABLE zk_election_ops AS SELECT log_line AS low_mark, + (SELECT zk_finishes.log_line + FROM zk_elections AS zk_finishes + WHERE zk_finishes.log_line > zk_elections.log_line AND zk_finishes.log_path = zk_elections.log_path + ORDER BY log_line ASC + LIMIT 1) AS high_mark, + log_path, + 'zk-election-#' || row_number() OVER (PARTITION BY log_path ORDER BY log_line) || '-' || basename(log_path) AS opid + FROM zk_elections + WHERE msg_type = 'start' + +# Use the SQLite `UPDATE FROM` syntax to assign the opids based +# on the table we just created. +;UPDATE zookeeper_log + SET log_opid = zk_ops.opid + FROM (SELECT * FROM zk_election_ops) AS zk_ops + WHERE + zookeeper_log.log_path = zk_ops.log_path AND + zookeeper_log.log_line IN (zk_ops.low_mark, zk_ops.high_mark) + +;UPDATE zookeeper_log + SET log_opid = zk_ops.opid + FROM (SELECT * FROM zk_election_ops) AS zk_ops + WHERE + zookeeper_log.log_path = zk_ops.log_path AND + zookeeper_log.log_line BETWEEN zk_ops.low_mark AND zk_ops.high_mark AND + (zookeeper_log.log_body LIKE '%Quorum%' OR + zookeeper_log.thread LIKE '%Quorum%' OR + zookeeper_log.logger LIKE '%Quorum%') + +# Use the search table that found the creation of the session +# IDs and assign an opid wherever we find the session ID in a +# log message. +;UPDATE all_logs + SET log_opid = session_opid + FROM ( + SELECT + log_line, + printf('zk--session-0x%x', + regexp_match((SELECT printf('(%s)', group_concat(session_id, '|')) FROM zk_session_ids), log_body)) AS session_opid + FROM all_logs + WHERE log_body REGEXP (SELECT printf('(%s)', group_concat(session_id, '|')) FROM zk_session_ids) + ) AS zk_session_tab + WHERE all_logs.log_line = zk_session_tab.log_line diff --git a/src/scripts/zookeeper.sql b/src/scripts/zookeeper.sql new file mode 100644 index 00000000..c21bdcfc --- /dev/null +++ b/src/scripts/zookeeper.sql @@ -0,0 +1,3 @@ + +CREATE VIEW zk_session_ids AS + SELECT DISTINCT session_id FROM zk_commit_session_id; diff --git a/src/session_data.cc b/src/session_data.cc index a7b5ef1d..064ebc12 100644 --- a/src/session_data.cc +++ b/src/session_data.cc @@ -76,6 +76,7 @@ CREATE TABLE IF NOT EXISTS bookmarks ( comment text DEFAULT '', tags text DEFAULT '', annotations text DEFAULT NULL, + log_opid text DEFAULT NULL, PRIMARY KEY (log_time, log_format, log_hash, session_time) ); @@ -129,6 +130,7 @@ static const char* UPGRADE_STMTS[] = { R"(ALTER TABLE bookmarks ADD COLUMN comment text DEFAULT '';)", R"(ALTER TABLE bookmarks ADD COLUMN tags text DEFAULT '';)", R"(ALTER TABLE bookmarks ADD COLUMN annotations text DEFAULT NULL;)", + R"(ALTER TABLE bookmarks ADD COLUMN log_opid text DEFAULT NULL;)", }; static const size_t MAX_SESSIONS = 8; @@ -414,6 +416,7 @@ load_time_bookmarks() comment, tags, annotations, + log_opid, session_time=? AS same_session FROM bookmarks WHERE log_time BETWEEN ? AND ? AND @@ -536,6 +539,7 @@ load_time_bookmarks() const char* tags = (const char*) sqlite3_column_text(stmt.in(), 7); const auto annotations = sqlite3_column_text(stmt.in(), 8); + const auto log_opid = sqlite3_column_text(stmt.in(), 9); int64_t mark_time = sqlite3_column_int64(stmt.in(), 3); struct timeval log_tv; struct exttm log_tm; @@ -669,6 +673,12 @@ load_time_bookmarks() meta = true; } } + if (log_opid != nullptr && log_opid[0] != '\0') { + auto opid_sf + = string_fragment::from_c_str(log_opid); + lf->set_logline_opid(line_number, opid_sf); + meta = true; + } if (!meta) { marked_session_lines.emplace_back( lf->original_line_time(line_iter), @@ -1001,7 +1011,6 @@ save_user_bookmarks(sqlite3* db, for (auto iter = user_marks.begin(); iter != user_marks.end(); ++iter) { content_line_t cl = *iter; - auto line_meta_opt = lss.find_bookmark_metadata(cl); auto lf = lss.find(cl); if (lf == nullptr) { continue; @@ -1035,85 +1044,134 @@ save_user_bookmarks(sqlite3* db, continue; } - if (!line_meta_opt) { - if (sqlite3_bind_text(stmt, 5, "", 0, SQLITE_TRANSIENT) - != SQLITE_OK) - { - log_error("could not bind log hash -- %s", sqlite3_errmsg(db)); - return; - } - } else { - const auto& line_meta = *(line_meta_opt.value()); - if (line_meta.empty(bookmark_metadata::categories::any)) { - continue; - } + if (sqlite3_bind_text(stmt, 5, "", 0, SQLITE_TRANSIENT) != SQLITE_OK) { + log_error("could not bind log hash -- %s", sqlite3_errmsg(db)); + return; + } - if (sqlite3_bind_text(stmt, - 5, - line_meta.bm_name.c_str(), - line_meta.bm_name.length(), - SQLITE_TRANSIENT) - != SQLITE_OK) - { - log_error("could not bind part name -- %s", sqlite3_errmsg(db)); - return; - } + if (sqlite3_step(stmt) != SQLITE_DONE) { + log_error("could not execute bookmark insert statement -- %s", + sqlite3_errmsg(db)); + return; + } - if (sqlite3_bind_text(stmt, - 6, - line_meta.bm_comment.c_str(), - line_meta.bm_comment.length(), - SQLITE_TRANSIENT) - != SQLITE_OK) - { - log_error("could not bind comment -- %s", sqlite3_errmsg(db)); - return; - } + marked_session_lines.emplace_back(lf->original_line_time(line_iter), + lf->get_format_ptr()->get_name(), + line_hash); - std::string tags; + sqlite3_reset(stmt); + } +} - if (!line_meta.bm_tags.empty()) { - yajlpp_gen gen; +static void +save_meta_bookmarks(sqlite3* db, sqlite3_stmt* stmt, logfile* lf) +{ + for (const auto& bm_pair : lf->get_bookmark_metadata()) { + auto cl = content_line_t(bm_pair.first); + sqlite3_clear_bindings(stmt); - yajl_gen_config(gen, yajl_gen_beautify, false); + auto line_iter = lf->begin() + cl; + auto read_result = lf->read_line(line_iter); - { - yajlpp_array arr(gen); + if (read_result.isErr()) { + continue; + } - for (const auto& str : line_meta.bm_tags) { - arr.gen(str); - } - } + auto line_hash = read_result + .map([cl](auto sbr) { + return hasher() + .update(sbr.get_data(), sbr.length()) + .update(cl) + .to_string(); + }) + .unwrap(); + + if (bind_values(stmt, + lf->original_line_time(line_iter), + lf->get_format()->get_name(), + line_hash, + lnav_data.ld_session_time) + != SQLITE_OK) + { + continue; + } + + const auto& line_meta = bm_pair.second; + if (line_meta.empty(bookmark_metadata::categories::any)) { + continue; + } + + if (sqlite3_bind_text(stmt, + 5, + line_meta.bm_name.c_str(), + line_meta.bm_name.length(), + SQLITE_TRANSIENT) + != SQLITE_OK) + { + log_error("could not bind part name -- %s", sqlite3_errmsg(db)); + return; + } + + if (sqlite3_bind_text(stmt, + 6, + line_meta.bm_comment.c_str(), + line_meta.bm_comment.length(), + SQLITE_TRANSIENT) + != SQLITE_OK) + { + log_error("could not bind comment -- %s", sqlite3_errmsg(db)); + return; + } + + std::string tags; + + if (!line_meta.bm_tags.empty()) { + yajlpp_gen gen; + + yajl_gen_config(gen, yajl_gen_beautify, false); + + { + yajlpp_array arr(gen); - tags = gen.to_string_fragment().to_string(); + for (const auto& str : line_meta.bm_tags) { + arr.gen(str); + } } - if (sqlite3_bind_text( - stmt, 7, tags.c_str(), tags.length(), SQLITE_TRANSIENT) + tags = gen.to_string_fragment().to_string(); + } + + if (sqlite3_bind_text( + stmt, 7, tags.c_str(), tags.length(), SQLITE_TRANSIENT) + != SQLITE_OK) + { + log_error("could not bind tags -- %s", sqlite3_errmsg(db)); + return; + } + + if (!line_meta.bm_annotations.la_pairs.empty()) { + auto anno_str = logmsg_annotations_handlers.to_string( + line_meta.bm_annotations); + + if (sqlite3_bind_text(stmt, + 8, + anno_str.c_str(), + anno_str.length(), + SQLITE_TRANSIENT) != SQLITE_OK) { - log_error("could not bind tags -- %s", sqlite3_errmsg(db)); + log_error("could not bind annotations -- %s", + sqlite3_errmsg(db)); return; } + } else { + sqlite3_bind_null(stmt, 8); + } - if (!line_meta.bm_annotations.la_pairs.empty()) { - auto anno_str = logmsg_annotations_handlers.to_string( - line_meta.bm_annotations); - - if (sqlite3_bind_text(stmt, - 8, - anno_str.c_str(), - anno_str.length(), - SQLITE_TRANSIENT) - != SQLITE_OK) - { - log_error("could not bind annotations -- %s", - sqlite3_errmsg(db)); - return; - } - } else { - sqlite3_bind_null(stmt, 8); - } + if (line_meta.bm_opid.empty()) { + sqlite3_bind_null(stmt, 9); + } else { + bind_to_sqlite(stmt, 9, line_meta.bm_opid); } if (sqlite3_step(stmt) != SQLITE_DONE) { @@ -1190,8 +1248,8 @@ save_time_bookmarks() recent_refs.rr_netlocs.insert(netlocs.begin(), netlocs.end()); } - logfile_sub_source& lss = lnav_data.ld_log_source; - bookmarks::type& bm = lss.get_user_bookmarks(); + auto& lss = lnav_data.ld_log_source; + auto& bm = lss.get_user_bookmarks(); if (sqlite3_prepare_v2(db.in(), "DELETE FROM bookmarks WHERE " @@ -1234,8 +1292,8 @@ save_time_bookmarks() if (sqlite3_prepare_v2(db.in(), "REPLACE INTO bookmarks" " (log_time, log_format, log_hash, session_time, " - "part_name, comment, tags, annotations)" - " VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + "part_name, comment, tags, annotations, log_opid)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", -1, stmt.out(), nullptr) @@ -1289,11 +1347,14 @@ save_time_bookmarks() } save_user_bookmarks(db.in(), stmt.in(), bm[&textview_curses::BM_USER]); - auto all_meta_marks = bm[&textview_curses::BM_META]; - const auto& bm_parts = bm[&textview_curses::BM_PARTITION]; - all_meta_marks.insert( - all_meta_marks.end(), bm_parts.begin(), bm_parts.end()); - save_user_bookmarks(db.in(), stmt.in(), all_meta_marks); + for (const auto& ldd : lss) { + auto* lf = ldd->get_file_ptr(); + if (lf == nullptr) { + continue; + } + + save_meta_bookmarks(db.in(), stmt.in(), lf); + } if (sqlite3_prepare_v2(db.in(), "DELETE FROM time_offset WHERE " diff --git a/src/spectro_impls.cc b/src/spectro_impls.cc index ddc2d36e..beab8b58 100644 --- a/src/spectro_impls.cc +++ b/src/spectro_impls.cc @@ -299,7 +299,7 @@ log_spectro_value_source::spectro_mark(textview_curses& tc, lf->read_full_message(ll, values.lvv_sbr); values.lvv_sbr.erase_ansi(); sa.clear(); - format->annotate(cl, sa, values, false); + format->annotate(lf.get(), cl, sa, values, false); auto lv_iter = find_if(values.lvv_values.begin(), values.lvv_values.end(), diff --git a/src/sqlitepp.client.hh b/src/sqlitepp.client.hh index 4eaa1626..fc4d7994 100644 --- a/src/sqlitepp.client.hh +++ b/src/sqlitepp.client.hh @@ -61,6 +61,13 @@ bind_to_sqlite(sqlite3_stmt* stmt, int index, intern_string_t ist) stmt, index, ist.get(), ist.size(), SQLITE_TRANSIENT); } +inline int +bind_to_sqlite(sqlite3_stmt* stmt, int index, string_fragment sf) +{ + return sqlite3_bind_text( + stmt, index, sf.data(), sf.length(), SQLITE_TRANSIENT); +} + inline int bind_to_sqlite(sqlite3_stmt* stmt, int index, const std::string& str) { diff --git a/src/textview_curses.hh b/src/textview_curses.hh index 3e6ab142..ea3bba23 100644 --- a/src/textview_curses.hh +++ b/src/textview_curses.hh @@ -259,8 +259,7 @@ public: virtual ~text_time_translator() = default; - virtual std::optional row_for_time( - struct timeval time_bucket) + virtual std::optional row_for_time(struct timeval time_bucket) = 0; virtual std::optional row_for(const row_info& ri) @@ -321,8 +320,7 @@ public: static std::string to_anchor_string(const std::string& raw); - virtual std::optional row_for_anchor(const std::string& id) - = 0; + virtual std::optional row_for_anchor(const std::string& id) = 0; enum class direction { prev, @@ -330,7 +328,7 @@ public: }; virtual std::optional adjacent_anchor(vis_line_t vl, - direction dir) + direction dir) { return std::nullopt; } @@ -346,8 +344,7 @@ public: virtual void loc_history_append(vis_line_t top) = 0; - virtual std::optional loc_history_back( - vis_line_t current_top) + virtual std::optional loc_history_back(vis_line_t current_top) = 0; virtual std::optional loc_history_forward( @@ -526,8 +523,7 @@ public: void loc_history_append(vis_line_t top) override; - std::optional loc_history_back( - vis_line_t current_top) override; + std::optional loc_history_back(vis_line_t current_top) override; std::optional loc_history_forward( vis_line_t current_top) override; @@ -628,8 +624,8 @@ public: } std::optional> horiz_shift(vis_line_t start, - vis_line_t end, - int off_start); + vis_line_t end, + int off_start); void set_search_action(action sa) { diff --git a/src/view_curses.cc b/src/view_curses.cc index 8efa3cd5..8f8ad7e1 100644 --- a/src/view_curses.cc +++ b/src/view_curses.cc @@ -784,8 +784,7 @@ view_colors::to_attrs(const lnav_theme& lt, reporter(&sc.sc_color, lnav::console::user_message::warning("")); #endif } else { - auto role_class_path - = ghc::filesystem::path(pp_sc.pp_path.to_string()).parent_path(); + auto role_class_path = ghc::filesystem::path(pp_sc.pp_path.to_string()); auto inner = role_class_path.filename().string(); auto outer = role_class_path.parent_path().filename().string(); diff --git a/src/yajlpp/yajlpp.cc b/src/yajlpp/yajlpp.cc index a00d8f90..cc9defe6 100644 --- a/src/yajlpp/yajlpp.cc +++ b/src/yajlpp/yajlpp.cc @@ -744,7 +744,8 @@ yajlpp_parse_context::update_callbacks(const json_path_container* orig_handlers, ? static_cast(-1) : this->ypc_array_index[this->ypc_array_handler_count - 1]; - if ((cap.sf_end != (int) this->ypc_path.size() - 1) + if ((!jph.is_array() + || cap.sf_end != (int) this->ypc_path.size() - 1) && (!jph.is_array() || index != yajlpp_provider_context::nindex)) { diff --git a/test/drive_data_scanner.cc b/test/drive_data_scanner.cc index 63e75793..b834ee06 100644 --- a/test/drive_data_scanner.cc +++ b/test/drive_data_scanner.cc @@ -194,7 +194,7 @@ main(int argc, char* argv[]) string_attrs_t sa; if (format.get() != nullptr) { - format->annotate(0, sa, ll_values, false); + format->annotate(nullptr, 0, sa, ll_values, false); body = find_string_attr_range(sa, &SA_BODY); } diff --git a/test/expected/expected.am b/test/expected/expected.am index 9b18fb0a..a9d8d424 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -302,8 +302,12 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.out \ $(srcdir)/%reldir%/test_gantt.sh_3af11588ee36bab7e2caea0f7a24d3c9cafd2310.err \ $(srcdir)/%reldir%/test_gantt.sh_3af11588ee36bab7e2caea0f7a24d3c9cafd2310.out \ + $(srcdir)/%reldir%/test_gantt.sh_3c2669075d0636fab98e2a67a72613420f22c322.err \ + $(srcdir)/%reldir%/test_gantt.sh_3c2669075d0636fab98e2a67a72613420f22c322.out \ $(srcdir)/%reldir%/test_gantt.sh_63500be50fc6743d8312133e2030cbbc39ca15ff.err \ $(srcdir)/%reldir%/test_gantt.sh_63500be50fc6743d8312133e2030cbbc39ca15ff.out \ + $(srcdir)/%reldir%/test_gantt.sh_740f22814c5d856234c2450168dae22989c85d1e.err \ + $(srcdir)/%reldir%/test_gantt.sh_740f22814c5d856234c2450168dae22989c85d1e.out \ $(srcdir)/%reldir%/test_gantt.sh_74a94ee9103eac5e8e78ca57bccf49efa3827a9d.err \ $(srcdir)/%reldir%/test_gantt.sh_74a94ee9103eac5e8e78ca57bccf49efa3827a9d.out \ $(srcdir)/%reldir%/test_gantt.sh_83db753dd2669f801810f311e2d7d74397e10f26.err \ diff --git a/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err b/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err index 578c06f7..4a970c17 100644 --- a/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err +++ b/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err @@ -109,7 +109,7 @@  --> /bad_file_format1/converter/header/expr/default  | :header REGEXP 'foobar  ✘ error: A command is required when a converter is defined - --> {test_dir}/bad-config/formats/invalid-file-format/format.json:4 + --> {test_dir}/bad-config/formats/invalid-file-format/format.json:3  = help: The converter command transforms the file into a format that can be consumed by lnav ✘ error: invalid line format element “/bad_json_log/line-format/0/field” reason: “” is not a defined value @@ -169,29 +169,29 @@  = help: every pattern should have at least one sample that it matches ✘ error: invalid pattern: “/invalid_props_log/regex/std” reason: no timestamp capture found in the pattern - --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 + --> {test_dir}/bad-config/formats/invalid-properties/format.json:3  = help: all log messages need a timestamp ⚠ warning: invalid value “/invalid_props_log/value/non-existent” reason: no patterns have a capture named “non-existent” - --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 + --> {test_dir}/bad-config/formats/invalid-properties/format.json:3  = note: the following captures are available: body, pid, timestamp  = help: values are populated from captures in patterns, so at least one pattern must have a capture with this value name ✘ error: invalid tag definition “/invalid_props_log/tags/badtag” reason: tag definitions must have a non-empty pattern - --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 + --> {test_dir}/bad-config/formats/invalid-properties/format.json:3 ✘ error: invalid tag definition “/invalid_props_log/tags/badtag2” reason: tag definitions must have a non-empty pattern - --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 + --> {test_dir}/bad-config/formats/invalid-properties/format.json:3 ✘ error: invalid tag definition “/invalid_props_log/tags/badtag3” reason: tag definitions must have a non-empty pattern - --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 + --> {test_dir}/bad-config/formats/invalid-properties/format.json:3 ✘ error: “invalid_props_log” is not a valid log format reason: “subsecond-unit” must be set when “subsecond-field” is used - --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 + --> {test_dir}/bad-config/formats/invalid-properties/format.json:3 ✘ error: invalid value for property “/invalid_props_log/timestamp-field” reason: “ts” was not found in the pattern at /invalid_props_log/regex/std - --> {test_dir}/bad-config/formats/invalid-properties/format.json:4 + --> {test_dir}/bad-config/formats/invalid-properties/format.json:3  = note: the following captures are available: body, pid, timestamp ✘ error: “not a color” is not a valid color value for property “/invalid_props_log/highlights/hl1/color” @@ -202,13 +202,13 @@  --> {test_dir}/bad-config/formats/invalid-properties/format.json:25 ✘ error: “no_regexes_log” is not a valid log format reason: no regexes specified - --> {test_dir}/bad-config/formats/no-regexes/format.json:4 + --> {test_dir}/bad-config/formats/no-regexes/format.json:3 ✘ error: “no_regexes_log” is not a valid log format reason: log message samples must be included in a format definition - --> {test_dir}/bad-config/formats/no-regexes/format.json:4 + --> {test_dir}/bad-config/formats/no-regexes/format.json:3 ✘ error: “no_sample_log” is not a valid log format reason: log message samples must be included in a format definition - --> {test_dir}/bad-config/formats/no-samples/format.json:4 + --> {test_dir}/bad-config/formats/no-samples/format.json:3 ✘ error: invalid sample log message: "gitea | 2023/09/24 22:15:55 cmd/web.go:223:runWeb() [I] Starting Gitea on PID: 7" reason: timestamp was not captured  --> {test_dir}/bad-config/formats/invalid-no-tscap/format.json:25 diff --git a/test/expected/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.err b/test/expected/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.err index 6ce508e2..f1d87a9d 100644 --- a/test/expected/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.err +++ b/test/expected/test_format_loader.sh_fca6c1fb9f3aaa69b3ffb2d1a8a86434b2f4a247.err @@ -54,17 +54,17 @@ The value to divide a numeric timestamp by in a JSON log. ✘ error: “foobar_log” is not a valid log format reason: no regexes specified - --> {test_dir}/bad-config-json/formats/invalid-json/format.json:3 + --> {test_dir}/bad-config-json/formats/invalid-json/format.json:2 ✘ error: “foobar_log” is not a valid log format reason: log message samples must be included in a format definition - --> {test_dir}/bad-config-json/formats/invalid-json/format.json:3 + --> {test_dir}/bad-config-json/formats/invalid-json/format.json:2 ✘ error: invalid pattern: “/invalid_key_log/regex/foo” reason: no timestamp capture found in the pattern - --> {test_dir}/bad-config-json/formats/invalid-key/format.json:4 + --> {test_dir}/bad-config-json/formats/invalid-key/format.json:3  = help: all log messages need a timestamp ✘ error: “invalid_key_log” is not a valid log format reason: structured logs cannot have regexes - --> {test_dir}/bad-config-json/formats/invalid-key/format.json:4 + --> {test_dir}/bad-config-json/formats/invalid-key/format.json:3 ✘ error: invalid line format element “/invalid_key_log/line-format/0/field” reason: “non-existent” is not a defined value  --> {test_dir}/bad-config-json/formats/invalid-key/format.json:22 diff --git a/test/expected/test_gantt.sh_3c2669075d0636fab98e2a67a72613420f22c322.err b/test/expected/test_gantt.sh_3c2669075d0636fab98e2a67a72613420f22c322.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_gantt.sh_3c2669075d0636fab98e2a67a72613420f22c322.out b/test/expected/test_gantt.sh_3c2669075d0636fab98e2a67a72613420f22c322.out new file mode 100644 index 00000000..2fe42dce --- /dev/null +++ b/test/expected/test_gantt.sh_3c2669075d0636fab98e2a67a72613420f22c322.out @@ -0,0 +1,10 @@ +E0517 15:04:22.619632 1952452992 logging_unittest.cc:253] Log every 3, iteration 19 +I0517 15:04:22.619642 952452992 logging_unittest.cc:259] Log if every 1, iteration 19 + Op ID: test1 +I0517 15:04:22.619740 52452992 logging_unittest.cc:259] Log if every 1, iteration 20 +W0517 15:04:22.619751 2452992 logging_unittest.cc:263] log_if this + Op ID: test1 +I0517 15:04:22.619760 452992 logging_unittest.cc:267] array +I0517 15:04:22.619768 52992 logging_unittest.cc:269] const array +E0517 15:04:22.619776 2992 logging_unittest.cc:271] foo 1000 0000001000 3e8 + Op ID: test1 diff --git a/test/expected/test_gantt.sh_740f22814c5d856234c2450168dae22989c85d1e.err b/test/expected/test_gantt.sh_740f22814c5d856234c2450168dae22989c85d1e.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_gantt.sh_740f22814c5d856234c2450168dae22989c85d1e.out b/test/expected/test_gantt.sh_740f22814c5d856234c2450168dae22989c85d1e.out new file mode 100644 index 00000000..4ef8c5f6 --- /dev/null +++ b/test/expected/test_gantt.sh_740f22814c5d856234c2450168dae22989c85d1e.out @@ -0,0 +1,4 @@ + 2007-05-17T15:02 5m 2007-05-17T15:07 + 5m + Duration | ✘▲ | Operation +  ▃▃ test1 diff --git a/test/test_gantt.sh b/test/test_gantt.sh index b64c1307..80651c5c 100644 --- a/test/test_gantt.sh +++ b/test/test_gantt.sh @@ -4,6 +4,15 @@ export YES_COLOR=1 export TZ=UTC +run_cap_test ${lnav_test} -n \ + -c ";UPDATE all_logs set log_opid = 'test1' where log_line in (1, 3, 6)" \ + ${test_dir}/logfile_glog.0 + +run_cap_test ${lnav_test} -n \ + -c ";UPDATE all_logs set log_opid = 'test1' where log_line in (1, 3, 6)" \ + -c ':switch-to-view gantt' \ + ${test_dir}/logfile_glog.0 + run_cap_test ${lnav_test} -n \ -c ':switch-to-view gantt' \ ${test_dir}/logfile_generic.0