mirror of https://github.com/tstack/lnav
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
490 lines
15 KiB
C++
490 lines
15 KiB
C++
/**
|
|
* Copyright (c) 2020, Timothy Stack
|
|
*
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are met:
|
|
*
|
|
* * Redistributions of source code must retain the above copyright notice, this
|
|
* list of conditions and the following disclaimer.
|
|
* * Redistributions in binary form must reproduce the above copyright notice,
|
|
* this list of conditions and the following disclaimer in the documentation
|
|
* and/or other materials provided with the distribution.
|
|
* * Neither the name of Timothy Stack nor the names of its contributors
|
|
* may be used to endorse or promote products derived from this software
|
|
* without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
|
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
|
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
|
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
|
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* @file log_format_ext.hh
|
|
*/
|
|
|
|
#ifndef lnav_log_format_ext_hh
|
|
#define lnav_log_format_ext_hh
|
|
|
|
#include <unordered_map>
|
|
|
|
#include "log_format.hh"
|
|
#include "yajlpp/yajlpp.hh"
|
|
|
|
class module_format;
|
|
|
|
class external_log_format : public log_format {
|
|
public:
|
|
struct sample {
|
|
sample() : s_level(LEVEL_UNKNOWN) {}
|
|
|
|
positioned_property<std::string> s_line;
|
|
log_level_t s_level;
|
|
};
|
|
|
|
struct value_def {
|
|
value_def(intern_string_t name,
|
|
value_kind_t kind,
|
|
int col,
|
|
log_format* format)
|
|
: vd_meta(name, kind, col, format){};
|
|
|
|
logline_value_meta vd_meta;
|
|
std::string vd_collate;
|
|
bool vd_foreign_key{false};
|
|
intern_string_t vd_unit_field;
|
|
std::map<const intern_string_t, scaling_factor> vd_unit_scaling;
|
|
ssize_t vd_values_index{-1};
|
|
bool vd_internal{false};
|
|
std::vector<std::string> vd_action_list;
|
|
std::string vd_rewriter;
|
|
std::string vd_description;
|
|
};
|
|
|
|
struct indexed_value_def {
|
|
indexed_value_def(int index = -1,
|
|
int unit_index = -1,
|
|
std::shared_ptr<value_def> vd = nullptr)
|
|
: ivd_index(index), ivd_unit_field_index(unit_index),
|
|
ivd_value_def(std::move(vd))
|
|
{
|
|
}
|
|
|
|
int ivd_index;
|
|
int ivd_unit_field_index;
|
|
std::shared_ptr<value_def> ivd_value_def;
|
|
|
|
bool operator<(const indexed_value_def& rhs) const
|
|
{
|
|
return this->ivd_index < rhs.ivd_index;
|
|
}
|
|
};
|
|
|
|
struct pattern {
|
|
std::string p_name;
|
|
std::string p_config_path;
|
|
std::shared_ptr<pcrepp> p_pcre;
|
|
std::vector<indexed_value_def> p_value_by_index;
|
|
std::vector<int> p_numeric_value_indexes;
|
|
int p_timestamp_field_index{-1};
|
|
int p_level_field_index{-1};
|
|
int p_module_field_index{-1};
|
|
int p_opid_field_index{-1};
|
|
int p_body_field_index{-1};
|
|
int p_timestamp_end{-1};
|
|
bool p_module_format{false};
|
|
};
|
|
|
|
struct level_pattern {
|
|
std::string lp_regex;
|
|
std::shared_ptr<pcrepp> lp_pcre;
|
|
};
|
|
|
|
struct yajl_handle_deleter {
|
|
void operator()(yajl_handle handle) const
|
|
{
|
|
if (handle != nullptr) {
|
|
yajl_free(handle);
|
|
}
|
|
}
|
|
};
|
|
|
|
external_log_format(const intern_string_t name)
|
|
: elf_level_field(intern_string::lookup("level", -1)),
|
|
elf_body_field(intern_string::lookup("body", -1)),
|
|
jlf_yajl_handle(nullptr, yajl_handle_deleter()), elf_name(name)
|
|
{
|
|
this->jlf_line_offsets.reserve(128);
|
|
};
|
|
|
|
const intern_string_t get_name() const
|
|
{
|
|
return this->elf_name;
|
|
};
|
|
|
|
bool match_name(const std::string& filename);
|
|
|
|
bool match_mime_type(const file_format_t ff) const;
|
|
|
|
scan_result_t scan(logfile& lf,
|
|
std::vector<logline>& dst,
|
|
const line_info& offset,
|
|
shared_buffer_ref& sbr);
|
|
|
|
bool scan_for_partial(shared_buffer_ref& sbr, size_t& len_out) const;
|
|
|
|
void annotate(uint64_t line_number,
|
|
shared_buffer_ref& line,
|
|
string_attrs_t& sa,
|
|
std::vector<logline_value>& values,
|
|
bool annotate_module = true) const;
|
|
|
|
void rewrite(exec_context& ec,
|
|
shared_buffer_ref& line,
|
|
string_attrs_t& sa,
|
|
std::string& value_out);
|
|
|
|
void build(std::vector<lnav::console::user_message>& errors);
|
|
|
|
void register_vtabs(log_vtab_manager* vtab_manager,
|
|
std::vector<lnav::console::user_message>& errors);
|
|
|
|
bool match_samples(const std::vector<sample>& samples) const;
|
|
|
|
bool hide_field(const intern_string_t field_name, bool val)
|
|
{
|
|
auto vd_iter = this->elf_value_defs.find(field_name);
|
|
|
|
if (vd_iter == this->elf_value_defs.end()) {
|
|
return false;
|
|
}
|
|
|
|
vd_iter->second->vd_meta.lvm_user_hidden = val;
|
|
return true;
|
|
};
|
|
|
|
std::shared_ptr<log_format> specialized(int fmt_lock);
|
|
|
|
const logline_value_stats* stats_for_value(
|
|
const intern_string_t& name) const
|
|
{
|
|
const logline_value_stats* retval = nullptr;
|
|
|
|
for (size_t lpc = 0; lpc < this->elf_numeric_value_defs.size(); lpc++) {
|
|
value_def& vd = *this->elf_numeric_value_defs[lpc];
|
|
|
|
if (vd.vd_meta.lvm_name == name) {
|
|
retval = &this->lf_value_stats[lpc];
|
|
break;
|
|
}
|
|
}
|
|
|
|
return retval;
|
|
};
|
|
|
|
void get_subline(const logline& ll,
|
|
shared_buffer_ref& sbr,
|
|
bool full_message);
|
|
|
|
std::shared_ptr<log_vtab_impl> get_vtab_impl() const;
|
|
|
|
const std::vector<std::string>* get_actions(const logline_value& lv) const
|
|
{
|
|
const std::vector<std::string>* retval = nullptr;
|
|
|
|
const auto iter = this->elf_value_defs.find(lv.lv_meta.lvm_name);
|
|
if (iter != this->elf_value_defs.end()) {
|
|
retval = &iter->second->vd_action_list;
|
|
}
|
|
|
|
return retval;
|
|
};
|
|
|
|
std::set<std::string> get_source_path() const
|
|
{
|
|
return this->elf_source_path;
|
|
};
|
|
|
|
enum json_log_field {
|
|
JLF_CONSTANT,
|
|
JLF_VARIABLE
|
|
};
|
|
|
|
struct json_format_element {
|
|
enum class align_t {
|
|
LEFT,
|
|
RIGHT,
|
|
};
|
|
|
|
enum class overflow_t {
|
|
ABBREV,
|
|
TRUNCATE,
|
|
DOTDOT,
|
|
};
|
|
|
|
enum class transform_t {
|
|
NONE,
|
|
UPPERCASE,
|
|
LOWERCASE,
|
|
CAPITALIZE,
|
|
};
|
|
|
|
json_format_element()
|
|
: jfe_type(JLF_CONSTANT), jfe_default_value("-"), jfe_min_width(0),
|
|
jfe_max_width(LLONG_MAX), jfe_align(align_t::LEFT),
|
|
jfe_overflow(overflow_t::ABBREV),
|
|
jfe_text_transform(transform_t::NONE){};
|
|
|
|
json_log_field jfe_type;
|
|
positioned_property<intern_string_t> jfe_value;
|
|
std::string jfe_default_value;
|
|
long long jfe_min_width;
|
|
long long jfe_max_width;
|
|
align_t jfe_align;
|
|
overflow_t jfe_overflow;
|
|
transform_t jfe_text_transform;
|
|
std::string jfe_ts_format;
|
|
};
|
|
|
|
struct json_field_cmp {
|
|
json_field_cmp(json_log_field type, const intern_string_t name)
|
|
: jfc_type(type), jfc_field_name(name){};
|
|
|
|
bool operator()(const json_format_element& jfe) const
|
|
{
|
|
return (this->jfc_type == jfe.jfe_type
|
|
&& this->jfc_field_name == jfe.jfe_value.pp_value);
|
|
};
|
|
|
|
json_log_field jfc_type;
|
|
const intern_string_t jfc_field_name;
|
|
};
|
|
|
|
struct highlighter_def {
|
|
highlighter_def() : hd_underline(false), hd_blink(false) {}
|
|
|
|
std::shared_ptr<pcrepp> hd_pattern;
|
|
positioned_property<std::string> hd_color;
|
|
positioned_property<std::string> hd_background_color;
|
|
bool hd_underline;
|
|
bool hd_blink;
|
|
};
|
|
|
|
long value_line_count(const intern_string_t ist,
|
|
bool top_level,
|
|
const unsigned char* str = nullptr,
|
|
ssize_t len = -1) const
|
|
{
|
|
const auto iter = this->elf_value_defs.find(ist);
|
|
long line_count
|
|
= (str != NULL) ? std::count(&str[0], &str[len], '\n') + 1 : 1;
|
|
|
|
if (iter == this->elf_value_defs.end()) {
|
|
return (this->jlf_hide_extra || !top_level) ? 0 : line_count;
|
|
}
|
|
|
|
if (iter->second->vd_meta.lvm_hidden) {
|
|
return 0;
|
|
}
|
|
|
|
if (std::find_if(this->jlf_line_format.begin(),
|
|
this->jlf_line_format.end(),
|
|
json_field_cmp(JLF_VARIABLE, ist))
|
|
!= this->jlf_line_format.end())
|
|
{
|
|
return line_count - 1;
|
|
}
|
|
|
|
return line_count;
|
|
};
|
|
|
|
bool has_value_def(const intern_string_t ist) const
|
|
{
|
|
const auto iter = this->elf_value_defs.find(ist);
|
|
|
|
return iter != this->elf_value_defs.end();
|
|
};
|
|
|
|
std::string get_pattern_name(uint64_t line_number) const
|
|
{
|
|
if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
|
|
return "structured";
|
|
}
|
|
int pat_index = this->pattern_index_for_line(line_number);
|
|
return this->elf_pattern_order[pat_index]->p_config_path;
|
|
}
|
|
|
|
std::string get_pattern_regex(uint64_t line_number) const
|
|
{
|
|
if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
|
|
return "";
|
|
}
|
|
int pat_index = this->pattern_index_for_line(line_number);
|
|
return this->elf_pattern_order[pat_index]->p_pcre->get_pattern();
|
|
}
|
|
|
|
log_level_t convert_level(const pcre_input& pi,
|
|
const pcre_context::capture_t* level_cap) const
|
|
{
|
|
log_level_t retval = LEVEL_INFO;
|
|
|
|
if (level_cap != nullptr && level_cap->is_valid()) {
|
|
pcre_context_static<128> pc_level;
|
|
pcre_input pi_level(
|
|
pi.get_substr_start(level_cap), 0, level_cap->length());
|
|
|
|
if (this->elf_level_patterns.empty()) {
|
|
retval
|
|
= string2level(pi_level.get_string(), level_cap->length());
|
|
} else {
|
|
for (const auto& elf_level_pattern : this->elf_level_patterns) {
|
|
if (elf_level_pattern.second.lp_pcre->match(pc_level,
|
|
pi_level)) {
|
|
retval = elf_level_pattern.first;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return retval;
|
|
}
|
|
|
|
using mod_map_t = std::map<intern_string_t, module_format>;
|
|
static mod_map_t MODULE_FORMATS;
|
|
static std::vector<std::shared_ptr<external_log_format>>
|
|
GRAPH_ORDERED_FORMATS;
|
|
|
|
std::set<std::string> elf_source_path;
|
|
std::unordered_map<std::string, int> elf_format_sources;
|
|
std::list<intern_string_t> elf_collision;
|
|
std::string elf_file_pattern;
|
|
std::set<file_format_t> elf_mime_types;
|
|
std::shared_ptr<pcrepp> elf_filename_pcre;
|
|
std::map<std::string, std::shared_ptr<pattern>> elf_patterns;
|
|
std::vector<std::shared_ptr<pattern>> elf_pattern_order;
|
|
std::vector<sample> elf_samples;
|
|
std::unordered_map<const intern_string_t, std::shared_ptr<value_def>>
|
|
elf_value_defs;
|
|
std::vector<std::shared_ptr<value_def>> elf_value_def_order;
|
|
std::vector<std::shared_ptr<value_def>> elf_numeric_value_defs;
|
|
int elf_column_count{0};
|
|
double elf_timestamp_divisor{1.0};
|
|
intern_string_t elf_level_field;
|
|
pcrepp elf_level_pointer;
|
|
intern_string_t elf_body_field;
|
|
intern_string_t elf_module_id_field;
|
|
intern_string_t elf_opid_field;
|
|
std::map<log_level_t, level_pattern> elf_level_patterns;
|
|
std::vector<std::pair<int64_t, log_level_t>> elf_level_pairs;
|
|
bool elf_container{false};
|
|
bool elf_has_module_format{false};
|
|
bool elf_builtin_format{false};
|
|
|
|
struct search_table_def {
|
|
std::shared_ptr<pcrepp> std_pattern;
|
|
};
|
|
|
|
std::map<intern_string_t, search_table_def> elf_search_tables;
|
|
std::map<const intern_string_t, highlighter_def> elf_highlighter_patterns;
|
|
|
|
enum class elf_type_t {
|
|
ELF_TYPE_TEXT,
|
|
ELF_TYPE_JSON,
|
|
ELF_TYPE_CSV,
|
|
};
|
|
|
|
elf_type_t elf_type{elf_type_t::ELF_TYPE_TEXT};
|
|
|
|
void json_append_to_cache(const char* value, ssize_t len)
|
|
{
|
|
size_t old_size = this->jlf_cached_line.size();
|
|
if (len == -1) {
|
|
len = strlen(value);
|
|
}
|
|
this->jlf_cached_line.resize(old_size + len);
|
|
memcpy(&(this->jlf_cached_line[old_size]), value, len);
|
|
};
|
|
|
|
void json_append_to_cache(ssize_t len)
|
|
{
|
|
size_t old_size = this->jlf_cached_line.size();
|
|
this->jlf_cached_line.resize(old_size + len);
|
|
memset(&this->jlf_cached_line[old_size], ' ', len);
|
|
};
|
|
|
|
void json_append(const json_format_element& jfe,
|
|
const char* value,
|
|
ssize_t len)
|
|
{
|
|
if (len == -1) {
|
|
len = strlen(value);
|
|
}
|
|
if (jfe.jfe_align == json_format_element::align_t::RIGHT) {
|
|
if (len < jfe.jfe_min_width) {
|
|
this->json_append_to_cache(jfe.jfe_min_width - len);
|
|
}
|
|
}
|
|
this->json_append_to_cache(value, len);
|
|
if (jfe.jfe_align == json_format_element::align_t::LEFT) {
|
|
if (len < jfe.jfe_min_width) {
|
|
this->json_append_to_cache(jfe.jfe_min_width - len);
|
|
}
|
|
}
|
|
};
|
|
|
|
logline_value_meta get_value_meta(intern_string_t field_name,
|
|
value_kind_t kind)
|
|
{
|
|
auto iter = this->elf_value_defs.find(field_name);
|
|
|
|
if (iter == this->elf_value_defs.end()) {
|
|
auto retval = logline_value_meta(field_name, kind, -1, this);
|
|
|
|
retval.lvm_hidden = this->jlf_hide_extra;
|
|
return retval;
|
|
}
|
|
|
|
auto lvm = iter->second->vd_meta;
|
|
|
|
lvm.lvm_kind = kind;
|
|
return lvm;
|
|
}
|
|
|
|
std::vector<lnav::console::snippet> get_snippets() const;
|
|
|
|
bool jlf_hide_extra{false};
|
|
std::vector<json_format_element> jlf_line_format;
|
|
int jlf_line_format_init_count{0};
|
|
std::vector<logline_value> jlf_line_values;
|
|
|
|
off_t jlf_cached_offset{-1};
|
|
bool jlf_cached_full{false};
|
|
std::vector<off_t> jlf_line_offsets;
|
|
shared_buffer jlf_share_manager;
|
|
std::vector<char> jlf_cached_line;
|
|
string_attrs_t jlf_line_attrs;
|
|
std::shared_ptr<yajlpp_parse_context> jlf_parse_context;
|
|
std::shared_ptr<yajl_handle_t> jlf_yajl_handle;
|
|
|
|
private:
|
|
const intern_string_t elf_name;
|
|
|
|
static uint8_t module_scan(const pcre_input& pi,
|
|
pcre_context::capture_t* body_cap,
|
|
const intern_string_t& mod_name);
|
|
};
|
|
|
|
class module_format {
|
|
public:
|
|
std::shared_ptr<log_format> mf_mod_format;
|
|
};
|
|
|
|
#endif
|