mirror of https://github.com/tstack/lnav
Timothy Stack
2 years ago
parent
1ea385ea3b
commit
6c52760cc9
@ -0,0 +1,26 @@
|
||||
{
|
||||
"$id": "https://lnav.org/event-file-format-detected-v1.schema.json",
|
||||
"title": "https://lnav.org/event-file-format-detected-v1.schema.json",
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"description": "Event fired when a log format is detected for a file.",
|
||||
"properties": {
|
||||
"$schema": {
|
||||
"title": "/$schema",
|
||||
"type": "string",
|
||||
"examples": [
|
||||
"https://lnav.org/event-file-format-detected-v1.schema.json"
|
||||
]
|
||||
},
|
||||
"filename": {
|
||||
"title": "/filename",
|
||||
"description": "The path of the file for which a matching format was found",
|
||||
"type": "string"
|
||||
},
|
||||
"format": {
|
||||
"title": "/format",
|
||||
"description": "The name of the format",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
{
|
||||
"$id": "https://lnav.org/event-file-open-v1.schema.json",
|
||||
"title": "https://lnav.org/event-file-open-v1.schema.json",
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"description": "Event fired when a file is opened.",
|
||||
"properties": {
|
||||
"$schema": {
|
||||
"title": "/$schema",
|
||||
"type": "string",
|
||||
"examples": [
|
||||
"https://lnav.org/event-file-open-v1.schema.json"
|
||||
]
|
||||
},
|
||||
"filename": {
|
||||
"title": "/filename",
|
||||
"description": "The path of the file that was opened",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
@ -0,0 +1,50 @@
|
||||
.. _Events:
|
||||
|
||||
Events (v0.10.2+)
|
||||
=================
|
||||
|
||||
The events mechanism allows **lnav** to be automated based on events that
|
||||
occur during processing. For example, filters could be added only when a
|
||||
particular log file format is detected instead of always installing them.
|
||||
Events are published through the :ref:`lnav_events<table_lnav_events>` SQLite
|
||||
table. Reacting to events can be done by creating a SQLite trigger on the
|
||||
table and inspecting the content of the event.
|
||||
|
||||
Trigger Example
|
||||
---------------
|
||||
|
||||
The following is an example of a trigger that adds an out filter when a
|
||||
syslog file is loaded. You can copy the code into an :file:`.sql` file and
|
||||
install it by running :code:`lnav -i my_trigger.sql`.
|
||||
|
||||
.. code-block:: sql
|
||||
:caption: my_trigger.sql
|
||||
:linenos:
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS add_format_specific_filters
|
||||
AFTER INSERT ON lnav_events WHEN
|
||||
-- Check the event type
|
||||
jget(NEW.content, '/$schema') =
|
||||
'https://lnav.org/event-file-format-detected-v1.schema.json' AND
|
||||
-- Only create the filter when a given format is seen
|
||||
jget(NEW.content, '/format') = 'syslog_log' AND
|
||||
-- Don't create the filter if it's already there
|
||||
NOT EXISTS (
|
||||
SELECT 1 FROM lnav_view_filters WHERE pattern = 'noisy message')
|
||||
BEGIN
|
||||
INSERT INTO lnav_view_filters (view_name, enabled, type, pattern) VALUES
|
||||
('log', 1, 'OUT', 'noisy message');
|
||||
END;
|
||||
|
||||
.. _event_reference:
|
||||
|
||||
Reference
|
||||
---------
|
||||
|
||||
The following tables describe the schema of the event JSON objects.
|
||||
|
||||
.. jsonschema:: ../schemas/event-file-open-v1.schema.json#
|
||||
:lift_description:
|
||||
|
||||
.. jsonschema:: ../schemas/event-file-format-detected-v1.schema.json#
|
||||
:lift_description:
|
||||
@ -0,0 +1,128 @@
|
||||
/**
|
||||
* Copyright (c) 2022, Timothy Stack
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* * Neither the name of Timothy Stack nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "lnav.events.hh"
|
||||
|
||||
namespace lnav {
|
||||
namespace events {
|
||||
|
||||
namespace file {
|
||||
|
||||
const std::string open::SCHEMA_ID
|
||||
= "https://lnav.org/event-file-open-v1.schema.json";
|
||||
|
||||
const auto open::handlers = typed_json_path_container<open>{
|
||||
yajlpp::property_handler("$schema").for_field(&open::o_schema)
|
||||
.with_example(open::SCHEMA_ID),
|
||||
yajlpp::property_handler("filename")
|
||||
.with_description("The path of the file that was opened")
|
||||
.for_field(&open::o_filename),
|
||||
}
|
||||
.with_schema_id(open::SCHEMA_ID)
|
||||
.with_description("Event fired when a file is opened.");
|
||||
|
||||
const std::string format_detected::SCHEMA_ID
|
||||
= "https://lnav.org/event-file-format-detected-v1.schema.json";
|
||||
|
||||
const auto format_detected::handlers = typed_json_path_container<format_detected>{
|
||||
yajlpp::property_handler("$schema").for_field(&format_detected::fd_schema)
|
||||
.with_example(format_detected::SCHEMA_ID),
|
||||
yajlpp::property_handler("filename")
|
||||
.with_description("The path of the file for which a matching format was found")
|
||||
.for_field(&format_detected::fd_filename),
|
||||
yajlpp::property_handler("format")
|
||||
.with_description("The name of the format")
|
||||
.for_field(&format_detected::fd_format),
|
||||
}
|
||||
.with_schema_id(format_detected::SCHEMA_ID)
|
||||
.with_description("Event fired when a log format is detected for a file.");
|
||||
|
||||
} // namespace file
|
||||
|
||||
int
|
||||
register_events_tab(sqlite3* db)
|
||||
{
|
||||
static const char* CREATE_EVENTS_TAB_SQL = R"(
|
||||
CREATE TABLE lnav_events (
|
||||
ts TEXT NOT NULL DEFAULT(strftime('%Y-%m-%dT%H:%M:%f', 'now')),
|
||||
content TEXT
|
||||
)
|
||||
)";
|
||||
static const char* DELETE_EVENTS_TRIGGER_SQL = R"(
|
||||
CREATE TRIGGER lnav_events_cleaner AFTER INSERT ON lnav_events
|
||||
BEGIN
|
||||
DELETE FROM lnav_events WHERE rowid <= NEW.rowid - 1000;
|
||||
END
|
||||
)";
|
||||
|
||||
auto_mem<char> errmsg(sqlite3_free);
|
||||
|
||||
if (sqlite3_exec(db, CREATE_EVENTS_TAB_SQL, nullptr, nullptr, errmsg.out())
|
||||
!= SQLITE_OK)
|
||||
{
|
||||
log_error("Unable to create events table: %s", errmsg.in());
|
||||
}
|
||||
if (sqlite3_exec(
|
||||
db, DELETE_EVENTS_TRIGGER_SQL, nullptr, nullptr, errmsg.out())
|
||||
!= SQLITE_OK)
|
||||
{
|
||||
log_error("Unable to create event cleaner trigger: %s", errmsg.in());
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void
|
||||
details::publish(sqlite3* db, const std::string& content)
|
||||
{
|
||||
static const char* INSERT_SQL = R"(
|
||||
INSERT INTO lnav_events (content) VALUES (?)
|
||||
)";
|
||||
|
||||
auto_mem<sqlite3_stmt> stmt(sqlite3_finalize);
|
||||
|
||||
if (sqlite3_prepare_v2(db, INSERT_SQL, -1, stmt.out(), nullptr)
|
||||
!= SQLITE_OK) {
|
||||
log_error("unable to prepare statement: %s", sqlite3_errmsg(db));
|
||||
return;
|
||||
}
|
||||
if (sqlite3_bind_text(
|
||||
stmt.in(), 1, content.c_str(), content.size(), SQLITE_TRANSIENT)
|
||||
!= SQLITE_OK)
|
||||
{
|
||||
log_error("unable to bind parameter: %s", sqlite3_errmsg(db));
|
||||
return;
|
||||
}
|
||||
if (sqlite3_step(stmt) != SQLITE_DONE) {
|
||||
log_error("failed to execute insert: %s", sqlite3_errmsg(db));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace events
|
||||
} // namespace lnav
|
||||
@ -0,0 +1,100 @@
|
||||
/**
|
||||
* Copyright (c) 2022, Timothy Stack
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
* * Neither the name of Timothy Stack nor the names of its contributors
|
||||
* may be used to endorse or promote products derived from this software
|
||||
* without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef lnav_events_hh
|
||||
#define lnav_events_hh
|
||||
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include "yajlpp/yajlpp_def.hh"
|
||||
|
||||
namespace lnav {
|
||||
namespace events {
|
||||
|
||||
namespace file {
|
||||
|
||||
struct open {
|
||||
std::string o_filename;
|
||||
std::string o_schema{SCHEMA_ID};
|
||||
|
||||
static const std::string SCHEMA_ID;
|
||||
static const typed_json_path_container<open> handlers;
|
||||
};
|
||||
|
||||
struct format_detected {
|
||||
std::string fd_filename;
|
||||
std::string fd_format;
|
||||
std::string fd_schema{SCHEMA_ID};
|
||||
|
||||
static const std::string SCHEMA_ID;
|
||||
static const typed_json_path_container<format_detected> handlers;
|
||||
};
|
||||
|
||||
} // namespace file
|
||||
|
||||
int register_events_tab(sqlite3* db);
|
||||
|
||||
namespace details {
|
||||
void publish(sqlite3* db, const std::string& content);
|
||||
} // namespace details
|
||||
|
||||
template<typename T>
|
||||
void
|
||||
publish(sqlite3* db, T event)
|
||||
{
|
||||
auto serialized = T::handlers.to_string(event);
|
||||
|
||||
details::publish(db, serialized);
|
||||
}
|
||||
|
||||
template<typename T, typename F>
|
||||
void
|
||||
publish(sqlite3* db, const T& container, F func)
|
||||
{
|
||||
auto_mem<char> errmsg(sqlite3_free);
|
||||
|
||||
if (sqlite3_exec(db, "BEGIN TRANSACTION", nullptr, nullptr, errmsg.out())
|
||||
!= SQLITE_OK)
|
||||
{
|
||||
log_error("unable to start event transaction: %s", errmsg.in());
|
||||
}
|
||||
for (const auto& elem : container) {
|
||||
publish(db, func(elem));
|
||||
}
|
||||
if (sqlite3_exec(db, "COMMIT TRANSACTION", nullptr, nullptr, errmsg.out())
|
||||
!= SQLITE_OK)
|
||||
{
|
||||
log_error("unable to commit event transaction: %s", errmsg.in());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace events
|
||||
} // namespace lnav
|
||||
|
||||
#endif
|
||||
@ -0,0 +1,2 @@
|
||||
{"content":{"$schema":"https://lnav.org/event-file-open-v1.schema.json","filename":"{test_dir}/logfile_access_log.0"}}
|
||||
{"content":{"$schema":"https://lnav.org/event-file-format-detected-v1.schema.json","filename":"{test_dir}/logfile_access_log.0","format":"access_log"}}
|
||||
@ -0,0 +1,10 @@
|
||||
{
|
||||
"$schema": "https://lnav.org/schemas/format-v1.schema.json",
|
||||
"unit_test_log": {
|
||||
"regex": {
|
||||
"std": {
|
||||
"pattern": "\\[(?<timestamp>\\d+\\/\\d+\\/\\d+ \\d+:\\d+:\\d+) (?<jobserver>[\\w.]+) (?<workqueue>[\\w.]+) (?<processid>\\d+)\\] (?<body>.*)$"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,2 @@
|
||||
regex101-home/.lnav/formats/installed/unit_test_log.json
|
||||
regex101-home/.lnav/formats/installed/unit_test_log.regex101-zpEnjV.json
|
||||
@ -0,0 +1,34 @@
|
||||
{
|
||||
"$schema": "https://lnav.org/schemas/format-v1.schema.json",
|
||||
"unit_test_log": {
|
||||
"description": "Format file generated from regex101 entry -- https://regex101.com/r/zpEnjV/2",
|
||||
"regex": {
|
||||
"std": {
|
||||
"pattern": "\\[(?<timestamp>\\d+\\/\\d+\\/\\d+ \\d+:\\d+:\\d+) (?<jobserver>[\\w.]+) (?<workqueue>[\\w.]+) (?<processid>\\d+)\\] (?<body>.*)$"
|
||||
}
|
||||
},
|
||||
"value": {
|
||||
"jobserver": {
|
||||
"kind": "string"
|
||||
},
|
||||
"processid": {
|
||||
"kind": "string"
|
||||
},
|
||||
"timestamp": {
|
||||
"kind": "string"
|
||||
},
|
||||
"workqueue": {
|
||||
"kind": "string"
|
||||
}
|
||||
},
|
||||
"sample": [
|
||||
{
|
||||
"line": "[03/22/2021 02:00:02 job1074.example.com db.db81.example_events 54026] {\"ELAPSED\":\"0.011\",\"LEVEL\":\"info\",\"MESSAGE\":\"finished in 0.011\\n\",\"PREFIX\":\"YFgyWQriCmsAAofJAAAAHg\",\"ROUTINGKEY\":\"EXAMPLE1366.Example.Events._Publish\"}"
|
||||
},
|
||||
{
|
||||
"description": "sample 1",
|
||||
"line": "[03/22/2021 02:00:02 job1074.example.com db.db81.example_events 54026] {\"ELAPSED\":\"0.011\",\"LEVEL\":\"info\",\"MESSAGE\":\"finished in 0.011\\n\",\"PREFIX\":\"YFgyWQriCmsAAofJAAAAHg\",\"ROUTINGKEY\":\"EXAMPLE1366.Example.Events._Publish\"}"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
{
|
||||
"$schema": "https://lnav.org/schemas/format-v1.schema.json",
|
||||
"unit_test_log": {
|
||||
"regex": {
|
||||
"alt": {
|
||||
"pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?)Z?\\](?<body>.*)$"
|
||||
}
|
||||
},
|
||||
"sample": [
|
||||
{
|
||||
"line": "[2021-05-21T21:58:57.022497Z]"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,6 @@
|
||||
#! /bin/bash
|
||||
|
||||
run_cap_test ${lnav_test} -n \
|
||||
-c ';SELECT json(content) as content FROM lnav_events' \
|
||||
-c ':write-jsonlines-to -' \
|
||||
${test_dir}/logfile_access_log.0
|
||||
Loading…
Reference in New Issue