diff --git a/NEWS.md b/NEWS.md index 039c27c2..186c72e0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -34,13 +34,18 @@ Features: * Added `config get` and `config blame` management CLI commands to get the current configuration and the file locations where the configuration options came from. - -Bug Fixes: * When piping data into **lnav**'s stdin, the input used to only be written to a single file without any rotation. Now, the input is written to a directory of rotating files. The same is true for the command-lines executed through the - new `:sh` command. + new `:sh` command. The piped data can be managed using the + new `piper` commands in the management CLI. +* The `$LNAV_HOME_DIR` and `$LNAV_WORK_DIR` environment + variables are now defined inside **lnav** and refer to + the location of the user's configuration directory and + the directory where cached data is stored, respectively. + +Bug Fixes: * Binary data piped into stdin should now be treated the same as if it was in a file that was passed on the command-line. * The `-I` option is now recognized in the management CLI @@ -62,6 +67,9 @@ Breaking changes: * Removed the `-t` command-line flag. Text data fed in on stdin and captured from a `:sh` execution is automatically timestamped. +* Data piped into **lnav** is now stored in the work + directory instead of the `stdin-captures` dot-lnav + directory. ## lnav v0.11.2 diff --git a/docs/schemas/config-v1.schema.json b/docs/schemas/config-v1.schema.json index 705ae0a8..97dde9da 100644 --- a/docs/schemas/config-v1.schema.json +++ b/docs/schemas/config-v1.schema.json @@ -55,6 +55,15 @@ "description": "The number of rotated files to keep", "type": "integer", "minimum": 2 + }, + "ttl": { + "title": "/tuning/piper/ttl", + "description": "The time-to-live for captured data, expressed as a duration (e.g. '3d' for three days)", + "type": "string", + "examples": [ + "3d", + "12h" + ] } }, "additionalProperties": false diff --git a/docs/source/cli.rst b/docs/source/cli.rst index e5f5bbd8..d58148b3 100644 --- a/docs/source/cli.rst +++ b/docs/source/cli.rst @@ -123,9 +123,14 @@ Subcommands Print out the configuration options as JSON-Pointers and the file/line-number where the configuration is sourced from. -.. option:: regex101 import [] +.. option:: format get - Convert a regex101.com entry into a skeleton log format file. + Print information about the given log format. + +.. option:: format source + + Print the name of the first file that contained this log format + definition. .. option:: format regex push @@ -135,6 +140,22 @@ Subcommands Pull changes to a regex that was previously pushed to regex101.com . +.. option:: piper clean + + Remove all of the files that stored data that was piped into **lnav**. + +.. option:: piper list + + List all of the data that was piped into **lnav** from oldest to newest. + The listing will show the creation time, the URL you can use to reopen + the data, and a description of the data. Passing the :option:`-v` + option will print out additional metadata that was captured, such as + the current working directory of **lnav** and the environment variables. + +.. option:: regex101 import [] + + Convert a regex101.com entry into a skeleton log format file. + Environment Variables --------------------- diff --git a/docs/source/formats.rst b/docs/source/formats.rst index d30ba8c3..45dc2c31 100644 --- a/docs/source/formats.rst +++ b/docs/source/formats.rst @@ -479,6 +479,15 @@ hash are treated as comments. The following variables are defined in a script: The arguments passed to the script. +.. envvar:: LNAV_HOME_DIR + + The path to the directory where the user's **lnav** configuration is stored. + +.. envvar:: LNAV_WORK_DIR + + The path to the directory where **lnav** caches files, like archives that + have been unpacked or piper captures. + Remember that you need to use the :ref:`:eval` command when referencing variables in most **lnav** commands. Scripts can provide help text to be displayed during interactive usage by adding the following tags in a comment diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b09952c4..c14a4f58 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -194,7 +194,9 @@ set(BUILTIN_LNAV_SCRIPTS scripts/docker-url-handler.lnav scripts/lnav-pop-view.lnav scripts/partition-by-boot.lnav scripts/rename-stdin.lnav - scripts/search-for.lnav) + scripts/search-for.lnav + scripts/workdir-url-handler.lnav + ) set(BUILTIN_LNAV_SCRIPT_PATHS ${BUILTIN_LNAV_SCRIPTS}) diff --git a/src/base/CMakeLists.txt b/src/base/CMakeLists.txt index 72c28c29..0b87cb92 100644 --- a/src/base/CMakeLists.txt +++ b/src/base/CMakeLists.txt @@ -19,6 +19,7 @@ add_library( lnav_log.cc network.tcp.cc paths.cc + piper.file.cc snippet_highlighters.cc string_attr_type.cc string_util.cc @@ -55,6 +56,7 @@ add_library( math_util.hh network.tcp.hh paths.hh + piper.file.hh result.h snippet_highlighters.hh string_attr_type.hh diff --git a/src/base/Makefile.am b/src/base/Makefile.am index 87d38bb5..7df65f35 100644 --- a/src/base/Makefile.am +++ b/src/base/Makefile.am @@ -55,6 +55,7 @@ noinst_HEADERS = \ network.tcp.hh \ opt_util.hh \ paths.hh \ + piper.file.hh \ result.h \ snippet_highlighters.hh \ string_attr_type.hh \ @@ -82,6 +83,7 @@ libbase_a_SOURCES = \ lnav_log.cc \ network.tcp.cc \ paths.cc \ + piper.file.cc \ snippet_highlighters.cc \ string_attr_type.cc \ string_util.cc \ diff --git a/src/base/auto_mem.hh b/src/base/auto_mem.hh index 25bea2e2..b404a1bc 100644 --- a/src/base/auto_mem.hh +++ b/src/base/auto_mem.hh @@ -66,6 +66,16 @@ public: return retval; } + static auto_mem calloc(size_t count) + { + return auto_mem(static_cast(::calloc(count, sizeof(T)))); + } + + static auto_mem malloc(size_t sz) + { + return auto_mem(static_cast(::malloc(sz))); + } + explicit auto_mem(T* ptr = nullptr) : am_ptr(ptr), am_free_func(default_free) { diff --git a/src/base/intern_string.hh b/src/base/intern_string.hh index 1f25d660..3505663e 100644 --- a/src/base/intern_string.hh +++ b/src/base/intern_string.hh @@ -434,8 +434,10 @@ struct string_fragment { }); } + using split_when_result = std::pair; + template - split_result split_when(P&& predicate) const + split_when_result split_when(P&& predicate) const { int consumed = 0; while (consumed < this->length()) { @@ -446,7 +448,33 @@ struct string_fragment { consumed += 1; } - if (consumed == 0) { + return std::make_pair( + string_fragment{ + this->sf_string, + this->sf_begin, + this->sf_begin + consumed, + }, + string_fragment{ + this->sf_string, + this->sf_begin + consumed + + ((consumed == this->length()) ? 0 : 1), + this->sf_end, + }); + } + + template + split_result split_pair(P&& predicate) const + { + int consumed = 0; + while (consumed < this->length()) { + if (predicate(this->data()[consumed])) { + break; + } + + consumed += 1; + } + + if (consumed == this->length()) { return nonstd::nullopt; } @@ -843,6 +871,12 @@ to_string_fragment(const std::string& s) return string_fragment(s.c_str(), 0, s.length()); } +inline string_fragment +to_string_fragment(const scn::string_view& sv) +{ + return string_fragment::from_bytes(sv.data(), sv.length()); +} + struct frag_hasher { size_t operator()(const string_fragment& sf) const { diff --git a/src/base/lnav.gzip.hh b/src/base/lnav.gzip.hh index bd739653..f2828257 100644 --- a/src/base/lnav.gzip.hh +++ b/src/base/lnav.gzip.hh @@ -40,6 +40,19 @@ namespace lnav { namespace gzip { +struct header { + timeval h_mtime{}; + auto_buffer h_extra{auto_buffer::alloc(0)}; + std::string h_name; + std::string h_comment; + + bool empty() const + { + return this->h_mtime.tv_sec == 0 && this->h_extra.empty() + && this->h_name.empty() && this->h_comment.empty(); + } +}; + bool is_gzipped(const char* buffer, size_t len); Result compress(const void* input, size_t len); diff --git a/src/base/piper.file.cc b/src/base/piper.file.cc new file mode 100644 index 00000000..8bb7bbda --- /dev/null +++ b/src/base/piper.file.cc @@ -0,0 +1,80 @@ +/** + * Copyright (c) 2023, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "piper.file.hh" + +#include + +#include "base/lnav_log.hh" +#include "base/paths.hh" + +namespace lnav { +namespace piper { + +const char HEADER_MAGIC[4] = {'L', 0, 'N', 1}; + +const ghc::filesystem::path& +storage_path() +{ + static auto INSTANCE = lnav::paths::workdir() / "piper"; + + return INSTANCE; +} + +nonstd::optional +read_header(int fd, const char* first8) +{ + if (memcmp(first8, HEADER_MAGIC, sizeof(HEADER_MAGIC)) != 0) { + log_trace("first 4 bytes are not a piper header: %02x%02x%02x%02x", + first8[0], + first8[1], + first8[2], + first8[3]); + return nonstd::nullopt; + } + + uint32_t meta_size = ntohl(*((uint32_t*) &first8[4])); + + auto meta_buf = auto_buffer::alloc(meta_size); + if (meta_buf.in() == nullptr) { + log_error("failed to alloc %d bytes for header", meta_size); + return nonstd::nullopt; + } + auto meta_prc = pread(fd, meta_buf.in(), meta_size, 8); + if (meta_prc != meta_size) { + log_error("failed to read piper header: %s", strerror(errno)); + return nonstd::nullopt; + } + meta_buf.resize(meta_size); + + return meta_buf; +} + +} // namespace piper +} // namespace lnav diff --git a/src/base/piper.file.hh b/src/base/piper.file.hh new file mode 100644 index 00000000..7a263ea2 --- /dev/null +++ b/src/base/piper.file.hh @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2023, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef lnav_piper_file_hh +#define lnav_piper_file_hh + +#include +#include + +#include + +#include "auto_mem.hh" +#include "ghc/filesystem.hpp" +#include "optional.hpp" +#include "time_util.hh" + +namespace lnav { +namespace piper { + +struct header { + timeval h_ctime{}; + std::string h_name; + std::string h_cwd; + std::map h_env; + + bool operator<(const header& rhs) const + { + if (this->h_ctime < rhs.h_ctime) { + return true; + } + + if (this->h_ctime == rhs.h_ctime) { + return this->h_name < rhs.h_name; + } + + return false; + } +}; + +const ghc::filesystem::path& storage_path(); + +constexpr size_t HEADER_SIZE = 8; +extern const char HEADER_MAGIC[4]; + +nonstd::optional read_header(int fd, const char* first8); + +} // namespace piper +} // namespace lnav + +#endif diff --git a/src/file_converter_manager.cc b/src/file_converter_manager.cc index f06953e3..6f016522 100644 --- a/src/file_converter_manager.cc +++ b/src/file_converter_manager.cc @@ -36,20 +36,31 @@ #include #include "base/fs_util.hh" +#include "base/injector.hh" #include "base/paths.hh" #include "config.h" #include "line_buffer.hh" +#include "piper.looper.cfg.hh" namespace file_converter_manager { +static const ghc::filesystem::path& +cache_dir() +{ + static auto INSTANCE = lnav::paths::workdir() / "conversion"; + + return INSTANCE; +} + Result convert(const external_file_format& eff, const std::string& filename) { log_info("attempting to convert file -- %s", filename.c_str()); - ghc::filesystem::create_directories(lnav::paths::workdir()); - auto outfile = TRY(lnav::filesystem::open_temp_file(lnav::paths::workdir() - / "conversion.XXXXXX")); + ghc::filesystem::create_directories(cache_dir()); + auto outfile = TRY(lnav::filesystem::open_temp_file( + cache_dir() + / fmt::format(FMT_STRING("{}.XXXXXX"), eff.eff_format_name))); auto err_pipe = TRY(auto_pipe::for_child_fd(STDERR_FILENO)); auto child = TRY(lnav::pid::from_fork()); @@ -144,4 +155,32 @@ convert(const external_file_format& eff, const std::string& filename) }); } +void +cleanup() +{ + (void) std::async(std::launch::async, []() { + const auto& cfg = injector::get(); + auto now = std::chrono::system_clock::now(); + auto cache_path = cache_dir(); + std::vector to_remove; + + for (const auto& entry : + ghc::filesystem::directory_iterator(cache_path)) + { + auto mtime = ghc::filesystem::last_write_time(entry.path()); + auto exp_time = mtime + cfg.c_ttl; + if (now < exp_time) { + continue; + } + + to_remove.emplace_back(entry); + } + + for (auto& entry : to_remove) { + log_debug("removing conversion: %s", entry.c_str()); + ghc::filesystem::remove_all(entry); + } + }); +} + } // namespace file_converter_manager diff --git a/src/file_converter_manager.hh b/src/file_converter_manager.hh index dd8c5eee..281f195a 100644 --- a/src/file_converter_manager.hh +++ b/src/file_converter_manager.hh @@ -50,6 +50,8 @@ struct convert_result { Result convert(const external_file_format& eff, const std::string& filename); +void cleanup(); + } // namespace file_converter_manager #endif diff --git a/src/file_vtab.cc b/src/file_vtab.cc index 3ddc56ad..67a70cfb 100644 --- a/src/file_vtab.cc +++ b/src/file_vtab.cc @@ -115,8 +115,7 @@ CREATE TABLE lnav_file ( sqlite3_result_error(ctx, "file is too large", -1); } else { auto fd = lf->get_fd(); - auto_mem buf; - buf = (char*) malloc(lf_stat.st_size); + auto buf = auto_mem::malloc(lf_stat.st_size); auto rc = pread(fd, buf, lf_stat.st_size, 0); if (rc == -1) { diff --git a/src/grep_proc.cc b/src/grep_proc.cc index e67ae2ba..cb3d0877 100644 --- a/src/grep_proc.cc +++ b/src/grep_proc.cc @@ -45,6 +45,7 @@ #include "base/string_util.hh" #include "config.h" #include "lnav_util.hh" +#include "scn/scn.h" #include "vis_line.hh" template @@ -270,17 +271,18 @@ grep_proc::cleanup() template void -grep_proc::dispatch_line(char* line) +grep_proc::dispatch_line(const string_fragment& line) { - int start, end, capture_start; + int start, end; - require(line != nullptr); + require(line.is_valid()); - if (sscanf(line, "h%d", this->gp_highest_line.out()) == 1) { - } else if (sscanf(line, "%d", this->gp_last_line.out()) == 1) { + auto sv = line.to_string_view(); + if (scn::scan(sv, "h{}", this->gp_highest_line.lvalue())) { + } else if (scn::scan(sv, "{}", this->gp_last_line.lvalue())) { /* Starting a new line with matches. */ ensure(this->gp_last_line >= 0); - } else if (sscanf(line, "[%d:%d]", &start, &end) == 2) { + } else if (scn::scan(sv, "[{}:{}]", start, end)) { require(start >= 0); require(end >= 0); @@ -288,25 +290,30 @@ grep_proc::dispatch_line(char* line) if (this->gp_sink != nullptr) { this->gp_sink->grep_match(*this, this->gp_last_line, start, end); } - } else if (sscanf(line, "(%d:%d)%n", &start, &end, &capture_start) == 2) { - require(start == -1 || start >= 0); - require(end >= 0); - - /* Pass the captured strings to the sink delegate. */ - if (this->gp_sink != nullptr) { - this->gp_sink->grep_capture( - *this, - this->gp_last_line, - start, - end, - start < 0 ? nullptr : &line[capture_start]); - } } else if (line[0] == '/') { if (this->gp_sink != nullptr) { this->gp_sink->grep_match_end(*this, this->gp_last_line); } } else { - log_error("bad line from child -- %s", line); + auto scan_res = scn::scan(sv, "({}:{})", start, end); + if (scan_res) { + require(start == -1 || start >= 0); + require(end >= 0); + + /* Pass the captured strings to the sink delegate. */ + if (this->gp_sink != nullptr) { + this->gp_sink->grep_capture( + *this, + this->gp_last_line, + start, + end, + start < 0 + ? string_fragment{} + : to_string_fragment(scan_res.range_as_string_view())); + } + } else { + log_error("bad line from child -- %s", line); + } } } @@ -369,13 +376,8 @@ grep_proc::check_poll_set(const std::vector& pollfds) this->gp_pipe_range = li.li_file_range; this->gp_line_buffer.read_range(li.li_file_range) .then([this](auto sbr) { - auto_mem buf; - - buf = (char*) malloc(sbr.length() + 1); sbr.rtrim(is_line_ending); - memcpy(buf, sbr.get_data(), sbr.length()); - buf[sbr.length()] = '\0'; - this->dispatch_line(buf); + this->dispatch_line(sbr.to_string_fragment()); }); loop_count += 1; diff --git a/src/grep_proc.hh b/src/grep_proc.hh index 58010e3f..ccc6df8b 100644 --- a/src/grep_proc.hh +++ b/src/grep_proc.hh @@ -148,9 +148,11 @@ public: LineType line, int start, int end, - char* capture){}; + const string_fragment& capture) + { + } - virtual void grep_match_end(grep_proc& gp, LineType line){}; + virtual void grep_match_end(grep_proc& gp, LineType line) {} }; /** @@ -255,7 +257,7 @@ protected: /** * Dispatch a line received from the child. */ - void dispatch_line(char* line); + void dispatch_line(const string_fragment& line); /** * Free any resources used by the object and make sure the child has been diff --git a/src/line_buffer.cc b/src/line_buffer.cc index b989bc6e..658bfa76 100644 --- a/src/line_buffer.cc +++ b/src/line_buffer.cc @@ -59,6 +59,7 @@ #include "fmtlib/fmt/format.h" #include "hasher.hh" #include "line_buffer.hh" +#include "piper.looper.hh" #include "scn/scn.h" using namespace std::chrono_literals; @@ -136,7 +137,7 @@ private: #define SYNCPOINT_SIZE (1024 * 1024) line_buffer::gz_indexed::gz_indexed() { - if ((this->inbuf = (Bytef*) malloc(Z_BUFSIZE)) == NULL) { + if ((this->inbuf = auto_mem::malloc(Z_BUFSIZE)) == NULL) { throw std::bad_alloc(); } } @@ -192,7 +193,7 @@ line_buffer::gz_indexed::continue_stream() } void -line_buffer::gz_indexed::open(int fd, header_data& hd) +line_buffer::gz_indexed::open(int fd, lnav::gzip::header& hd) { this->close(); this->init_stream(); @@ -239,9 +240,9 @@ line_buffer::gz_indexed::open(int fd, header_data& hd) log_debug("%d: no gzip header data", fd); break; case 1: - hd.hd_mtime.tv_sec = gz_hd.time; - hd.hd_name = std::string((char*) name); - hd.hd_comment = std::string((char*) comment); + hd.h_mtime.tv_sec = gz_hd.time; + hd.h_name = std::string((char*) name); + hd.h_comment = std::string((char*) comment); break; default: log_error("%d: failed to read gzip header data", fd); @@ -409,11 +410,32 @@ line_buffer::set_fd(auto_fd& fd) char gz_id[2 + 1 + 1 + 4]; if (pread(fd, gz_id, sizeof(gz_id), 0) == sizeof(gz_id)) { - if (gz_id[0] == 'L' && gz_id[1] == 0 && gz_id[2] == 'N' - && gz_id[3] == 1 && gz_id[4] == 0) - { + auto piper_hdr_opt = lnav::piper::read_header(fd, gz_id); + + if (piper_hdr_opt) { + static intern_string_t SRC = intern_string::lookup("piper"); + + auto meta_buf = std::move(piper_hdr_opt.value()); + + auto meta_sf = string_fragment::from_bytes(meta_buf.in(), + meta_buf.size()); + auto meta_parse_res + = lnav::piper::header_handlers.parser_for(SRC).of( + meta_sf); + if (meta_parse_res.isErr()) { + log_error("failed to parse piper header: %s", + meta_parse_res.unwrapErr()[0] + .to_attr_line() + .get_string() + .c_str()); + throw error(EINVAL); + } + this->lb_line_metadata = true; - this->lb_file_offset = 8; + this->lb_file_offset + = lnav::piper::HEADER_SIZE + meta_buf.size(); + this->lb_piper_header_size = this->lb_file_offset; + this->lb_header = meta_parse_res.unwrap(); } else if (gz_id[0] == '\037' && gz_id[1] == '\213') { int gzfd = dup(fd); @@ -422,14 +444,19 @@ line_buffer::set_fd(auto_fd& fd) close(gzfd); throw error(errno); } - this->lb_gz_file.writeAccess()->open(gzfd, this->lb_header); + lnav::gzip::header hdr; + + this->lb_gz_file.writeAccess()->open(gzfd, hdr); this->lb_compressed = true; - this->lb_file_time = this->lb_header.hd_mtime.tv_sec; + this->lb_file_time = hdr.h_mtime.tv_sec; if (this->lb_file_time < 0) { this->lb_file_time = 0; } this->lb_compressed_offset = lseek(this->lb_fd, 0, SEEK_CUR); + if (!hdr.empty()) { + this->lb_header = std::move(hdr); + } this->resize_buffer(INITIAL_COMPRESSED_BUFFER_SIZE); } #ifdef HAVE_BZLIB_H @@ -1035,7 +1062,7 @@ line_buffer::load_next_line(file_range prev_line) require(this->lb_fd != -1); if (this->lb_line_metadata && prev_line.fr_offset == 0) { - prev_line.fr_offset = 8; + prev_line.fr_offset = this->lb_piper_header_size; } auto offset = prev_line.next_offset(); diff --git a/src/line_buffer.hh b/src/line_buffer.hh index b4ddca69..0c1ff149 100644 --- a/src/line_buffer.hh +++ b/src/line_buffer.hh @@ -46,8 +46,11 @@ #include "base/auto_mem.hh" #include "base/file_range.hh" #include "base/is_utf8.hh" +#include "base/lnav.gzip.hh" +#include "base/piper.file.hh" #include "base/result.h" #include "log_level.hh" +#include "mapbox/variant.hpp" #include "safe/safe.h" #include "shared_buffer.hh" @@ -81,19 +84,6 @@ public: int e_err; }; - struct header_data { - timeval hd_mtime{}; - auto_buffer hd_extra{auto_buffer::alloc(0)}; - std::string hd_name; - std::string hd_comment; - - bool empty() const - { - return this->hd_mtime.tv_sec == 0 && this->hd_extra.empty() - && this->hd_name.empty() && this->hd_comment.empty(); - } - }; - #define GZ_WINSIZE 32768U /*> gzip's max supported dictionary is 15-bits */ #define GZ_RAW_MODE (-15) /*> Raw inflate data mode */ #define GZ_HEADER_MODE (15 + 32) /*> Automatic zstd or gzip decoding */ @@ -121,7 +111,7 @@ public: void close(); void init_stream(); void continue_stream(); - void open(int fd, header_data& hd); + void open(int fd, lnav::gzip::header& hd); int stream_data(void* buf, size_t size); void seek(off_t offset); @@ -263,7 +253,10 @@ public: size_t get_buffer_size() const { return this->lb_buffer.size(); } - const header_data& get_header_data() const { return this->lb_header; } + using file_header_t + = mapbox::util::variant; + + const file_header_t& get_header_data() const { return this->lb_header; } void enable_cache(); @@ -340,6 +333,7 @@ private: safe_gz_indexed lb_gz_file; /*< File reader for gzipped files. */ bool lb_bz_file{false}; /*< Flag set for bzip2 compressed files. */ bool lb_line_metadata{false}; + size_t lb_piper_header_size{0}; auto_buffer lb_buffer{auto_buffer::alloc(DEFAULT_LINE_BUFFER_SIZE)}; nonstd::optional lb_alt_buffer; @@ -374,7 +368,7 @@ private: nonstd::optional lb_cached_fd; - header_data lb_header; + file_header_t lb_header; }; #endif diff --git a/src/lnav.cc b/src/lnav.cc index 16b7b9ce..2017ccaf 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -95,6 +95,7 @@ #include "CLI/CLI.hpp" #include "dump_internals.hh" #include "environ_vtab.hh" +#include "file_converter_manager.hh" #include "filter_sub_source.hh" #include "fstat_vtab.hh" #include "grep_proc.hh" @@ -1914,6 +1915,8 @@ looper() line_buffer::cleanup_cache(); archive_manager::cleanup_cache(); tailer::cleanup_cache(); + lnav::piper::cleanup(); + file_converter_manager::cleanup(); ran_cleanup = true; } } @@ -2099,11 +2102,7 @@ print_user_msgs(std::vector error_list, return retval; } -enum class verbosity_t : int { - quiet, - standard, - verbose, -}; +verbosity_t verbosity = verbosity_t::standard; int main(int argc, char* argv[]) @@ -2116,7 +2115,6 @@ main(int argc, char* argv[]) bool exec_stdin = false, load_stdin = false; mode_flags_t mode_flags; const char* LANG = getenv("LANG"); - verbosity_t verbosity = verbosity_t::standard; if (LANG == nullptr || strcmp(LANG, "C") == 0) { setenv("LANG", "en_US.UTF-8", 1); @@ -2139,6 +2137,9 @@ main(int argc, char* argv[]) lnav_data.ld_flags |= LNF_SECURE_MODE; } + setenv("LNAV_HOME_DIR", lnav::paths::dotlnav().c_str(), 1); + setenv("LNAV_WORK_DIR", lnav::paths::workdir().c_str(), 1); + lnav_data.ld_exec_context.ec_sql_callback = sql_callback; lnav_data.ld_exec_context.ec_pipe_callback = pipe_callback; @@ -2843,6 +2844,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' } for (auto& file_path : file_args) { + scrub_ansi_string(file_path, nullptr); auto file_path_without_trailer = file_path; auto file_loc = file_location_t{mapbox::util::no_init{}}; auto_mem abspath; @@ -2885,7 +2887,8 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' isc::to().send( [ul](auto& clooper) { clooper.add_request(ul); }); } else if (file_path.find("://") != std::string::npos) { - lnav_data.ld_commands.emplace_back( + lnav_data.ld_commands.insert( + lnav_data.ld_commands.begin(), fmt::format(FMT_STRING(":open {}"), file_path)); } #endif @@ -3037,7 +3040,8 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' retval = EXIT_FAILURE; } - nonstd::optional stdin_pattern; + nonstd::optional stdin_url; + ghc::filesystem::path stdin_dir; if (load_stdin && !isatty(STDIN_FILENO) && !is_dev_null(STDIN_FILENO) && !exec_stdin) { @@ -3063,7 +3067,8 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' STDIN_NAME, auto_fd::dup_of(STDIN_FILENO), auto_fd{}); if (stdin_piper_res.isOk()) { auto stdin_piper = stdin_piper_res.unwrap(); - stdin_pattern = stdin_piper.get_out_pattern(); + stdin_url = stdin_piper.get_url(); + stdin_dir = stdin_piper.get_out_dir(); lnav_data.ld_active_files .fc_file_names[stdin_piper.get_name()] .with_piper(stdin_piper) @@ -3216,6 +3221,8 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' archive_manager::cleanup_cache(); tailer::cleanup_cache(); line_buffer::cleanup_cache(); + lnav::piper::cleanup(); + file_converter_manager::cleanup(); wait_for_pipers(); rescan_files(true); isc::to() @@ -3352,10 +3359,10 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' // When reading from stdin, tell the user where the capture file is // stored so they can look at it later. - if (stdin_pattern && !(lnav_data.ld_flags & LNF_HEADLESS)) { + if (stdin_url && !(lnav_data.ld_flags & LNF_HEADLESS)) { file_size_t stdin_size = 0; - for (const auto& ent : ghc::filesystem::directory_iterator( - stdin_pattern.value().parent_path())) + for (const auto& ent : + ghc::filesystem::directory_iterator(stdin_dir)) { stdin_size += ent.file_size(); } @@ -3371,7 +3378,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' "reopen it by running:\n") .appendf(FMT_STRING(" {} "), lnav_data.ld_program_name) - .append(lnav::roles::file(stdin_pattern.value())))); + .append(lnav::roles::file(stdin_url.value())))); } } diff --git a/src/lnav.hh b/src/lnav.hh index 68e9aee2..c4b7ae97 100644 --- a/src/lnav.hh +++ b/src/lnav.hh @@ -268,7 +268,14 @@ class main_looper public: }; +enum class verbosity_t : int { + quiet, + standard, + verbose, +}; + extern struct lnav_data_t lnav_data; +extern verbosity_t verbosity; extern readline_context::command_map_t lnav_commands; extern const int ZOOM_LEVELS[]; diff --git a/src/lnav.indexing.cc b/src/lnav.indexing.cc index d2a68d73..52473995 100644 --- a/src/lnav.indexing.cc +++ b/src/lnav.indexing.cc @@ -167,8 +167,11 @@ public: void scanned_file(const std::shared_ptr& lf) override { - if (!lnav_data.ld_files_to_front.empty() - && lnav_data.ld_files_to_front.front().first == lf->get_filename()) + const auto& ftf = lnav_data.ld_files_to_front; + + if (!ftf.empty() + && (ftf.front().first == lf->get_filename() + || ftf.front().first == lf->get_open_options().loo_filename)) { this->front_file = lf; this->front_top = lnav_data.ld_files_to_front.front().second; diff --git a/src/lnav.management_cli.cc b/src/lnav.management_cli.cc index baf4cfeb..ef8f2447 100644 --- a/src/lnav.management_cli.cc +++ b/src/lnav.management_cli.cc @@ -29,9 +29,14 @@ #include "lnav.management_cli.hh" +#include "base/fs_util.hh" +#include "base/humanize.hh" +#include "base/humanize.time.hh" #include "base/itertools.hh" +#include "base/paths.hh" #include "base/result.h" #include "base/string_util.hh" +#include "fmt/chrono.h" #include "fmt/format.h" #include "itertools.similar.hh" #include "lnav.hh" @@ -39,6 +44,7 @@ #include "log_format.hh" #include "log_format_ext.hh" #include "mapbox/variant.hpp" +#include "piper.looper.hh" #include "regex101.import.hh" #include "session_data.hh" @@ -61,7 +67,9 @@ symbol_reducer(const std::string& elem, attr_line_t& accum) inline attr_line_t& subcmd_reducer(const CLI::App* app, attr_line_t& accum) { - return accum.append("\n \u2022 ") + return accum.append("\n ") + .append("\u2022"_list_glyph) + .append(" ") .append(lnav::roles::keyword(app->get_name())) .append(": ") .append(app->get_description()); @@ -673,6 +681,283 @@ struct subcmd_format_t { } }; +struct subcmd_piper_t { + using action_t = std::function; + + CLI::App* sp_app{nullptr}; + action_t sp_action; + + subcmd_piper_t& set_action(action_t act) + { + if (!this->sp_action) { + this->sp_action = std::move(act); + } + return *this; + } + + static perform_result_t default_action(const subcmd_piper_t& sp) + { + auto um = console::user_message::error( + "expecting an operation related to piper storage"); + um.with_help( + sp.sp_app->get_subcommands({}) + | lnav::itertools::fold( + subcmd_reducer, attr_line_t{"the available operations are:"})); + + return {um}; + } + + static perform_result_t list_action(const subcmd_piper_t&) + { + static const intern_string_t SRC = intern_string::lookup("piper"); + static const auto DOT_HEADER = ghc::filesystem::path(".header"); + + struct item { + lnav::piper::header i_header; + std::string i_url; + file_size_t i_total_size{0}; + }; + + file_size_t grand_total{0}; + std::vector items; + std::error_code ec; + + for (const auto& instance_dir : ghc::filesystem::directory_iterator( + lnav::piper::storage_path(), ec)) + { + if (!instance_dir.is_directory()) { + log_warning("piper directory entry is not a directory: %s", + instance_dir.path().c_str()); + continue; + } + + nonstd::optional hdr_opt; + auto url = fmt::format(FMT_STRING("piper://{}"), + instance_dir.path().filename().string()); + file_size_t total_size{0}; + auto hdr_path = instance_dir / DOT_HEADER; + if (ghc::filesystem::exists(hdr_path)) { + auto hdr_read_res = lnav::filesystem::read_file(hdr_path); + if (hdr_read_res.isOk()) { + auto parse_res + = lnav::piper::header_handlers.parser_for(SRC).of( + hdr_read_res.unwrap()); + if (parse_res.isOk()) { + hdr_opt = parse_res.unwrap(); + } else { + log_error("failed to parse header: %s -- %s", + hdr_path.c_str(), + parse_res.unwrapErr()[0] + .to_attr_line() + .get_string() + .c_str()); + } + } else { + log_error("failed to read header file: %s -- %s", + hdr_path.c_str(), + hdr_read_res.unwrapErr().c_str()); + } + } + + for (const auto& entry : + ghc::filesystem::directory_iterator(instance_dir.path())) + { + if (entry.path().filename() == DOT_HEADER) { + continue; + } + + total_size += entry.file_size(); + char buffer[lnav::piper::HEADER_SIZE]; + + auto entry_open_res + = lnav::filesystem::open_file(entry.path(), O_RDONLY); + if (entry_open_res.isErr()) { + log_warning("unable to open piper file: %s -- %s", + entry.path().c_str(), + entry_open_res.unwrapErr().c_str()); + continue; + } + + auto entry_fd = entry_open_res.unwrap(); + if (read(entry_fd, buffer, sizeof(buffer)) != sizeof(buffer)) { + log_warning("piper file is too small: %s", + entry.path().c_str()); + continue; + } + auto hdr_bits_opt = lnav::piper::read_header(entry_fd, buffer); + if (!hdr_bits_opt) { + log_warning("could not read piper header: %s", + entry.path().c_str()); + continue; + } + + auto hdr_buf = std::move(hdr_bits_opt.value()); + + total_size -= hdr_buf.size(); + auto hdr_sf + = string_fragment::from_bytes(hdr_buf.in(), hdr_buf.size()); + auto hdr_parse_res + = lnav::piper::header_handlers.parser_for(SRC).of(hdr_sf); + if (hdr_parse_res.isErr()) { + log_error("failed to parse piper header: %s", + hdr_parse_res.unwrapErr()[0] + .to_attr_line() + .get_string() + .c_str()); + continue; + } + + auto hdr = hdr_parse_res.unwrap(); + + if (!hdr_opt || hdr < hdr_opt.value()) { + hdr_opt = hdr; + } + } + + if (hdr_opt) { + items.emplace_back(item{hdr_opt.value(), url, total_size}); + } + + grand_total += total_size; + } + + if (ec && ec.value() != ENOENT) { + auto um = lnav::console::user_message::error( + attr_line_t("unable to access piper directory: ") + .append(lnav::roles::file( + lnav::piper::storage_path().string()))) + .with_reason(ec.message()); + return {um}; + } + + if (items.empty()) { + if (verbosity != verbosity_t::quiet) { + auto um + = lnav::console::user_message::info( + attr_line_t("no piper captures were found in:\n\t") + .append(lnav::roles::file( + lnav::piper::storage_path().string()))) + .with_help( + attr_line_t("You can create a capture by " + "piping data into ") + .append(lnav::roles::file("lnav")) + .append(" or using the ") + .append_quoted(lnav::roles::symbol(":sh")) + .append(" command")); + return {um}; + } + + return {}; + } + + auto txt + = items + | lnav::itertools::sort_with([](const item& lhs, const item& rhs) { + if (lhs.i_header < rhs.i_header) { + return true; + } + + if (rhs.i_header < lhs.i_header) { + return false; + } + + return lhs.i_url < rhs.i_url; + }) + | lnav::itertools::map([](const item& it) { + auto ago = humanize::time::point::from_tv(it.i_header.h_ctime) + .as_time_ago(); + auto retval = attr_line_t() + .append(lnav::roles::list_glyph( + fmt::format(FMT_STRING("{:>18}"), ago))) + .append(" ") + .append(lnav::roles::file(it.i_url)) + .append(" ") + .append(lnav::roles::number(fmt::format( + FMT_STRING("{:>8}"), + humanize::file_size( + it.i_total_size, + humanize::alignment::columnar)))) + .append(" ") + .append_quoted(lnav::roles::comment( + it.i_header.h_name)) + .append("\n"); + if (verbosity == verbosity_t::verbose) { + auto env_al + = it.i_header.h_env + | lnav::itertools::map([](const auto& pair) { + return attr_line_t() + .append(lnav::roles::identifier(pair.first)) + .append("=") + .append(pair.second) + .append("\n"); + }) + | lnav::itertools::fold( + [](const auto& elem, auto& accum) { + if (!accum.empty()) { + accum.append(28, ' '); + } + return accum.append(elem); + }, + attr_line_t()); + + retval.append(23, ' ') + .append("cwd: ") + .append(lnav::roles::file(it.i_header.h_cwd)) + .append("\n") + .append(23, ' ') + .append("env: ") + .append(env_al); + } + return retval; + }) + | lnav::itertools::fold( + [](const auto& elem, auto& accum) { + return accum.append(elem); + }, + attr_line_t{}); + txt.rtrim(); + + perform_result_t retval; + if (verbosity != verbosity_t::quiet) { + auto extra_um + = lnav::console::user_message::info( + attr_line_t( + "the following piper captures were found in:\n\t") + .append(lnav::roles::file( + lnav::piper::storage_path().string()))) + .with_note( + attr_line_t("The captures currently consume ") + .append(lnav::roles::number(humanize::file_size( + grand_total, humanize::alignment::none))) + .append(" of disk space. File sizes include " + "associated metadata.")) + .with_help( + "You can reopen a capture by passing the piper URL " + "to lnav"); + retval.emplace_back(extra_um); + } + retval.emplace_back(lnav::console::user_message::raw(txt)); + + return retval; + } + + static perform_result_t clean_action(const subcmd_piper_t&) + { + std::error_code ec; + + ghc::filesystem::remove_all(lnav::piper::storage_path(), ec); + if (ec) { + return { + lnav::console::user_message::error( + "unable to remove piper storage directory") + .with_reason(ec.message()), + }; + } + + return {}; + } +}; + struct subcmd_regex101_t { using action_t = std::function; @@ -762,8 +1047,11 @@ struct subcmd_regex101_t { } }; -using operations_v = mapbox::util:: - variant; +using operations_v = mapbox::util::variant; class operations { public: @@ -783,6 +1071,7 @@ describe_cli(CLI::App& app, int argc, char* argv[]) subcmd_config_t config_args; subcmd_format_t format_args; + subcmd_piper_t piper_args; subcmd_regex101_t regex101_args; { @@ -910,6 +1199,25 @@ describe_cli(CLI::App& app, int argc, char* argv[]) } } + { + auto* subcmd_piper + = app.add_subcommand("piper", "perform operations on piper storage") + ->callback([&]() { + piper_args.set_action(subcmd_piper_t::default_action); + retval->o_ops = piper_args; + }); + piper_args.sp_app = subcmd_piper; + + subcmd_piper + ->add_subcommand("list", "print the available piper captures") + ->callback( + [&]() { piper_args.set_action(subcmd_piper_t::list_action); }); + + subcmd_piper->add_subcommand("clean", "remove all piper captures") + ->callback( + [&]() { piper_args.set_action(subcmd_piper_t::clean_action); }); + } + { auto* subcmd_regex101 = app.add_subcommand("regex101", @@ -978,6 +1286,7 @@ perform(std::shared_ptr opts) }, [](const subcmd_config_t& sc) { return sc.sc_action(sc); }, [](const subcmd_format_t& sf) { return sf.sf_action(sf); }, + [](const subcmd_piper_t& sp) { return sp.sp_action(sp); }, [](const subcmd_regex101_t& sr) { return sr.sr_action(sr); }); } diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index aa23ff89..4153921d 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -2602,6 +2602,8 @@ com_open(exec_context& ec, std::string cmdline, std::vector& args) retval = "info: watching -- " + fn; } else if (is_glob(fn.c_str())) { fc.fc_file_names.emplace(fn, loo); + files_to_front.emplace_back( + loo.loo_filename.empty() ? fn : loo.loo_filename, file_loc); retval = "info: watching -- " + fn; } else if (stat(fn.c_str(), &st) == -1) { if (fn.find(':') != std::string::npos) { @@ -4204,6 +4206,8 @@ com_sh(exec_context& ec, std::string cmdline, std::vector& args) return ec.make_error("{} -- unavailable in secure mode", args[0]); } + static size_t EXEC_COUNT = 0; + if (!ec.ec_dry_run) { auto carg = trim(cmdline.substr(args[0].size())); @@ -4261,7 +4265,8 @@ com_sh(exec_context& ec, std::string cmdline, std::vector& args) } auto display_name = ec.get_provenance() - .value_or(exec_context::file_open{carg}) + .value_or(exec_context::file_open{fmt::format( + FMT_STRING("[{}] {}"), EXEC_COUNT++, carg)}) .fo_name; auto create_piper_res = lnav::piper::create_looper(display_name, diff --git a/src/lnav_config.cc b/src/lnav_config.cc index 307e2937..364fea8b 100644 --- a/src/lnav_config.cc +++ b/src/lnav_config.cc @@ -1074,6 +1074,14 @@ static const struct json_path_container piper_handlers = { .with_min_value(2) .with_description("The number of rotated files to keep") .for_field(&_lnav_config::lc_piper, &lnav::piper::config::c_rotations), + yajlpp::property_handler("ttl") + .with_synopsis("") + .with_description( + "The time-to-live for captured data, expressed as a duration " + "(e.g. '3d' for three days)") + .with_example("3d") + .with_example("12h") + .for_field(&_lnav_config::lc_piper, &lnav::piper::config::c_ttl), }; static const struct json_path_container file_vtab_handlers = { diff --git a/src/logfile.cc b/src/logfile.cc index a21d8dbd..39aee1fb 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -51,20 +51,20 @@ #include "log.watch.hh" #include "log_format.hh" #include "logfile.cfg.hh" +#include "piper.looper.hh" #include "yajlpp/yajlpp_def.hh" static auto intern_lifetime = intern_string::get_table_lifetime(); static const size_t INDEX_RESERVE_INCREMENT = 1024; -static const typed_json_path_container - file_header_handlers = { - yajlpp::property_handler("name").for_field( - &line_buffer::header_data::hd_name), +static const typed_json_path_container file_header_handlers + = { + yajlpp::property_handler("name").for_field(&lnav::gzip::header::h_name), yajlpp::property_handler("mtime").for_field( - &line_buffer::header_data::hd_mtime), + &lnav::gzip::header::h_mtime), yajlpp::property_handler("comment").for_field( - &line_buffer::header_data::hd_comment), + &lnav::gzip::header::h_comment), }; Result, std::string> @@ -134,10 +134,25 @@ logfile::open(std::string filename, const logfile_open_options& loo, auto_fd fd) lf->lf_indexing = lf->lf_options.loo_is_visible; const auto& hdr = lf->lf_line_buffer.get_header_data(); - if (!hdr.empty()) { - lf->lf_embedded_metadata["net.zlib.gzip.header"] - = {text_format_t::TF_JSON, file_header_handlers.to_string(hdr)}; - } + log_info("%s: has header %d", lf->lf_filename.c_str(), hdr.valid()); + hdr.match( + [&lf](const lnav::gzip::header& gzhdr) { + if (!gzhdr.empty()) { + lf->lf_embedded_metadata["net.zlib.gzip.header"] = { + text_format_t::TF_JSON, + file_header_handlers.to_string(gzhdr), + }; + } + }, + [&lf](const lnav::piper::header& phdr) { + lf->lf_embedded_metadata["org.lnav.piper.header"] = { + text_format_t::TF_JSON, + lnav::piper::header_handlers.to_string(phdr), + }; + log_debug("setting file name: %s", phdr.h_name.c_str()); + lf->set_filename(phdr.h_name); + lf->lf_valid_filename = false; + }); ensure(lf->invariant()); diff --git a/src/piper.looper.cc b/src/piper.looper.cc index 22f8b849..f8634382 100644 --- a/src/piper.looper.cc +++ b/src/piper.looper.cc @@ -32,6 +32,7 @@ #include "piper.looper.hh" +#include #include #include "base/fs_util.hh" @@ -41,6 +42,7 @@ #include "config.h" #include "hasher.hh" #include "line_buffer.hh" +#include "pcrepp/pcre2pp.hh" #include "piper.looper.cfg.hh" using namespace std::chrono_literals; @@ -62,19 +64,60 @@ write_timestamp(int fd, log_level_t level, off_t woff) return pwrite(fd, time_str, fmt_res.size, woff); } +extern char** environ; + namespace lnav { namespace piper { +const json_path_container header_env_handlers = { + yajlpp::pattern_property_handler("(?.*)") + .with_synopsis("") + .for_field(&lnav::piper::header::h_env), +}; + +const typed_json_path_container header_handlers = { + yajlpp::property_handler("name").for_field(&lnav::piper::header::h_name), + yajlpp::property_handler("ctime").for_field(&lnav::piper::header::h_ctime), + yajlpp::property_handler("cwd").for_field(&lnav::piper::header::h_cwd), + yajlpp::property_handler("env").with_children(header_env_handlers), +}; + +static std::map +environ_to_map() +{ + static const auto SENSITIVE_VARS + = lnav::pcre2pp::code::from_const(R"((?i)token|pass)"); + + std::map retval; + + for (size_t lpc = 0; environ[lpc]; lpc++) { + auto full_sf = string_fragment::from_c_str(environ[lpc]); + auto pair_opt = full_sf.split_pair(string_fragment::tag1{'='}); + + if (!pair_opt) { + continue; + } + if (SENSITIVE_VARS.find_in(pair_opt->first).ignore_error()) { + retval[pair_opt->first.to_string()] = "******"; + } else { + retval[pair_opt->first.to_string()] = pair_opt->second.to_string(); + } + } + + return retval; +} + looper::looper(std::string name, auto_fd stdout_fd, auto_fd stderr_fd) - : l_name(std::move(name)), l_stdout(std::move(stdout_fd)), + : l_name(std::move(name)), l_cwd(ghc::filesystem::current_path().string()), + l_env(environ_to_map()), l_stdout(std::move(stdout_fd)), l_stderr(std::move(stderr_fd)) { size_t count = 0; do { this->l_out_dir - = lnav::paths::workdir() + = storage_path() / fmt::format( - FMT_STRING("piper-{}-{}"), + FMT_STRING("p-{}-{:03}"), hasher().update(getmstime()).update(l_name).to_string(), count); count += 1; @@ -237,6 +280,22 @@ looper::loop() break; } + auto hdr_path = this->l_out_dir / ".header"; + auto hdr = header{ + current_timeval(), + this->l_name, + this->l_cwd, + this->l_env, + }; + auto write_hdr_res = lnav::filesystem::write_file( + hdr_path, header_handlers.to_string(hdr)); + if (write_hdr_res.isErr()) { + log_error("unable to write header file: %s -- %s", + hdr_path.c_str(), + write_hdr_res.unwrapErr().c_str()); + break; + } + outfd = create_res.unwrap(); auto header_avail = cap.lb.get_available(); auto read_res = cap.lb.read_range(header_avail); @@ -299,11 +358,41 @@ looper::loop() outfd = create_res.unwrap(); rotate_count += 1; - static const char lnav_header[] - = {'L', 0, 'N', 1, 0, 0, 0, 0}; - auto prc - = write(outfd.get(), lnav_header, sizeof(lnav_header)); - woff = prc; + auto hdr = header{ + current_timeval(), + this->l_name, + this->l_cwd, + this->l_env, + }; + + woff = 0; + auto hdr_str = header_handlers.to_string(hdr); + uint32_t meta_size = htonl(hdr_str.length()); + auto prc = write( + outfd.get(), HEADER_MAGIC, sizeof(HEADER_MAGIC)); + if (prc < sizeof(HEADER_MAGIC)) { + log_error("unable to write file header: %s -- %s", + this->l_name.c_str(), + strerror(errno)); + break; + } + woff += prc; + prc = write(outfd.get(), &meta_size, sizeof(meta_size)); + if (prc < sizeof(meta_size)) { + log_error("unable to write file header: %s -- %s", + this->l_name.c_str(), + strerror(errno)); + break; + } + woff += prc; + prc = write(outfd.get(), hdr_str.c_str(), hdr_str.size()); + if (prc < hdr_str.size()) { + log_error("unable to write file header: %s -- %s", + this->l_name.c_str(), + strerror(errno)); + break; + } + woff += prc; } ssize_t wrc; @@ -352,5 +441,47 @@ create_looper(std::string name, auto_fd stdout_fd, auto_fd stderr_fd) name, std::move(stdout_fd), std::move(stderr_fd)))); } +void +cleanup() +{ + (void) std::async(std::launch::async, []() { + const auto& cfg = injector::get(); + auto now = std::chrono::system_clock::now(); + auto cache_path = storage_path(); + std::vector to_remove; + + for (const auto& cache_subdir : + ghc::filesystem::directory_iterator(cache_path)) + { + auto mtime = ghc::filesystem::last_write_time(cache_subdir.path()); + auto exp_time = mtime + cfg.c_ttl; + if (now < exp_time) { + continue; + } + + bool is_recent = false; + + for (const auto& entry : + ghc::filesystem::directory_iterator(cache_subdir)) + { + auto mtime = ghc::filesystem::last_write_time(entry.path()); + auto exp_time = mtime + cfg.c_ttl; + if (now < exp_time) { + is_recent = true; + break; + } + } + if (!is_recent) { + to_remove.emplace_back(cache_subdir); + } + } + + for (auto& entry : to_remove) { + log_debug("removing piper directory: %s", entry.c_str()); + ghc::filesystem::remove_all(entry); + } + }); +} + } // namespace piper } // namespace lnav diff --git a/src/piper.looper.cfg.hh b/src/piper.looper.cfg.hh index 365bc9ce..b9740281 100644 --- a/src/piper.looper.cfg.hh +++ b/src/piper.looper.cfg.hh @@ -38,6 +38,7 @@ namespace piper { struct config { uint64_t c_max_size{10ULL * 1024ULL * 1024ULL}; uint32_t c_rotations{4}; + std::chrono::seconds c_ttl{std::chrono::hours(48)}; }; } // namespace piper diff --git a/src/piper.looper.hh b/src/piper.looper.hh index 7446f1d3..e0766f7b 100644 --- a/src/piper.looper.hh +++ b/src/piper.looper.hh @@ -35,8 +35,10 @@ #include #include "base/auto_fd.hh" +#include "base/piper.file.hh" #include "base/result.h" #include "ghc/filesystem.hpp" +#include "yajlpp/yajlpp_def.hh" namespace lnav { namespace piper { @@ -61,6 +63,12 @@ public: return this->l_out_dir / "out.*"; } + std::string get_url() const + { + return fmt::format(FMT_STRING("piper://{}"), + this->l_out_dir.filename().string()); + } + bool is_finished() const { return this->l_future.wait_for(std::chrono::seconds(0)) @@ -72,6 +80,8 @@ private: std::atomic l_looping{true}; const std::string l_name; + const std::string l_cwd; + const std::map l_env; ghc::filesystem::path l_out_dir; auto_fd l_stdout; auto_fd l_stderr; @@ -98,6 +108,8 @@ public: return this->h_looper->get_out_pattern(); } + std::string get_url() const { return this->h_looper->get_url(); } + bool is_finished() const { return this->h_looper->is_finished(); } bool operator==(const handle& other) const @@ -109,12 +121,16 @@ private: std::shared_ptr h_looper; }; +extern const typed_json_path_container header_handlers; + using running_handle = handle; Result, std::string> create_looper(std::string name, auto_fd stdout_fd, auto_fd stderr_fd); +void cleanup(); + } // namespace piper } // namespace lnav diff --git a/src/pretty_printer.cc b/src/pretty_printer.cc index 2ff3e11f..625f26f3 100644 --- a/src/pretty_printer.cc +++ b/src/pretty_printer.cc @@ -188,7 +188,7 @@ pretty_printer::write_element(const pretty_printer::element& el) } ssize_t start_size = this->pp_stream.tellp(); if (el.e_token == DT_QUOTED_STRING) { - auto_mem unquoted_str((char*) malloc(el.e_capture.length() + 1)); + auto unquoted_str = auto_mem::malloc(el.e_capture.length() + 1); const char* start = this->pp_scanner->to_string_fragment(el.e_capture).data(); auto unq_len = unquote(unquoted_str.in(), start, el.e_capture.length()); diff --git a/src/root-config.json b/src/root-config.json index 8bf8194c..410fe404 100644 --- a/src/root-config.json +++ b/src/root-config.json @@ -28,7 +28,8 @@ }, "piper": { "max-size": 10485760, - "rotations": 4 + "rotations": 4, + "ttl": "2d" }, "clipboard": { "impls": { @@ -82,6 +83,9 @@ "url-scheme": { "docker": { "handler": "docker-url-handler" + }, + "piper": { + "handler": "piper-url-handler" } } } diff --git a/src/scripts/piper-url-handler.lnav b/src/scripts/piper-url-handler.lnav new file mode 100755 index 00000000..f1867673 --- /dev/null +++ b/src/scripts/piper-url-handler.lnav @@ -0,0 +1,8 @@ +# +# @synopsis: piper-url-handler +# @description: Internal script to handle opening piper URLs +# + +;SELECT jget(url, '/host') AS uhost FROM (SELECT parse_url($1) AS url) + +:open ${LNAV_WORK_DIR}/piper/$uhost/out.* diff --git a/src/scripts/scripts.am b/src/scripts/scripts.am index 3505e3f8..5af898a1 100644 --- a/src/scripts/scripts.am +++ b/src/scripts/scripts.am @@ -4,6 +4,7 @@ BUILTIN_LNAVSCRIPTS = \ $(srcdir)/scripts/docker-url-handler.lnav \ $(srcdir)/scripts/lnav-pop-view.lnav \ $(srcdir)/scripts/partition-by-boot.lnav \ + $(srcdir)/scripts/piper-url-handler.lnav \ $(srcdir)/scripts/rename-stdin.lnav \ $(srcdir)/scripts/search-for.lnav \ $() diff --git a/src/string-extension-functions.cc b/src/string-extension-functions.cc index 23606684..cf8e03dd 100644 --- a/src/string-extension-functions.cc +++ b/src/string-extension-functions.cc @@ -365,8 +365,8 @@ sparkline_final(sqlite3_context* context) return; } - auto* retval = (char*) malloc(sc->sc_values.size() * 3 + 1); - auto* start = retval; + auto retval = auto_mem::malloc(sc->sc_values.size() * 3 + 1); + auto* start = retval.in(); for (const auto& value : sc->sc_values) { auto bar = humanize::sparkline(value, sc->sc_max_value); @@ -376,7 +376,7 @@ sparkline_final(sqlite3_context* context) } *start = '\0'; - sqlite3_result_text(context, retval, -1, free); + to_sqlite(context, std::move(retval)); sc->~sparkline_context(); } @@ -720,13 +720,8 @@ sql_parse_url(std::string url) while (true) { auto split_res = remaining.split_when(string_fragment::tag1{'&'}); - - if (!split_res) { - break; - } - auto_mem kv_pair(curl_free); - auto kv_pair_encoded = split_res->first; + auto kv_pair_encoded = split_res.first; int out_len = 0; kv_pair = curl_easy_unescape(CURL_HANDLE, @@ -748,20 +743,20 @@ sql_parse_url(std::string url) query_map.gen(val); } } else { - auto val_str = split_res->first.to_string(); + auto val_str = split_res.first.to_string(); if (seen_keys.count(val_str) == 0) { seen_keys.insert(val_str); - query_map.gen(split_res->first); + query_map.gen(split_res.first); query_map.gen(); } } - if (split_res->second.empty()) { + if (split_res.second.empty()) { break; } - remaining = split_res->second; + remaining = split_res.second; } } else { root.gen("query"); diff --git a/src/strong_int.hh b/src/strong_int.hh index af18da99..36c05038 100644 --- a/src/strong_int.hh +++ b/src/strong_int.hh @@ -45,59 +45,55 @@ template class strong_int { public: explicit constexpr strong_int(T v = 0) noexcept : value(v){}; - operator const T&() const - { - return this->value; - }; + operator const T&() const { return this->value; } strong_int operator+(const strong_int& rhs) const { return strong_int(this->value + rhs.value); - }; + } strong_int operator-(const strong_int& rhs) const { return strong_int(this->value - rhs.value); - }; + } strong_int operator/(const strong_int& rhs) const { return strong_int(this->value / rhs.value); - }; + } bool operator<(const strong_int& rhs) const { return this->value < rhs.value; - }; + } strong_int& operator+=(const strong_int& rhs) { this->value += rhs.value; return *this; - }; + } strong_int& operator-=(const strong_int& rhs) { this->value -= rhs.value; return *this; - }; + } strong_int& operator-() { this->value = -this->value; return *this; - }; + } strong_int& operator++() { this->value++; return *this; - }; + } strong_int& operator--() { this->value--; return *this; - }; + } bool operator==(const strong_int& rhs) const { return this->value == rhs.value; - }; - T* out() - { - return &this->value; - }; + } + T* out() { return &this->value; } + + T& lvalue() { return this->value; } private: T value; diff --git a/src/tailer/tailer.looper.cc b/src/tailer/tailer.looper.cc index 82a9fdc2..dcf1e0c7 100644 --- a/src/tailer/tailer.looper.cc +++ b/src/tailer/tailer.looper.cc @@ -787,9 +787,7 @@ tailer::looper::host_tailer::loop_body() } constexpr int64_t BUFFER_SIZE = 4 * 1024 * 1024; - auto_mem buffer; - - buffer = (unsigned char*) malloc(BUFFER_SIZE); + auto buffer = auto_mem::malloc(BUFFER_SIZE); auto remaining = pob.pob_length; auto remaining_offset = pob.pob_offset; tailer::hash_frag thf; diff --git a/src/xterm_mouse.cc b/src/xterm_mouse.cc index f1f330a2..217e306d 100644 --- a/src/xterm_mouse.cc +++ b/src/xterm_mouse.cc @@ -42,7 +42,6 @@ void xterm_mouse::handle_mouse() { bool release = false; - int ch; size_t index = 0; int bstate, x, y; char buffer[64]; @@ -52,7 +51,7 @@ xterm_mouse::handle_mouse() if (index >= sizeof(buffer) - 1) { break; } - ch = getch(); + auto ch = getch(); switch (ch) { case 'm': release = true; diff --git a/test/Makefile.am b/test/Makefile.am index 7b310119..4be9d384 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -512,6 +512,7 @@ distclean-local: $(RM_V)rm -rf remote remote-tmp not:a:remote:dir $(RM_V)rm -rf sessions $(RM_V)rm -rf tmp + $(RM_V)rm -rf piper-tmp $(RM_V)rm -rf rotmp $(RM_V)rm -rf meta-sessions $(RM_V)rm -rf mgmt-config diff --git a/test/drive_grep_proc.cc b/test/drive_grep_proc.cc index 4118f43e..d90cde47 100644 --- a/test/drive_grep_proc.cc +++ b/test/drive_grep_proc.cc @@ -89,7 +89,7 @@ public: void grep_match(grep_proc& gp, vis_line_t line, int start, - int end) + int end) override { printf("%d:%d:%d\n", (int) line, start, end); } @@ -98,12 +98,21 @@ public: vis_line_t line, int start, int end, - char* capture) + const string_fragment& capture) override { - fprintf(stderr, "%d(%d:%d)%s\n", (int) line, start, end, capture); + fprintf(stderr, + "%d(%d:%d)%.*s\n", + (int) line, + start, + end, + capture.length(), + capture.data()); } - void grep_end(grep_proc& gp) { this->ms_finished = true; } + void grep_end(grep_proc& gp) override + { + this->ms_finished = true; + } bool ms_finished; }; diff --git a/test/expected/expected.am b/test/expected/expected.am index b45ad752..e1bf64ba 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -2,10 +2,18 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.err \ $(srcdir)/%reldir%/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out \ + $(srcdir)/%reldir%/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.err \ + $(srcdir)/%reldir%/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.out \ $(srcdir)/%reldir%/test_cli.sh_17a68b798354f9a6cdfab372006caeb74038d15c.err \ $(srcdir)/%reldir%/test_cli.sh_17a68b798354f9a6cdfab372006caeb74038d15c.out \ + $(srcdir)/%reldir%/test_cli.sh_3114508cf42fb2608ef77f4bc294a84885c97a79.err \ + $(srcdir)/%reldir%/test_cli.sh_3114508cf42fb2608ef77f4bc294a84885c97a79.out \ + $(srcdir)/%reldir%/test_cli.sh_4327033cfae0d4c170a38a3c4a570520bfabb493.err \ + $(srcdir)/%reldir%/test_cli.sh_4327033cfae0d4c170a38a3c4a570520bfabb493.out \ $(srcdir)/%reldir%/test_cli.sh_5524542b1a6954ff9741155101497270a2f0c557.err \ $(srcdir)/%reldir%/test_cli.sh_5524542b1a6954ff9741155101497270a2f0c557.out \ + $(srcdir)/%reldir%/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.err \ + $(srcdir)/%reldir%/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.out \ $(srcdir)/%reldir%/test_cli.sh_97e19b9ff3775d84074455a2e8993a0611b1c269.err \ $(srcdir)/%reldir%/test_cli.sh_97e19b9ff3775d84074455a2e8993a0611b1c269.out \ $(srcdir)/%reldir%/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.err \ @@ -146,8 +154,6 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_cmds.sh_95beaabe41d72cf4c6810e79c623da759ac1c71b.out \ $(srcdir)/%reldir%/test_cmds.sh_968dac54dc80d91a5da2322890c6c26dfa0d8462.err \ $(srcdir)/%reldir%/test_cmds.sh_968dac54dc80d91a5da2322890c6c26dfa0d8462.out \ - $(srcdir)/%reldir%/test_cmds.sh_9dfc433f14b811afb3ec4daef0f33e4c9b14a6d7.err \ - $(srcdir)/%reldir%/test_cmds.sh_9dfc433f14b811afb3ec4daef0f33e4c9b14a6d7.out \ $(srcdir)/%reldir%/test_cmds.sh_a00943ef715598c7554b85de8502454e41bb9e28.err \ $(srcdir)/%reldir%/test_cmds.sh_a00943ef715598c7554b85de8502454e41bb9e28.out \ $(srcdir)/%reldir%/test_cmds.sh_a1123427c31c022433d66d05ee5d5e1c8ab415e4.err \ diff --git a/test/expected/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out b/test/expected/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out index 687b3c30..f28c6663 100644 --- a/test/expected/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out +++ b/test/expected/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out @@ -6,7 +6,8 @@ }, "piper": { "max-size": 10485760, - "rotations": 4 + "rotations": 4, + "ttl": "2d" }, "file-vtab": { "max-content-size": 33554432 @@ -106,6 +107,9 @@ }, "hw": { "handler": "hw-url-handler" + }, + "piper": { + "handler": "piper-url-handler" } } }, diff --git a/test/expected/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.err b/test/expected/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.err new file mode 100644 index 00000000..111ae4e1 --- /dev/null +++ b/test/expected/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.err @@ -0,0 +1,4 @@ +ⓘ info: the following piper captures were found in: + piper-tmp/lnav-user-501-work/piper + = note: The captures currently consume 31B of disk space. File sizes include associated metadata. + = help: You can reopen a capture by passing the piper URL to lnav diff --git a/test/expected/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.out b/test/expected/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.out new file mode 100644 index 00000000..98ea9528 --- /dev/null +++ b/test/expected/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.out @@ -0,0 +1 @@ + just now piper://p-e25e2eb68547f31e42da0818b4d0084f-000  31.0 B “[0] echo hi” diff --git a/test/expected/test_cli.sh_3114508cf42fb2608ef77f4bc294a84885c97a79.err b/test/expected/test_cli.sh_3114508cf42fb2608ef77f4bc294a84885c97a79.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_cli.sh_3114508cf42fb2608ef77f4bc294a84885c97a79.out b/test/expected/test_cli.sh_3114508cf42fb2608ef77f4bc294a84885c97a79.out new file mode 100644 index 00000000..45b983be --- /dev/null +++ b/test/expected/test_cli.sh_3114508cf42fb2608ef77f4bc294a84885c97a79.out @@ -0,0 +1 @@ +hi diff --git a/test/expected/test_cli.sh_4327033cfae0d4c170a38a3c4a570520bfabb493.err b/test/expected/test_cli.sh_4327033cfae0d4c170a38a3c4a570520bfabb493.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_cli.sh_4327033cfae0d4c170a38a3c4a570520bfabb493.out b/test/expected/test_cli.sh_4327033cfae0d4c170a38a3c4a570520bfabb493.out new file mode 100644 index 00000000..45b983be --- /dev/null +++ b/test/expected/test_cli.sh_4327033cfae0d4c170a38a3c4a570520bfabb493.out @@ -0,0 +1 @@ +hi diff --git a/test/expected/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.err b/test/expected/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.out b/test/expected/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.out new file mode 100644 index 00000000..f6223f24 --- /dev/null +++ b/test/expected/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.out @@ -0,0 +1,9 @@ +[ + { + "filepath": "[0] echo hi", + "descriptor": "org.lnav.piper.header", + "mimetype": "application/json", + "ctime": "2013-06-06T19:13:20.000", + "cwd": "{test_dir}" + } +] diff --git a/test/expected/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.out b/test/expected/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.out index 8534914d..7600bd00 100644 --- a/test/expected/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.out +++ b/test/expected/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.out @@ -1,4 +1,4 @@ -Feb 25 16:20:12 192.168.4.2 haproxy[7]: 95.216.197.33:56224 [25/Feb/2019:16:20:10.111] prod_http_in/sktst2: SSL handshake failure -Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50189 [25/Feb/2019:16:20:12.331] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 2496 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/core/core.css?1550939640 HTTP/1.1" -Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50187 [25/Feb/2019:16:20:12.325] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 1859 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/pi_popup/1.1.0/magnific-popup.css?1550939704 HTTP/1.1" -Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50188 [25/Feb/2019:16:20:12.321] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 5959 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/pi_fontawesome/css/font-awesome.css?1550939694 HTTP/1.1" +Feb 25 16:20:15 192.168.4.2 haproxy[7]: 95.216.197.33:56224 [25/Feb/2019:16:20:10.111] prod_http_in/sktst2: SSL handshake failure +Feb 25 16:20:16 192.168.4.2 haproxy[7]: 87.183.41.77:50188 [25/Feb/2019:16:20:12.321] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 5959 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/pi_fontawesome/css/font-awesome.css?1550939694 HTTP/1.1" +Feb 25 16:20:17 192.168.4.2 haproxy[7]: 87.183.41.77:50187 [25/Feb/2019:16:20:12.325] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 1859 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/pi_popup/1.1.0/magnific-popup.css?1550939704 HTTP/1.1" +Feb 25 16:20:18 192.168.4.2 haproxy[7]: 87.183.41.77:50189 [25/Feb/2019:16:20:12.331] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 2496 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/core/core.css?1550939640 HTTP/1.1" diff --git a/test/expected/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.out b/test/expected/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.out index 938513c4..583cdb2e 100644 --- a/test/expected/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.out +++ b/test/expected/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.out @@ -13,34 +13,36 @@ /global/keymap_def_zoom -> default-keymap.json:12 /tuning/archive-manager/cache-ttl -> root-config.json:16 /tuning/archive-manager/min-free-space -> root-config.json:15 -/tuning/clipboard/impls/MacOS/find/read -> root-config.json:43 -/tuning/clipboard/impls/MacOS/find/write -> root-config.json:42 -/tuning/clipboard/impls/MacOS/general/read -> root-config.json:39 -/tuning/clipboard/impls/MacOS/general/write -> root-config.json:38 -/tuning/clipboard/impls/MacOS/test -> root-config.json:36 -/tuning/clipboard/impls/NeoVim/general/read -> root-config.json:71 -/tuning/clipboard/impls/NeoVim/general/write -> root-config.json:70 -/tuning/clipboard/impls/NeoVim/test -> root-config.json:68 -/tuning/clipboard/impls/Wayland/general/read -> root-config.json:50 -/tuning/clipboard/impls/Wayland/general/write -> root-config.json:49 -/tuning/clipboard/impls/Wayland/test -> root-config.json:47 -/tuning/clipboard/impls/Windows/general/write -> root-config.json:77 -/tuning/clipboard/impls/Windows/test -> root-config.json:75 -/tuning/clipboard/impls/X11-xclip/general/read -> root-config.json:57 -/tuning/clipboard/impls/X11-xclip/general/write -> root-config.json:56 -/tuning/clipboard/impls/X11-xclip/test -> root-config.json:54 -/tuning/clipboard/impls/tmux/general/read -> root-config.json:64 -/tuning/clipboard/impls/tmux/general/write -> root-config.json:63 -/tuning/clipboard/impls/tmux/test -> root-config.json:61 +/tuning/clipboard/impls/MacOS/find/read -> root-config.json:44 +/tuning/clipboard/impls/MacOS/find/write -> root-config.json:43 +/tuning/clipboard/impls/MacOS/general/read -> root-config.json:40 +/tuning/clipboard/impls/MacOS/general/write -> root-config.json:39 +/tuning/clipboard/impls/MacOS/test -> root-config.json:37 +/tuning/clipboard/impls/NeoVim/general/read -> root-config.json:72 +/tuning/clipboard/impls/NeoVim/general/write -> root-config.json:71 +/tuning/clipboard/impls/NeoVim/test -> root-config.json:69 +/tuning/clipboard/impls/Wayland/general/read -> root-config.json:51 +/tuning/clipboard/impls/Wayland/general/write -> root-config.json:50 +/tuning/clipboard/impls/Wayland/test -> root-config.json:48 +/tuning/clipboard/impls/Windows/general/write -> root-config.json:78 +/tuning/clipboard/impls/Windows/test -> root-config.json:76 +/tuning/clipboard/impls/X11-xclip/general/read -> root-config.json:58 +/tuning/clipboard/impls/X11-xclip/general/write -> root-config.json:57 +/tuning/clipboard/impls/X11-xclip/test -> root-config.json:55 +/tuning/clipboard/impls/tmux/general/read -> root-config.json:65 +/tuning/clipboard/impls/tmux/general/write -> root-config.json:64 +/tuning/clipboard/impls/tmux/test -> root-config.json:62 /tuning/piper/max-size -> root-config.json:30 /tuning/piper/rotations -> root-config.json:31 +/tuning/piper/ttl -> root-config.json:32 /tuning/remote/ssh/command -> root-config.json:20 /tuning/remote/ssh/config/BatchMode -> root-config.json:22 /tuning/remote/ssh/config/ConnectTimeout -> root-config.json:23 /tuning/remote/ssh/start-command -> root-config.json:25 /tuning/remote/ssh/transfer-command -> root-config.json:26 -/tuning/url-scheme/docker/handler -> root-config.json:84 +/tuning/url-scheme/docker/handler -> root-config.json:85 /tuning/url-scheme/hw/handler -> {test_dir}/configs/installed/hw-url-handler.json:6 +/tuning/url-scheme/piper/handler -> root-config.json:88 /ui/clock-format -> root-config.json:4 /ui/default-colors -> root-config.json:6 /ui/dim-text -> root-config.json:5 diff --git a/test/logfile_haproxy.0 b/test/logfile_haproxy.0 index a9ee9e6c..92e1a8b0 100644 --- a/test/logfile_haproxy.0 +++ b/test/logfile_haproxy.0 @@ -9,9 +9,9 @@ Feb 25 16:20:04 192.168.4.2 haproxy[7]: 141.35.244.171:53337 [25/Feb/2019:16:20: Feb 25 16:20:09 192.168.4.2 haproxy[7]: 89.247.124.65:15564 [25/Feb/2019:16:20:06.321] prod_http_in~ bk_ktest_kt/nginx_sonst 0/0/1/43/2707 200 26170 - - ---- 3/3/1/1/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /portal/?Script=934&onlinetest=korrektur&anm=13915¤tPage=0 HTTP/1.1" Feb 25 16:20:11 192.168.4.2 haproxy[7]: 89.247.124.65:15565 [25/Feb/2019:16:20:08.872] prod_http_in~ bk_ktest_kt/nginx_sonst 0/0/1/26/2442 200 26170 - - ---- 3/2/1/1/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /portal/?Script=934&onlinetest=korrektur&anm=13915¤tPage=0 HTTP/1.1" Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50186 [25/Feb/2019:16:20:11.910] prod_http_in~ bk_ktest_kt/nginx_sonst 0/0/0/236/236 200 4586 - - ---- 4/4/3/3/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /portal/?Script=934&lehrer=77798 HTTP/1.1" -Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50186 [25/Feb/2019:16:20:12.234] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 16416 - - ---- 4/4/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/core/bootstrap_3.3.7/css/bootstrap.css?1550939643 HTTP/1.1" -Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50186 [25/Feb/2019:16:20:12.317] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/0/1/1 200 11065 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/core/jquery/jquery-ui-1.12.1.js?1550939557 HTTP/1.1" -Feb 25 16:20:12 192.168.4.2 haproxy[7]: 95.216.197.33:56224 [25/Feb/2019:16:20:10.111] prod_http_in/sktst2: SSL handshake failure -Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50188 [25/Feb/2019:16:20:12.321] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 5959 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/pi_fontawesome/css/font-awesome.css?1550939694 HTTP/1.1" -Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50187 [25/Feb/2019:16:20:12.325] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 1859 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/pi_popup/1.1.0/magnific-popup.css?1550939704 HTTP/1.1" -Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50189 [25/Feb/2019:16:20:12.331] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 2496 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/core/core.css?1550939640 HTTP/1.1" +Feb 25 16:20:13 192.168.4.2 haproxy[7]: 87.183.41.77:50186 [25/Feb/2019:16:20:13.234] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 16416 - - ---- 4/4/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/core/bootstrap_3.3.7/css/bootstrap.css?1550939643 HTTP/1.1" +Feb 25 16:20:14 192.168.4.2 haproxy[7]: 87.183.41.77:50186 [25/Feb/2019:16:20:14.317] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/0/1/1 200 11065 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/core/jquery/jquery-ui-1.12.1.js?1550939557 HTTP/1.1" +Feb 25 16:20:15 192.168.4.2 haproxy[7]: 95.216.197.33:56224 [25/Feb/2019:16:20:10.111] prod_http_in/sktst2: SSL handshake failure +Feb 25 16:20:16 192.168.4.2 haproxy[7]: 87.183.41.77:50188 [25/Feb/2019:16:20:12.321] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 5959 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/pi_fontawesome/css/font-awesome.css?1550939694 HTTP/1.1" +Feb 25 16:20:17 192.168.4.2 haproxy[7]: 87.183.41.77:50187 [25/Feb/2019:16:20:12.325] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 1859 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/pi_popup/1.1.0/magnific-popup.css?1550939704 HTTP/1.1" +Feb 25 16:20:18 192.168.4.2 haproxy[7]: 87.183.41.77:50189 [25/Feb/2019:16:20:12.331] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 2496 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/core/core.css?1550939640 HTTP/1.1" diff --git a/test/test_cli.sh b/test/test_cli.sh index 75b7d472..b8698f7e 100644 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -45,3 +45,17 @@ mkdir -p $HOME/.lnav run_cap_test ${lnav_test} -m -I ${test_dir} config get run_cap_test ${lnav_test} -m -I ${test_dir} config blame + +export TMPDIR="piper-tmp" +rm -rf ./piper-tmp +run_cap_test ${lnav_test} -n -e 'echo hi' + +run_cap_test ${lnav_test} -m piper list + +PIPER_URL=$(${lnav_test} -m -q piper list | tail -1 | sed -r -e 's;.*(piper://[^ ]+).*;\1;g') + +run_cap_test ${lnav_test} -n $PIPER_URL + +run_cap_test ${lnav_test} -n $PIPER_URL \ + -c ";SELECT filepath, descriptor, mimetype, jget(content, '/ctime') as ctime, jget(content, '/cwd') as cwd FROM lnav_file_metadata" \ + -c ':write-json-to -'