diff --git a/NEWS.md b/NEWS.md index 41e5a0dc..7e5405c6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ Features: * Added the "last-word" line-format field shortening algorithm from @flicus. +* Added a `stats.hist` PRQL transform that produces a histogram + of values over time. Bug Fixes: * With the recent xz backdoor shenanigans, it seems like a good diff --git a/src/db_sub_source.cc b/src/db_sub_source.cc index 45cc271d..ffd112bf 100644 --- a/src/db_sub_source.cc +++ b/src/db_sub_source.cc @@ -359,7 +359,9 @@ db_overlay_source::list_value_for_overlay(const listview_curses& lv, stacked_bar_chart chart; int start_line = value_out.size(); - chart.with_stacking_enabled(false).with_margins(3, 0); + chart.with_stacking_enabled(false) + .with_margins(3, 0) + .with_show_state(stacked_bar_chart_base::show_all{}); for (auto& jpw_value : jpw.jpw_values) { value_out.emplace_back(" " + jpw_value.wt_ptr + " = " diff --git a/src/internals/sql-ref.rst b/src/internals/sql-ref.rst index 345842a1..15644c5f 100644 --- a/src/internals/sql-ref.rst +++ b/src/internals/sql-ref.rst @@ -4548,7 +4548,7 @@ aggregate *expr* [1,2] **See Also** - :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4564,7 +4564,7 @@ append *table* * **table\*** --- The table to use as a source **See Also** - :ref:`prql_aggregate`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4590,7 +4590,7 @@ derive *column* 2 4 **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4614,7 +4614,7 @@ filter *expr* 2 **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4659,7 +4659,7 @@ from *table* 2 def **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4688,7 +4688,7 @@ group *key_columns* *pipeline* error 1 **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4706,7 +4706,7 @@ join *\[side:inner\]* *table* *condition* * **condition\*** --- The condition used to join rows **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4741,7 +4741,7 @@ select *expr* 4 **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4767,7 +4767,7 @@ sort *expr* 1 **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4791,7 +4791,7 @@ stats.average_of *col* 1.3333333333333333 **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4818,7 +4818,7 @@ stats.by *col* *values* 2 1 **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4844,7 +4844,35 @@ stats.count_by *column* 2 1 **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` + +---- + + +.. _stats_hist: + +stats.hist *col* *\[slice:'5m'\]* +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + Count values per bucket of time + + **Parameters** + * **col\*** --- The column to count + * **slice** --- The time slice + + **Examples** + To chart the values of ex_procname over time: + + .. code-block:: custsqlite + + ;from lnav_example_log | stats.hist ex_procname + tslice v + 2017-02⋯:00.000 {"gw":1,"hw":1} + 2017-02⋯:00.000 {"gw":1} + 2017-02⋯:00.000 {"gw":1} + + **See Also** + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4868,7 +4896,7 @@ stats.sum_of *col* 4 **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`utils_distinct` ---- @@ -4901,7 +4929,7 @@ take *n_or_range* 3 **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of`, :ref:`utils_distinct` ---- @@ -4927,7 +4955,7 @@ utils.distinct *col* 2 **See Also** - :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of` + :ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_hist`, :ref:`stats_sum_of` ---- diff --git a/src/prql/stats.prql b/src/prql/stats.prql index 66c4bb0a..f96f66c7 100644 --- a/src/prql/stats.prql +++ b/src/prql/stats.prql @@ -17,3 +17,13 @@ let by = func column values rel -> ( rel group {column} (aggregate values) ) + +let hist = func column slice:'5m' rel -> ( + rel + group { tslice = (time.slice log_time_msecs slice), column } ( + aggregate { total = count(this) } + ) + group { tslice } ( + aggregate { v = json.group_object column total } + ) +) diff --git a/src/sql_commands.cc b/src/sql_commands.cc index 2d154cb5..b6fba63a 100644 --- a/src/sql_commands.cc +++ b/src/sql_commands.cc @@ -667,6 +667,25 @@ static readline_context::command_t sql_commands[] = { "prql-source", {"prql-source"}, }, + { + "stats.hist", + prql_cmd_sort, + help_text("stats.hist", "Count values per bucket of time") + .prql_function() + .with_tags({"prql"}) + .with_parameter(help_text{"col", "The column to count"}) + .with_parameter(help_text{"slice", "The time slice"} + .optional() + .with_default_value("'5m'")) + .with_example({ + "To chart the values of ex_procname over time", + "from lnav_example_log | stats.hist ex_procname", + help_example::language::prql, + }), + nullptr, + "prql-source", + {"prql-source"}, + }, { "stats.sum_of", prql_cmd_sort, diff --git a/src/sql_util.cc b/src/sql_util.cc index 6b1459a4..67cbde20 100644 --- a/src/sql_util.cc +++ b/src/sql_util.cc @@ -1384,10 +1384,6 @@ annotate_prql_statement(attr_line_t& al) lnav::pcre2pp::code::from_const(R"(\A(?:\[|\]|\{|\}|\(|\)))"), &PRQL_PAREN_ATTR, }, - { - lnav::pcre2pp::code::from_const(R"(\A\|)"), - &PRQL_PIPE_ATTR, - }, { lnav::pcre2pp::code::from(transform_re_str).unwrap(), &PRQL_TRANSFORM_ATTR, @@ -1424,9 +1420,13 @@ annotate_prql_statement(attr_line_t& al) }, { lnav::pcre2pp::code::from_const( - R"(\A(\*|\->{1,2}|<|>|=>|={1,2}|!|\-|\+|~=|\.\.|,))"), + R"(\A(\*|\->{1,2}|<|>|=>|={1,2}|\|\||&&|!|\-|\+|~=|\.\.|,|\?\?))"), &PRQL_OPERATOR_ATTR, }, + { + lnav::pcre2pp::code::from_const(R"(\A\|)"), + &PRQL_PIPE_ATTR, + }, { lnav::pcre2pp::code::from_const(R"(\A\.)"), &PRQL_DOT_ATTR, diff --git a/test/expected/expected.am b/test/expected/expected.am index 47b56cf6..78cb0959 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -776,6 +776,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql_anno.sh_ce0506ee7a12eb0f7b970522cc6a79180ecb20cc.out \ $(srcdir)/%reldir%/test_sql_anno.sh_de46094b6e005285dc0921ef9979e36240c5042d.err \ $(srcdir)/%reldir%/test_sql_anno.sh_de46094b6e005285dc0921ef9979e36240c5042d.out \ + $(srcdir)/%reldir%/test_sql_anno.sh_e7dae4ba18c42c416ed03afd8819200f63e89ac8.err \ + $(srcdir)/%reldir%/test_sql_anno.sh_e7dae4ba18c42c416ed03afd8819200f63e89ac8.out \ $(srcdir)/%reldir%/test_sql_anno.sh_f3c64191d6016767a5857fbb1bad26548586bb96.err \ $(srcdir)/%reldir%/test_sql_anno.sh_f3c64191d6016767a5857fbb1bad26548586bb96.out \ $(srcdir)/%reldir%/test_sql_coll_func.sh_077cab6e271c914daf5b221cc512853077891f35.err \ diff --git a/test/expected/test_sql_anno.sh_e7dae4ba18c42c416ed03afd8819200f63e89ac8.err b/test/expected/test_sql_anno.sh_e7dae4ba18c42c416ed03afd8819200f63e89ac8.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_sql_anno.sh_e7dae4ba18c42c416ed03afd8819200f63e89ac8.out b/test/expected/test_sql_anno.sh_e7dae4ba18c42c416ed03afd8819200f63e89ac8.out new file mode 100644 index 00000000..fa790882 --- /dev/null +++ b/test/expected/test_sql_anno.sh_e7dae4ba18c42c416ed03afd8819200f63e89ac8.out @@ -0,0 +1,17 @@ + from access_log | filter cs_method == 'GET' || cs_method == 'PUT' + prql_stage ---------------- + prql_transform ---- + prql_ident ---------- + prql_fqid ---------- + prql_stage ------------------------------------------------- + prql_pipe - + prql_transform ------ + prql_ident --------- + prql_fqid --------- + prql_oper -- + prql_string ----- + prql_oper -- + prql_ident --------- + prql_fqid --------- + prql_oper -- + prql_string ----- diff --git a/test/test_sql_anno.sh b/test/test_sql_anno.sh index 27896c16..74af141b 100644 --- a/test/test_sql_anno.sh +++ b/test/test_sql_anno.sh @@ -52,3 +52,5 @@ run_cap_test ./drive_sql_anno "SELECT * FROM foo.bar" run_cap_test ./drive_sql_anno "SELECT json_object('abc', 'def') ->> '$.abc'" run_cap_test ./drive_sql_anno "SELECT 0x77, 123, 123e4" + +run_cap_test ./drive_sql_anno "from access_log | filter cs_method == 'GET' || cs_method == 'PUT'"