From 5d478fc17afa48f8f45a92caae2d70bfeb290568 Mon Sep 17 00:00:00 2001 From: Timothy Stack Date: Sat, 8 Jun 2013 06:10:18 -0700 Subject: [PATCH] [cleanup] start a tracer/debugger for the data parser --- Makefile.in | 1 + configure | 42 +++ configure.ac | 2 + src/Makefile.am | 6 +- src/Makefile.in | 24 +- src/auto_mem.hh | 2 +- src/data_parser.cc | 20 ++ src/data_parser.hh | 269 ++++++++++++++---- src/data_scanner.cc | 1 + src/data_scanner.hh | 1 + src/log_data_table.hh | 2 +- src/log_format.cc | 1 + src/log_format_impls.cc | 10 +- src/pcrepp.hh | 18 +- src/sql_util.cc | 2 +- src/sqlite-extension-func.c | 1 + src/sqlite-extension-func.h | 3 + src/statusview_curses.hh | 2 +- src/string-extension-functions.cc | 125 ++++++++ src/test_override.cc | 15 + src/textview_curses.cc | 2 +- src/textview_curses.hh | 17 +- test/Makefile.am | 11 + test/Makefile.in | 14 +- test/datafile_simple.8 | 2 - test/drive_data_scanner.cc | 4 + ...f7fbb3546c676c686fac0ed096d026f46c875f.txt | 25 ++ test/parser_debugger.py | 204 +++++++++++++ test/simple-db.sql | 12 + test/sql.0.in | 6 + test/sql.0.out | 7 + 31 files changed, 779 insertions(+), 72 deletions(-) create mode 100644 src/string-extension-functions.cc create mode 100644 src/test_override.cc create mode 100644 test/log-samples/sample-9cf7fbb3546c676c686fac0ed096d026f46c875f.txt create mode 100755 test/parser_debugger.py create mode 100644 test/simple-db.sql create mode 100644 test/sql.0.in create mode 100644 test/sql.0.out diff --git a/Makefile.in b/Makefile.in index c6ee9e88..663e4809 100644 --- a/Makefile.in +++ b/Makefile.in @@ -249,6 +249,7 @@ READLINE_LIBS = @READLINE_LIBS@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ SQLITE3_CFLAGS = @SQLITE3_CFLAGS@ +SQLITE3_CMD = @SQLITE3_CMD@ SQLITE3_LDFLAGS = @SQLITE3_LDFLAGS@ SQLITE3_LIBS = @SQLITE3_LIBS@ SQLITE3_VERSION = @SQLITE3_VERSION@ diff --git a/configure b/configure index eddc6edb..ddc79aad 100755 --- a/configure +++ b/configure @@ -645,6 +645,7 @@ CCDEPMODE ac_ct_CC CFLAGS CC +SQLITE3_CMD LN_S RANLIB CFLAGS_PG @@ -3946,6 +3947,47 @@ $as_echo "no" >&6; } fi +# Extract the first word of "sqlite3", so it can be a program name with args. +set dummy sqlite3; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_SQLITE3_CMD+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $SQLITE3_CMD in + [\\/]* | ?:[\\/]*) + ac_cv_path_SQLITE3_CMD="$SQLITE3_CMD" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_SQLITE3_CMD="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +SQLITE3_CMD=$ac_cv_path_SQLITE3_CMD +if test -n "$SQLITE3_CMD"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $SQLITE3_CMD" >&5 +$as_echo "$SQLITE3_CMD" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' diff --git a/configure.ac b/configure.ac index 8d2d8ddb..7be2f1da 100644 --- a/configure.ac +++ b/configure.ac @@ -88,6 +88,8 @@ AC_PROG_RANLIB AC_PROG_LN_S AC_PROG_MAKE_SET +AC_PATH_PROG(SQLITE3_CMD, [sqlite3]) + AC_CHECK_SIZEOF(off_t) AC_CHECK_SIZEOF(size_t) diff --git a/src/Makefile.am b/src/Makefile.am index d880062b..e7d537d4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,7 +1,7 @@ bin_PROGRAMS = lnav -noinst_PROGRAMS = bin2c +noinst_PROGRAMS = bin2c lnav-test noinst_LIBRARIES = libdiag.a @@ -111,6 +111,7 @@ libdiag_a_SOURCES = \ sequence_matcher.cc \ sqlite-extension-func.c \ statusview_curses.cc \ + string-extension-functions.cc \ piper_proc.cc \ sql_util.cc \ strnatcmp.c \ @@ -139,6 +140,9 @@ libdiag_a_SOURCES = \ lnav_SOURCES = lnav.cc $(HELP_SRC) lnav_LDADD = help.o $(LDADD) +lnav_test_SOURCES = lnav.cc $(HELP_SRC) test_override.cc +lnav_test_LDADD = help.o $(LDADD) + bin2c_SOURCES = bin2c.c DISTCLEANFILES = help.c diff --git a/src/Makefile.in b/src/Makefile.in index 210a74ea..866ad459 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -81,7 +81,7 @@ POST_UNINSTALL = : build_triplet = @build@ host_triplet = @host@ bin_PROGRAMS = lnav$(EXEEXT) -noinst_PROGRAMS = bin2c$(EXEEXT) +noinst_PROGRAMS = bin2c$(EXEEXT) lnav-test$(EXEEXT) subdir = src DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \ $(srcdir)/config.h.in $(top_srcdir)/mkinstalldirs \ @@ -116,7 +116,8 @@ am_libdiag_a_OBJECTS = bookmarks.$(OBJEXT) \ data_parser.$(OBJEXT) readline_curses.$(OBJEXT) \ session_data.$(OBJEXT) sequence_matcher.$(OBJEXT) \ sqlite-extension-func.$(OBJEXT) statusview_curses.$(OBJEXT) \ - piper_proc.$(OBJEXT) sql_util.$(OBJEXT) strnatcmp.$(OBJEXT) \ + string-extension-functions.$(OBJEXT) piper_proc.$(OBJEXT) \ + sql_util.$(OBJEXT) strnatcmp.$(OBJEXT) \ textview_curses.$(OBJEXT) view_curses.$(OBJEXT) \ vt52_curses.$(OBJEXT) log_vtab_impl.$(OBJEXT) \ xterm_mouse.$(OBJEXT) yajlpp.$(OBJEXT) yajl.$(OBJEXT) \ @@ -137,6 +138,9 @@ lnav_OBJECTS = $(am_lnav_OBJECTS) am__DEPENDENCIES_2 = libdiag.a $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) lnav_DEPENDENCIES = help.o $(am__DEPENDENCIES_2) +am_lnav_test_OBJECTS = lnav.$(OBJEXT) test_override.$(OBJEXT) +lnav_test_OBJECTS = $(am_lnav_test_OBJECTS) +lnav_test_DEPENDENCIES = help.o $(am__DEPENDENCIES_2) AM_V_P = $(am__v_P_@AM_V@) am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) am__v_P_0 = false @@ -182,8 +186,10 @@ AM_V_CXXLD = $(am__v_CXXLD_@AM_V@) am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@) am__v_CXXLD_0 = @echo " CXXLD " $@; am__v_CXXLD_1 = -SOURCES = $(libdiag_a_SOURCES) $(bin2c_SOURCES) $(lnav_SOURCES) -DIST_SOURCES = $(libdiag_a_SOURCES) $(bin2c_SOURCES) $(lnav_SOURCES) +SOURCES = $(libdiag_a_SOURCES) $(bin2c_SOURCES) $(lnav_SOURCES) \ + $(lnav_test_SOURCES) +DIST_SOURCES = $(libdiag_a_SOURCES) $(bin2c_SOURCES) $(lnav_SOURCES) \ + $(lnav_test_SOURCES) am__can_run_installinfo = \ case $$AM_UPDATE_INFO_DIR in \ n|no|NO) false;; \ @@ -266,6 +272,7 @@ READLINE_LIBS = @READLINE_LIBS@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ SQLITE3_CFLAGS = @SQLITE3_CFLAGS@ +SQLITE3_CMD = @SQLITE3_CMD@ SQLITE3_LDFLAGS = @SQLITE3_LDFLAGS@ SQLITE3_LIBS = @SQLITE3_LIBS@ SQLITE3_VERSION = @SQLITE3_VERSION@ @@ -428,6 +435,7 @@ libdiag_a_SOURCES = \ sequence_matcher.cc \ sqlite-extension-func.c \ statusview_curses.cc \ + string-extension-functions.cc \ piper_proc.cc \ sql_util.cc \ strnatcmp.c \ @@ -455,6 +463,8 @@ libdiag_a_SOURCES = \ lnav_SOURCES = lnav.cc $(HELP_SRC) lnav_LDADD = help.o $(LDADD) +lnav_test_SOURCES = lnav.cc $(HELP_SRC) test_override.cc +lnav_test_LDADD = help.o $(LDADD) bin2c_SOURCES = bin2c.c DISTCLEANFILES = help.c all: config.h @@ -569,6 +579,10 @@ lnav$(EXEEXT): $(lnav_OBJECTS) $(lnav_DEPENDENCIES) $(EXTRA_lnav_DEPENDENCIES) @rm -f lnav$(EXEEXT) $(AM_V_CXXLD)$(CXXLINK) $(lnav_OBJECTS) $(lnav_LDADD) $(LIBS) +lnav-test$(EXEEXT): $(lnav_test_OBJECTS) $(lnav_test_DEPENDENCIES) $(EXTRA_lnav_test_DEPENDENCIES) + @rm -f lnav-test$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(lnav_test_OBJECTS) $(lnav_test_LDADD) $(LIBS) + mostlyclean-compile: -rm -f *.$(OBJEXT) @@ -602,7 +616,9 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sql_util.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sqlite-extension-func.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/statusview_curses.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/string-extension-functions.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/strnatcmp.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_override.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/textview_curses.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/view_curses.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/vt52_curses.Po@am__quote@ diff --git a/src/auto_mem.hh b/src/auto_mem.hh index f752d3e8..464ace9e 100644 --- a/src/auto_mem.hh +++ b/src/auto_mem.hh @@ -87,7 +87,7 @@ public: return retval; }; - T *in(void) + T *in(void) const { return this->am_ptr; }; diff --git a/src/data_parser.cc b/src/data_parser.cc index 573e3928..33ba6895 100644 --- a/src/data_parser.cc +++ b/src/data_parser.cc @@ -37,6 +37,8 @@ data_format data_parser::FORMAT_SEMI("semi", DT_COMMA, DT_SEMI); data_format data_parser::FORMAT_COMMA("comma", DT_INVALID, DT_COMMA); data_format data_parser::FORMAT_PLAIN("plain", DT_INVALID, DT_INVALID); +FILE *data_parser::TRACE_FILE; + data_format_state_t dfs_prefix_next(data_format_state_t state, data_token_t next_token) { @@ -63,6 +65,7 @@ data_format_state_t dfs_prefix_next(data_format_state_t state, break; } break; + case DFS_EXPECTING_SEP: case DFS_ERROR: retval = DFS_ERROR; break; @@ -108,6 +111,7 @@ data_format_state_t dfs_semi_next(data_format_state_t state, } break; + case DFS_EXPECTING_SEP: case DFS_ERROR: retval = DFS_ERROR; break; } @@ -144,6 +148,10 @@ data_format_state_t dfs_comma_next(data_format_state_t state, retval = DFS_INIT; break; + case DT_WORD: + retval = DFS_EXPECTING_SEP; + break; + case DT_SEMI: retval = DFS_ERROR; break; @@ -151,6 +159,18 @@ data_format_state_t dfs_comma_next(data_format_state_t state, default: break; } break; + case DFS_EXPECTING_SEP: + switch (next_token) { + case DT_SEPARATOR: + retval = DFS_VALUE; + break; + case DT_COMMA: + case DT_SEMI: + retval = DFS_ERROR; + break; + default: break; + } + break; case DFS_VALUE: switch (next_token) { diff --git a/src/data_parser.hh b/src/data_parser.hh index d32ec9c8..cab8687b 100644 --- a/src/data_parser.hh +++ b/src/data_parser.hh @@ -39,6 +39,7 @@ #include #include +#include "yajlpp.hh" #include "pcrepp.hh" #include "byte_array.hh" #include "data_scanner.hh" @@ -109,15 +110,20 @@ * into the 'bookmarks' table to create new user bookmarks. */ +#define ELEMENT_LIST_T(var) var("" #var, __FILE__, __LINE__) +#define PUSH_BACK(elem) push_back(elem, __FILE__, __LINE__) +#define POP_FRONT(elem) pop_front(__FILE__, __LINE__) +#define POP_BACK(elem) pop_back(__FILE__, __LINE__) +#define SPLICE(pos, other, first, last) splice(pos, other, first, last, __FILE__, __LINE__) template void strip(Container &container, UnaryPredicate p) { while (!container.empty() && p(container.front())) { - container.pop_front(); + container.POP_FRONT(); } while (!container.empty() && p(container.back())) { - container.pop_back(); + container.POP_BACK(); } } @@ -125,6 +131,7 @@ enum data_format_state_t { DFS_ERROR = -1, DFS_INIT, DFS_KEY, + DFS_EXPECTING_SEP, DFS_VALUE, }; @@ -147,16 +154,142 @@ data_format_state_t dfs_semi_next(data_format_state_t state, data_format_state_t dfs_comma_next(data_format_state_t state, data_token_t next_token); +#define LIST_INIT_TRACE \ + do { \ + if (TRACE_FILE != NULL) { \ + fprintf(TRACE_FILE, \ + "%p %s:%d %s %s\n", \ + this, \ + fn, line, \ + __func__, \ + varname); \ + } \ + } while (false) + +#define LIST_DEINIT_TRACE \ + do { \ + if (TRACE_FILE != NULL) { \ + fprintf(TRACE_FILE, \ + "%p %s:%d %s\n", \ + this, \ + fn, line, \ + __func__); \ + } \ + } while (false) + +#define ELEMENT_TRACE \ + do { \ + if (TRACE_FILE != NULL) { \ + fprintf(TRACE_FILE, \ + "%p %s:%d %s %s %d:%d\n", \ + this, \ + fn, line, \ + __func__, \ + data_scanner::token2name(elem.e_token), \ + elem.e_capture.c_begin, \ + elem.e_capture.c_end); \ + } \ + } while (false) + +#define LIST_TRACE \ + do { \ + if (TRACE_FILE != NULL) { \ + fprintf(TRACE_FILE, \ + "%p %s:%d %s\n", \ + this, \ + fn, line, \ + __func__); \ + } \ + } while (false) + +#define SPLICE_TRACE \ + do { \ + if (TRACE_FILE != NULL) { \ + fprintf(TRACE_FILE, \ + "%p %s:%d %s %d %p %d:%d\n", \ + this, \ + fn, line, \ + __func__, \ + (int)std::distance(this->begin(), pos), \ + &other, \ + (int)std::distance(other.begin(), first), \ + (int)std::distance(last, other.end())); \ + } \ + } while (false); + +#define POINT_TRACE(name) \ + do { \ + if (TRACE_FILE) { \ + fprintf(TRACE_FILE, \ + "0x0 %s:%d point %s\n", \ + __FILE__, __LINE__, \ + name); \ + } \ + } while(false); + class data_parser { public: static data_format FORMAT_SEMI; static data_format FORMAT_COMMA; static data_format FORMAT_PLAIN; + static FILE *TRACE_FILE; + typedef byte_array schema_id_t; struct element; - typedef std::list element_list_t; + // typedef std::list element_list_t; + + class element_list_t : public std::list { + public: + element_list_t(const char *varname, const char *fn, int line) { + LIST_INIT_TRACE; + } + + element_list_t() { + const char *varname = "_anon2_"; + const char *fn = __FILE__; + int line = __LINE__; + + LIST_INIT_TRACE; + }; + + ~element_list_t() { + const char *fn = __FILE__; + int line = __LINE__; + + LIST_DEINIT_TRACE; + }; + + void push_back(const element &elem, const char *fn, int line) { + ELEMENT_TRACE; + + this->std::list::push_back(elem); + }; + + void pop_front(const char *fn, int line) { + LIST_TRACE; + + this->std::list::pop_front(); + }; + + void pop_back(const char *fn, int line) { + LIST_TRACE; + + this->std::list::pop_back(); + }; + + void splice(iterator pos, + element_list_t &other, + iterator first, + iterator last, + const char *fn, + int line) { + SPLICE_TRACE; + + this->std::list::splice(pos, other, first, last); + } + }; struct element { element() : e_token(DT_INVALID), e_sub_elements(NULL) { }; @@ -207,7 +340,7 @@ public: void assign_elements(element_list_t &subs) { if (this->e_sub_elements == NULL) { - this->e_sub_elements = new element_list_t(); + this->e_sub_elements = new element_list_t("_sub_", __FILE__, __LINE__); } this->e_sub_elements->swap(subs); this->update_capture(); @@ -315,19 +448,29 @@ private: data_token_t ei_token; }; - data_parser(data_scanner *ds) : dp_format(NULL), dp_scanner(ds) { }; + data_parser(data_scanner *ds) + : dp_errors("dp_errors", __FILE__, __LINE__), + dp_pairs("dp_pairs", __FILE__, __LINE__), + dp_format(NULL), + dp_scanner(ds) { + if (TRACE_FILE != NULL) { + fprintf(TRACE_FILE, "input %s\n", ds->get_input().get_string()); + } + }; void pairup(schema_id_t *schema, element_list_t &pairs_out, element_list_t &in_list) { - element_list_t el_stack, free_row, key_comps, value, prefix; + element_list_t ELEMENT_LIST_T(el_stack), ELEMENT_LIST_T(free_row), ELEMENT_LIST_T(key_comps), ELEMENT_LIST_T(value), ELEMENT_LIST_T(prefix); SHA_CTX context; + POINT_TRACE("pairup_start"); + for (element_list_t::iterator iter = in_list.begin(); iter != in_list.end(); ++iter) { if (iter->e_token == DNT_GROUP) { - element_list_t group_pairs; + element_list_t ELEMENT_LIST_T(group_pairs); this->pairup(NULL, group_pairs, *iter->e_sub_elements); if (!group_pairs.empty()) { @@ -338,7 +481,7 @@ private: if (iter->e_token == this->dp_format->df_terminator) { std::vector key_copy; - value.splice(value.end(), + value.SPLICE(value.end(), key_comps, key_comps.begin(), key_comps.end()); @@ -346,11 +489,11 @@ private: strip(value, element_if(DT_WHITE)); value.remove_if(element_if(DT_COMMA)); if (!value.empty()) { - el_stack.push_back(element(value, DNT_VALUE)); + el_stack.PUSH_BACK(element(value, DNT_VALUE)); } value.clear(); - key_comps.push_back(*iter); + key_comps.PUSH_BACK(*iter); } else if (iter->e_token == DT_SEPARATOR) { element_list_t::iterator key_iter = key_comps.end(); @@ -360,12 +503,13 @@ private: --key_iter; if (key_iter->e_token == this->dp_format->df_appender) { ++key_iter; - value.splice(value.end(), + value.SPLICE(value.end(), key_comps, key_comps.begin(), key_iter); - key_comps.splice(key_comps.begin(), + key_comps.SPLICE(key_comps.begin(), key_comps, + --(key_comps.end()), key_comps.end()); key_comps.resize(1); found = true; @@ -374,12 +518,12 @@ private: this->dp_format->df_terminator) { std::vector key_copy; - value.splice(value.end(), + value.SPLICE(value.end(), key_comps, key_comps.begin(), key_iter); ++key_iter; - key_comps.pop_front(); + key_comps.POP_FRONT(); strip(key_comps, element_if(DT_WHITE)); found = true; } @@ -395,41 +539,46 @@ private: continue; } - value.splice(value.end(), + value.SPLICE(value.end(), key_comps, key_comps.begin(), key_comps.end()); value_iter = value.end(); std::advance(value_iter, -1); - key_comps.splice(key_comps.begin(), + key_comps.SPLICE(key_comps.begin(), value, - value_iter); + value_iter, + value.end()); key_comps.resize(1); } strip(value, element_if(DT_WHITE)); value.remove_if(element_if(DT_COMMA)); if (!value.empty()) { - el_stack.push_back(element(value, DNT_VALUE)); + el_stack.PUSH_BACK(element(value, DNT_VALUE)); } strip(key_comps, element_if(DT_WHITE)); if (!key_comps.empty()) { - el_stack.push_back(element(key_comps, DNT_KEY, false)); + el_stack.PUSH_BACK(element(key_comps, DNT_KEY, false)); } key_comps.clear(); value.clear(); } else { - key_comps.push_back(*iter); + key_comps.PUSH_BACK(*iter); } + + POINT_TRACE("pairup_loop"); } + POINT_TRACE("pairup_eol"); + if (el_stack.empty()) { - free_row.splice(free_row.begin(), + free_row.SPLICE(free_row.begin(), key_comps, key_comps.begin(), key_comps.end()); } else { - value.splice(value.begin(), + value.SPLICE(value.begin(), key_comps, key_comps.begin(), key_comps.end()); @@ -438,72 +587,74 @@ private: strip(value, element_if(DT_WHITE)); value.remove_if(element_if(DT_COMMA)); if (!value.empty()) { - el_stack.push_back(element(value, DNT_VALUE)); + el_stack.PUSH_BACK(element(value, DNT_VALUE)); } } + POINT_TRACE("pairup_stack"); + SHA_Init(&context); while (!el_stack.empty()) { element_list_t::iterator kv_iter = el_stack.begin(); if (kv_iter->e_token == DNT_VALUE) { if (pairs_out.empty()) { - free_row.push_back(el_stack.front()); + free_row.PUSH_BACK(el_stack.front()); } else { - element_list_t free_pair_subs; + element_list_t ELEMENT_LIST_T(free_pair_subs); struct element blank; blank.e_capture.c_begin = blank.e_capture.c_end = el_stack.front().e_capture.c_begin; blank.e_token = DNT_KEY; - free_pair_subs.push_back(blank); - free_pair_subs.push_back(el_stack.front()); - pairs_out.push_back(element(free_pair_subs, DNT_PAIR)); + free_pair_subs.PUSH_BACK(blank); + free_pair_subs.PUSH_BACK(el_stack.front()); + pairs_out.PUSH_BACK(element(free_pair_subs, DNT_PAIR)); } } if (kv_iter->e_token != DNT_KEY) { - el_stack.pop_front(); + el_stack.POP_FRONT(); continue; } ++kv_iter; if (kv_iter == el_stack.end()) { - el_stack.pop_front(); + el_stack.POP_FRONT(); continue; } if (kv_iter->e_token != DNT_VALUE) { - el_stack.pop_front(); + el_stack.POP_FRONT(); continue; } std::string key_val = this->get_element_string(el_stack.front()); - element_list_t pair_subs; + element_list_t ELEMENT_LIST_T(pair_subs); if (schema != NULL) { SHA_Update(&context, key_val.c_str(), key_val.length()); } while (!free_row.empty()) { - element_list_t free_pair_subs; + element_list_t ELEMENT_LIST_T(free_pair_subs); struct element blank; blank.e_capture.c_begin = blank.e_capture.c_end = free_row.front().e_capture.c_begin; blank.e_token = DNT_KEY; - free_pair_subs.push_back(blank); - free_pair_subs.push_back(free_row.front()); - pairs_out.push_back(element(free_pair_subs, DNT_PAIR)); - free_row.pop_front(); + free_pair_subs.PUSH_BACK(blank); + free_pair_subs.PUSH_BACK(free_row.front()); + pairs_out.PUSH_BACK(element(free_pair_subs, DNT_PAIR)); + free_row.POP_FRONT(); } ++kv_iter; - pair_subs.splice(pair_subs.begin(), + pair_subs.SPLICE(pair_subs.begin(), el_stack, el_stack.begin(), kv_iter); - pairs_out.push_back(element(pair_subs, DNT_PAIR)); + pairs_out.PUSH_BACK(element(pair_subs, DNT_PAIR)); } if (pairs_out.size() == 1) { @@ -513,11 +664,16 @@ private: if (value.e_token == DNT_VALUE && value.e_sub_elements != NULL && value.e_sub_elements->size() > 1) { - prefix.splice(prefix.begin(), + element_list_t::iterator next_sub; + + next_sub = pair.e_sub_elements->begin(); + ++next_sub; + prefix.SPLICE(prefix.begin(), *pair.e_sub_elements, - pair.e_sub_elements->begin()); + pair.e_sub_elements->begin(), + next_sub); free_row.clear(); - free_row.splice(free_row.begin(), + free_row.SPLICE(free_row.begin(), *value.e_sub_elements, value.e_sub_elements->begin(), value.e_sub_elements->end()); @@ -531,6 +687,7 @@ private: switch (free_row.front().e_token) { case DNT_GROUP: case DNT_VALUE: + case DT_CONSTANT: case DT_NUMBER: case DT_SYMBOL: case DT_HEX_NUMBER: @@ -545,16 +702,16 @@ private: case DT_PATH: case DT_TIME: case DT_PERCENTAGE: { - element_list_t pair_subs; + element_list_t ELEMENT_LIST_T(pair_subs); struct element blank; blank.e_capture.c_begin = blank.e_capture.c_end = free_row.front().e_capture. c_begin; blank.e_token = DNT_KEY; - pair_subs.push_back(blank); - pair_subs.push_back(free_row.front()); - pairs_out.push_back(element(pair_subs, DNT_PAIR)); + pair_subs.PUSH_BACK(blank); + pair_subs.PUSH_BACK(free_row.front()); + pairs_out.PUSH_BACK(element(pair_subs, DNT_PAIR)); } break; @@ -567,19 +724,19 @@ private: break; } - free_row.pop_front(); + free_row.POP_FRONT(); } } if (!prefix.empty()) { - element_list_t pair_subs; + element_list_t ELEMENT_LIST_T(pair_subs); struct element blank; blank.e_capture.c_begin = blank.e_capture.c_end = prefix.front().e_capture.c_begin; blank.e_token = DNT_KEY; - pair_subs.push_back(blank); - pair_subs.push_back(prefix.front()); + pair_subs.PUSH_BACK(blank); + pair_subs.PUSH_BACK(prefix.front()); pairs_out.push_front(element(pair_subs, DNT_PAIR)); } @@ -629,7 +786,7 @@ private: case DT_LCURLY: case DT_LSQUARE: this->dp_group_token.push_back(elem.e_token); - this->dp_group_stack.push_back(element_list_t()); + this->dp_group_stack.push_back(element_list_t("_anon_", __FILE__, __LINE__)); break; case DT_RPAREN: @@ -643,18 +800,18 @@ private: this->dp_group_stack.rbegin(); ++riter; if (!this->dp_group_stack.back().empty()) { - (*riter).push_back(element(this->dp_group_stack.back(), + (*riter).PUSH_BACK(element(this->dp_group_stack.back(), DNT_GROUP)); } this->dp_group_stack.pop_back(); } else { - this->dp_group_stack.back().push_back(elem); + this->dp_group_stack.back().PUSH_BACK(elem); } break; default: - this->dp_group_stack.back().push_back(elem); + this->dp_group_stack.back().PUSH_BACK(elem); break; } } @@ -666,7 +823,7 @@ private: this->dp_group_stack.rbegin(); ++riter; if (!this->dp_group_stack.back().empty()) { - (*riter).push_back(element(this->dp_group_stack.back(), + (*riter).PUSH_BACK(element(this->dp_group_stack.back(), DNT_GROUP)); } this->dp_group_stack.pop_back(); @@ -702,7 +859,7 @@ private: } }; - std::string get_element_string(const element &elem) + std::string get_element_string(const element &elem) const { pcre_input &pi = this->dp_scanner->get_input(); diff --git a/src/data_scanner.cc b/src/data_scanner.cc index 787646be..4744b038 100644 --- a/src/data_scanner.cc +++ b/src/data_scanner.cc @@ -96,6 +96,7 @@ static struct { { "hex", pcrepp("(-?(?:0x|[0-9])[0-9a-fA-F]+\\b)"), }, + { "cnst", pcrepp("(true|True|TRUE|false|False|FALSE|None|null)\\b") }, { "word", pcrepp( "([a-zA-Z][a-z']+(?=[\\s\\(\\)!\\*:;'\\\"\\?,]|\\.\\s|$))"), }, { "sym", pcrepp("([^\";\\s:=,(){}\\[\\]]+)"), diff --git a/src/data_scanner.hh b/src/data_scanner.hh index e0a5e68c..db78b4f2 100644 --- a/src/data_scanner.hh +++ b/src/data_scanner.hh @@ -70,6 +70,7 @@ enum data_token_t { DT_NUMBER, DT_HEX_NUMBER, + DT_CONSTANT, DT_WORD, DT_SYMBOL, DT_LINE, diff --git a/src/log_data_table.hh b/src/log_data_table.hh index d322a248..ff32437c 100644 --- a/src/log_data_table.hh +++ b/src/log_data_table.hh @@ -26,7 +26,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * - * @file lnav.cc + * @file log_data_table.hh * * XXX This file has become a dumping ground for code and needs to be broken up * a bit. diff --git a/src/log_format.cc b/src/log_format.cc index 74ef11d5..65496eda 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -186,6 +186,7 @@ char *log_format::log_scanf(const char *line, "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%dT%H:%M:%SZ", "%Y/%m/%d %H:%M:%S", "%Y/%m/%d %H:%M", diff --git a/src/log_format_impls.cc b/src/log_format_impls.cc index 1f0f5447..51592e2a 100644 --- a/src/log_format_impls.cc +++ b/src/log_format_impls.cc @@ -575,7 +575,15 @@ class generic_log_format : public log_format { lr.lr_end = lr.lr_start + strlen(timestr); sa[lr].insert(make_string_attr("timestamp", 0)); - if (logline::string2level(level) == logline::LEVEL_UNKNOWN) { + for (int lpc = 0; level[lpc]; lpc++) { + if (!isalpha(level[lpc])) { + level[lpc] = '\0'; + prefix_len = strlen(timestr) + lpc; + break; + } + } + + if (logline::string2level(level, true) == logline::LEVEL_UNKNOWN) { prefix_len = strlen(timestr); } diff --git a/src/pcrepp.hh b/src/pcrepp.hh index 1c8593af..9ec7ce8e 100644 --- a/src/pcrepp.hh +++ b/src/pcrepp.hh @@ -224,6 +224,14 @@ public: if (!this->p_code_extra && errptr) { fprintf(stderr, "pcre_study error: %s\n", errptr); } + if (this->p_code_extra != NULL) { + pcre_extra *extra = this->p_code_extra; + + extra->flags |= (PCRE_EXTRA_MATCH_LIMIT| + PCRE_EXTRA_MATCH_LIMIT_RECURSION); + extra->match_limit = 10000; + extra->match_limit_recursion = 500; + } }; pcrepp(const char *pattern, int options = 0) @@ -244,6 +252,14 @@ public: if (!this->p_code_extra && errptr) { fprintf(stderr, "pcre_study error: %s\n", errptr); } + if (this->p_code_extra != NULL) { + pcre_extra *extra = this->p_code_extra; + + extra->flags |= (PCRE_EXTRA_MATCH_LIMIT| + PCRE_EXTRA_MATCH_LIMIT_RECURSION); + extra->match_limit = 10000; + extra->match_limit_recursion = 500; + } }; pcrepp(const pcrepp &other) @@ -273,7 +289,7 @@ public: pi.pi_offset = pi.pi_next_offset; rc = pcre_exec(this->p_code, - this->p_code_extra, + this->p_code_extra.in(), pi.get_string(), pi.pi_length, pi.pi_offset, diff --git a/src/sql_util.cc b/src/sql_util.cc index d9ac3472..275756dd 100644 --- a/src/sql_util.cc +++ b/src/sql_util.cc @@ -396,7 +396,7 @@ void attach_sqlite_db(sqlite3 *db, const std::string &filename) return; } - if (sqlite3_step(stmt.in()) != SQLITE_OK) { + if (sqlite3_step(stmt.in()) != SQLITE_DONE) { fprintf(stderr, "error: could not execute DB attach statement -- %s\n", sqlite3_errmsg(db)); diff --git a/src/sqlite-extension-func.c b/src/sqlite-extension-func.c index c72ea07a..d9f5c8e6 100644 --- a/src/sqlite-extension-func.c +++ b/src/sqlite-extension-func.c @@ -38,6 +38,7 @@ sqlite_registration_func_t sqlite_registration_funcs[] = { common_extension_functions, + string_extension_functions, network_extension_functions, fs_extension_functions, diff --git a/src/sqlite-extension-func.h b/src/sqlite-extension-func.h index 02128be4..5ecc896f 100644 --- a/src/sqlite-extension-func.h +++ b/src/sqlite-extension-func.h @@ -63,6 +63,9 @@ typedef int (*sqlite_registration_func_t)(const struct FuncDef **basic_funcs, int common_extension_functions(const struct FuncDef **basic_funcs, const struct FuncDefAgg **agg_funcs); +int string_extension_functions(const struct FuncDef **basic_funcs, + const struct FuncDefAgg **agg_funcs); + int network_extension_functions(const struct FuncDef **basic_funcs, const struct FuncDefAgg **agg_funcs); diff --git a/src/statusview_curses.hh b/src/statusview_curses.hh index 06c84f39..b1ea068f 100644 --- a/src/statusview_curses.hh +++ b/src/statusview_curses.hh @@ -162,7 +162,7 @@ public: protected: size_t sf_width; /*< The maximum display width, in chars. */ - size_t sf_min_width; /*< The maximum display width, in chars. */ + size_t sf_min_width; /*< The minimum display width, in chars. */ bool sf_right_justify; bool sf_cylon; size_t sf_cylon_pos; diff --git a/src/string-extension-functions.cc b/src/string-extension-functions.cc new file mode 100644 index 00000000..1e9f7176 --- /dev/null +++ b/src/string-extension-functions.cc @@ -0,0 +1,125 @@ +/* + * Written by Alexey Tourbin . + * + * The author has dedicated the code to the public domain. Anyone is free + * to copy, modify, publish, use, compile, sell, or distribute the original + * code, either in source code form or as a compiled binary, for any purpose, + * commercial or non-commercial, and by any means. + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "pcrepp.hh" + +#include "sqlite-extension-func.h" + +typedef struct { + char *s; + pcre *p; + pcre_extra *e; +} cache_entry; + +#ifndef CACHE_SIZE +#define CACHE_SIZE 16 +#endif + +static +void regexp(sqlite3_context *ctx, int argc, sqlite3_value **argv) +{ + const char *re, *str; + pcre *p; + pcre_extra *e; + + assert(argc == 2); + + re = (const char *) sqlite3_value_text(argv[0]); + if (!re) { + sqlite3_result_error(ctx, "no regexp", -1); + return; + } + + str = (const char *) sqlite3_value_text(argv[1]); + if (!str) { + sqlite3_result_error(ctx, "no string", -1); + return; + } + + /* simple LRU cache */ + { + static cache_entry cache[CACHE_SIZE]; + int i; + int found = 0; + + for (i = 0; i < CACHE_SIZE && cache[i].s; i++) { + if (strcmp(re, cache[i].s) == 0) { + found = 1; + break; + } + } + if (found) { + if (i > 0) { + cache_entry c = cache[i]; + memmove(cache + 1, cache, i * sizeof(cache_entry)); + cache[0] = c; + } + } + else { + cache_entry c; + const char *err; + int pos; + c.p = pcre_compile(re, 0, &err, &pos, NULL); + if (!c.p) { + char *e2 = sqlite3_mprintf("%s: %s (offset %d)", re, err, pos); + sqlite3_result_error(ctx, e2, -1); + sqlite3_free(e2); + return; + } + c.e = pcre_study(c.p, 0, &err); + c.s = strdup(re); + if (!c.s) { + sqlite3_result_error(ctx, "strdup: ENOMEM", -1); + pcre_free(c.p); + pcre_free(c.e); + return; + } + i = CACHE_SIZE - 1; + if (cache[i].s) { + free(cache[i].s); + assert(cache[i].p); + pcre_free(cache[i].p); + pcre_free(cache[i].e); + } + memmove(cache + 1, cache, i * sizeof(cache_entry)); + cache[0] = c; + } + p = cache[0].p; + e = cache[0].e; + } + + { + int rc; + assert(p); + rc = pcre_exec(p, e, str, strlen(str), 0, 0, NULL, 0); + sqlite3_result_int(ctx, rc >= 0); + return; + } +} + +int string_extension_functions(const struct FuncDef **basic_funcs, + const struct FuncDefAgg **agg_funcs) +{ + static const struct FuncDef string_funcs[] = { + { "regexp", 2, 0, SQLITE_UTF8, 0, regexp }, + + { NULL } + }; + + *basic_funcs = string_funcs; + + return SQLITE_OK; +} diff --git a/src/test_override.cc b/src/test_override.cc new file mode 100644 index 00000000..712cb009 --- /dev/null +++ b/src/test_override.cc @@ -0,0 +1,15 @@ + +#include "config.h" + +#include + +time_t time(time_t *loc) +{ + time_t retval = 1370546000; + + if (loc != NULL) { + *loc = retval; + } + + return retval; +} diff --git a/src/textview_curses.cc b/src/textview_curses.cc index 7b500899..7385f264 100644 --- a/src/textview_curses.cc +++ b/src/textview_curses.cc @@ -262,7 +262,7 @@ void textview_curses::listview_value_for_row(const listview_curses &lv, int rc, matches[60]; rc = pcre_exec(iter->second.h_code, - NULL, + iter->second.h_code_extra, str.c_str(), str.size(), off, diff --git a/src/textview_curses.hh b/src/textview_curses.hh index 1dbf4419..819ef31e 100644 --- a/src/textview_curses.hh +++ b/src/textview_curses.hh @@ -137,6 +137,8 @@ public: : h_code(code), h_multiple(multiple) { + const char *errptr; + if (!multiple) { if (role == view_colors::VCR_NONE) { this->h_roles. @@ -146,6 +148,18 @@ public: this->h_roles.push_back(role); } } + this->h_code_extra = pcre_study(this->h_code, 0, &errptr); + if (!this->h_code_extra && errptr) { + fprintf(stderr, "pcre_study error: %s\n", errptr); + } + if (this->h_code_extra != NULL) { + pcre_extra *extra = this->h_code_extra; + + extra->flags |= (PCRE_EXTRA_MATCH_LIMIT| + PCRE_EXTRA_MATCH_LIMIT_RECURSION); + extra->match_limit = 10000; + extra->match_limit_recursion = 500; + } }; view_colors::role_t get_role(unsigned int index) @@ -173,6 +187,7 @@ public: }; pcre * h_code; + pcre_extra *h_code_extra; bool h_multiple; std::vector h_roles; }; @@ -207,7 +222,7 @@ public: int rc, matches[60]; rc = pcre_exec(hl.h_code, - NULL, + hl.h_code_extra, str.c_str(), str.size(), off, diff --git a/test/Makefile.am b/test/Makefile.am index fd559633..e9567eff 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -2,6 +2,10 @@ TESTS_ENVIRONMENT = $(SHELL) $(top_builddir)/TESTS_ENVIRONMENT LOG_COMPILER = $(SHELL) $(top_builddir)/TESTS_ENVIRONMENT +simple-db.db: simple-db.sql + rm -f $@ + $(SQLITE3_CMD) $@ < $< + AM_CPPFLAGS = \ -I$(top_srcdir)/src \ $(SQLITE3_CFLAGS) @@ -140,6 +144,7 @@ slicer_SOURCES = \ scripty_SOURCES = scripty.cc dist_noinst_SCRIPTS = \ + parser_debugger.py \ test_data_parser.sh \ test_grep_proc.sh \ test_line_buffer.sh \ @@ -175,6 +180,7 @@ dist_noinst_DATA = \ logfile_syslog.1 \ logfile_tcsh_history.0 \ logfile_with_a_really_long_name_to_test_a_bug_with_long_names.0 \ + simple-db.sql \ view_colors_output.0 \ vt52_curses_input.0 \ vt52_curses_input.1 \ @@ -184,6 +190,9 @@ dist_noinst_DATA = \ log-samples/sample-70c906b3c1a1cf03f15bde92ee78edfa6f9b7960.txt \ log-samples/sample-ad31f12d2adabd07e3ddda3ad5b0dbf6b49c4c99.txt +nodist_noinst_DATA = \ + simple-db.db + TESTS = test_bookmarks \ test_auto_fd \ test_auto_mem \ @@ -203,6 +212,8 @@ TESTS = test_bookmarks \ DISTCLEANFILES = \ *.dat \ + *.db \ + *.dpt \ *.diff \ *.index \ *.tmp diff --git a/test/Makefile.in b/test/Makefile.in index d87042e3..d874e281 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -260,7 +260,7 @@ am__can_run_installinfo = \ n|no|NO) false;; \ *) (install-info --version) >/dev/null 2>&1;; \ esac -DATA = $(dist_noinst_DATA) +DATA = $(dist_noinst_DATA) $(nodist_noinst_DATA) am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) # Read a list of newline-separated strings from the standard input, # and print each of them once, without duplicates. Input order is @@ -540,6 +540,7 @@ READLINE_LIBS = @READLINE_LIBS@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ SQLITE3_CFLAGS = @SQLITE3_CFLAGS@ +SQLITE3_CMD = @SQLITE3_CMD@ SQLITE3_LDFLAGS = @SQLITE3_LDFLAGS@ SQLITE3_LIBS = @SQLITE3_LIBS@ SQLITE3_VERSION = @SQLITE3_VERSION@ @@ -708,6 +709,7 @@ slicer_SOURCES = \ scripty_SOURCES = scripty.cc dist_noinst_SCRIPTS = \ + parser_debugger.py \ test_data_parser.sh \ test_grep_proc.sh \ test_line_buffer.sh \ @@ -743,6 +745,7 @@ dist_noinst_DATA = \ logfile_syslog.1 \ logfile_tcsh_history.0 \ logfile_with_a_really_long_name_to_test_a_bug_with_long_names.0 \ + simple-db.sql \ view_colors_output.0 \ vt52_curses_input.0 \ vt52_curses_input.1 \ @@ -752,8 +755,13 @@ dist_noinst_DATA = \ log-samples/sample-70c906b3c1a1cf03f15bde92ee78edfa6f9b7960.txt \ log-samples/sample-ad31f12d2adabd07e3ddda3ad5b0dbf6b49c4c99.txt +nodist_noinst_DATA = \ + simple-db.db + DISTCLEANFILES = \ *.dat \ + *.db \ + *.dpt \ *.diff \ *.index \ *.tmp @@ -1617,6 +1625,10 @@ uninstall-am: recheck tags tags-am uninstall uninstall-am +simple-db.db: simple-db.sql + rm -f $@ + $(SQLITE3_CMD) $@ < $< + # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. .NOEXPORT: diff --git a/test/datafile_simple.8 b/test/datafile_simple.8 index 9d15d931..e76d5c7f 100644 --- a/test/datafile_simple.8 +++ b/test/datafile_simple.8 @@ -23,11 +23,9 @@ pair 160:165 key 168:168 ^ key 168:168 ^ num 168:174 ^----^ 100003 - val 168:174 ^----^ 100003 pair 168:174 ^----^ 100003 key 175:175 ^ num 175:176 ^ 1 - val 175:176 ^ 1 pair 175:176 ^ 1 grp 168:176 ^------^ 100003,1 pair 168:176 ^------^ 100003,1 diff --git a/test/drive_data_scanner.cc b/test/drive_data_scanner.cc index b33582cf..3e31a638 100644 --- a/test/drive_data_scanner.cc +++ b/test/drive_data_scanner.cc @@ -133,6 +133,8 @@ int main(int argc, char *argv[]) body = find_string_attr_range(sa, "body"); } + data_parser::TRACE_FILE = fopen("scanned.dpt", "w"); + data_scanner ds(sub_line, body.lr_start, sub_line.length()); data_parser dp(&ds); @@ -161,6 +163,8 @@ int main(int argc, char *argv[]) retval = EXIT_FAILURE; } } + + fclose(data_parser::TRACE_FILE); } } } diff --git a/test/log-samples/sample-9cf7fbb3546c676c686fac0ed096d026f46c875f.txt b/test/log-samples/sample-9cf7fbb3546c676c686fac0ed096d026f46c875f.txt new file mode 100644 index 00000000..47e3e3a8 --- /dev/null +++ b/test/log-samples/sample-9cf7fbb3546c676c686fac0ed096d026f46c875f.txt @@ -0,0 +1,25 @@ + 2013-06-05T14:20:24 DEBUG cc2.main CC - 4672610200547811617359537811896212984085567168.114723023 Json_Reader - Doing prepare for resource name "Json_Reader", component "com.json.components.JSONReader" + key 26:26 ^ + sym 26:34 ^------^ cc2.main +pair 26:34 ^------^ cc2.main + key 35:35 ^ + sym 35:37 ^^ CC +pair 35:37 ^^ CC + key 38:38 ^ + sym 38:39 ^ - +pair 38:39 ^ - + key 40:40 ^ + num 40:96 ^------------------------------------------------------^ 4672610200547811617359537811896212984085567168.114723023 +pair 40:96 ^------------------------------------------------------^ 4672610200547811617359537811896212984085567168.114723023 + key 97:97 ^ + sym 97:108 ^---------^ Json_Reader +pair 97:108 ^---------^ Json_Reader + key 109:109 ^ + sym 109:110 ^ - +pair 109:110 ^ - + key 144:144 ^ +quot 144:155 ^---------^ Json_Reader +pair 144:155 ^---------^ Json_Reader + key 169:169 ^ +quot 169:199 ^----------------------------^ com.json.components.JSONReader +pair 169:199 ^----------------------------^ com.json.components.JSONReader diff --git a/test/parser_debugger.py b/test/parser_debugger.py new file mode 100755 index 00000000..6bfb6a01 --- /dev/null +++ b/test/parser_debugger.py @@ -0,0 +1,204 @@ +#! /usr/bin/env python + +# Copyright (c) 2013, Timothy Stack +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# * Neither the name of Timothy Stack nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import json +import string +import readline +import itertools +import collections + +TEST_DIR = os.path.dirname(__file__) +ROOT_DIR = os.path.dirname(TEST_DIR) +SRC_DIR = os.path.join(ROOT_DIR, "src") + +addr_to_name = {} +name_to_addr = {} +element_lists = collections.defaultdict(list) +breakpoints = set() + +def completer(text, state): + options = [x for x in itertools.chain(name_to_addr, + element_lists, + breakpoints) + if x.startswith(text)] + try: + return options[state] + except IndexError: + return None + +readline.set_completer(completer) + +if 'libedit' in readline.__doc__: + readline.parse_and_bind('bind ^I rl_complete') +else: + readline.parse_and_bind('tab: complete') + +input_line = '' +ops = [] +for line in open("scanned.dpt"): + if line.startswith("input "): + input_line = line[6:-1] + else: + ops.append(map(string.strip, line.split())) + +def getstr(capture): + start, end = capture.split(':') + return input_line[int(start):int(end)] + +def printlist(name_or_addr): + if name_or_addr in name_to_addr: + print "(%s) %s" % (name_or_addr, element_lists[name_to_addr[name_or_addr]]) + elif name_or_addr in element_lists: + print "(%s) %s" % (addr_to_name.get(name_or_addr, name_or_addr), + element_lists[name_or_addr]) + else: + print "error: unknown list --", name_or_addr + +def handleop(fields): + addr = fields[0] + loc = fields[1].split(':') + method_name = fields[2] + method_args = fields[3:] + + if addr == '0x0': + el = None + else: + el = element_lists[addr] + + if method_name == 'element_list_t': + addr_to_name[addr] = method_args[0] + name_to_addr[method_args[0]] = addr + elif method_name == '~element_list_t': + pass + elif method_name == 'push_back': + el.append((method_args[0], getstr(method_args[1]))) + elif method_name == 'pop_front': + el.pop(0) + elif method_name == 'pop_back': + el.pop() + elif method_name == 'splice': + pos = int(method_args[0]) + other = element_lists[method_args[1]] + start, from_end = map(int, method_args[2].split(':')) + end = len(other) - from_end + sub_list = other[start:end] + del other[start:end] + el[pos:pos] = sub_list + elif method_name == 'point': + breakpoints.add(method_args[0]) + else: + print "Unhandled method: ", method_name + +def playupto(length): + addr_to_name.clear() + name_to_addr.clear() + element_lists.clear() + for index in range(length): + handleop(ops[index]) + +def find_prev_point(start, name): + orig_start = start + while start > 0: + start -= 1; + fields = ops[start] + if fields[2] != 'point': + continue + if not name or fields[3] == name: + return start + 1 + return orig_start + 1 + +def find_next_point(start, name): + orig_start = start + while start < len(ops): + start += 1; + fields = ops[start] + if fields[2] != 'point': + continue + if not name or fields[3] == name: + return start + 1 + return orig_start + 1 + +index = len(ops) +last_cmd = [''] +watch_list = set() +while True: + playupto(index) + + if index == 0: + print "init" + else: + print "#%s %s" % (index -1, ops[index - 1]) + + for list_name in watch_list: + printlist(list_name) + + try: + cmd = raw_input("> ").split() + except EOFError: + print + break + + if not cmd or cmd[0] == '': + cmd = last_cmd + + if not cmd or cmd[0] == '': + pass + elif cmd[0] == 'q': + break + elif cmd[0] == 'n': + if index < len(ops): + index += 1 + elif cmd[0] == 'r': + if index > 0: + index -= 1 + elif cmd[0] == 'b': + if len(cmd) == 1: + cmd.append('') + + index = find_prev_point(index - 1, cmd[1]) + elif cmd[0] == 'c': + if len(cmd) == 1: + cmd.append('') + index = find_next_point(index - 1, cmd[1]) + elif cmd[0] == 'p': + if len(cmd) > 1: + printlist(cmd[1]) + else: + print input_line + for addr in element_lists: + printlist(addr) + elif cmd[0] == 'w': + watch_list.add(cmd[1]) + elif cmd[0] == 'u': + watch_list.remove(cmd[1]) + else: + print "error: unknown command --", cmd + + last_cmd = cmd diff --git a/test/simple-db.sql b/test/simple-db.sql new file mode 100644 index 00000000..a5c30c7b --- /dev/null +++ b/test/simple-db.sql @@ -0,0 +1,12 @@ + +CREATE TABLE IF NOT EXISTS person ( + id integer PRIMARY KEY, + first_name text, + last_name text, + age integer +); + +INSERT INTO person (id, first_name, last_name, age) + VALUES (0, "Phil", "Myman", 30); +INSERT INTO person (id, first_name, last_name, age) + VALUES (1, "Lem", "Hewitt", 35); diff --git a/test/sql.0.in b/test/sql.0.in new file mode 100644 index 00000000..e72339bc --- /dev/null +++ b/test/sql.0.in @@ -0,0 +1,6 @@ +sleep 0.913123 +write 3b +sleep 0.822303 +write 73656c656374202a2066726f6d20706572736f6e206f726465722062792061676520646573633b0d +sleep 9.068423 +write 71 diff --git a/test/sql.0.out b/test/sql.0.out new file mode 100644 index 00000000..c0f0bbbb --- /dev/null +++ b/test/sql.0.out @@ -0,0 +1,7 @@ +read 1b29301b371b5b3f3437681b5b313b3234721b5b6d1b5b346c1b5b3f313030306c1b5b3f313030326c1b5b3f31681b3d1b5b6d1b5b6d1b5b33376d1b5b34306d1b5b313b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b323b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b333b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b343b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b353b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b363b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b373b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b383b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b393b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b31303b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b31313b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b31323b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b31333b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b31343b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b31353b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b31363b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b31373b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b31383b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b31393b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b32303b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b32313b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b32323b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b32333b314820202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b32343b3148202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200820081b5b3468651b5b346c1b5b481b5b33306d1b5b34376d20546875204a756e2030362031323a31333a32302050445420202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b32333b31482020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b33316d1b5b34376d202020201b5b33306d1b5b34376d3f3a566965772048656c702020201b5b316d201b5b6d1b5b33306d1b5b34376d202020202020202020202020202020201b5b32343b3332481b5b6d1b5b6d1b5b33376d1b5b34306d652f453a204d6f766520666f72776172642f6261636b77617264207468726f756768206572726f72206d6573736167650873081b5b3468651b5b346c0d1b5b411b5b431b5b33306d1b5b34376d4c301b5b32333b31354830251b5b32333b34334830570d1b5b31421b5b6d1b5b6d1b5b33376d1b5b34306d +# write 3b +read 1b5b32343b3332482020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200820081b5b3468201b5b346c0d1b5b373943081b5b6d1b5b33376d1b5b34306d1b5b376d20081b5b34681b5b6d1b5b6d1b5b33376d1b5b34306d201b5b346c0d3b +# write 73656c656374202a2066726f6d20706572736f6e206f726465722062792061676520646573633b0d +read 73656c656374202a2066726f6d20706572736f6e206f726465722062792061676520646573630d1b5b3232411b5b4d1b5b323242202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200820081b5b3468201b5b346c1b5b32333b3430483b1b5b333943201b5b32343b383048081b5b6d1b5b33376d1b5b34306d1b5b376d20081b5b34681b5b6d1b5b6d1b5b33376d1b5b34306d201b5b346c0d1b5b411b5b3231411b5b6d1b5b33376d1b5b34306d1b5b346d69642066697273745f6e616d65206c6173745f6e616d65201b5b33346d1b5b34306d616765201b5b6d1b5b33376d1b5b34306d1b5b316d2020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d1b5b376d201b5b333b31481b5b6d1b5b33376d1b5b34306d202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d1b5b376d201b5b343b31481b5b33346d1b5b34376d20311b5b6d1b5b6d1b5b33376d1b5b34306d0e780f1b5b33346d1b5b34376d1b5b376d4c656d202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d0e780f1b5b33346d1b5b34376d1b5b376d4865776974742020201b5b6d1b5b6d1b5b33376d1b5b34306d0e780f1b5b33346d1b5b34376d1b5b376d203335202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b33376d1b5b34306d201b5b353b31481b5b33346d1b5b34376d201b5b6d1b5b33376d1b5b34306d301b5b6d1b5b6d1b5b33376d1b5b34306d0e780f1b5b33376d1b5b34306d5068696c2020202020201b5b6d1b5b6d1b5b33376d1b5b34306d0e780f1b5b33376d1b5b34306d4d796d616e202020201b5b6d1b5b6d1b5b33376d1b5b34306d0e780f1b5b33376d1b5b34306d203330202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d1b5b376d201b5b363b31481b5b6d1b5b33376d1b5b34306d202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d1b5b376d201b5b373b31481b5b6d1b5b33376d1b5b34306d202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d1b5b376d201b5b383b31481b5b6d1b5b33376d1b5b34306d202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d1b5b376d201b5b393b31481b5b6d1b5b33376d1b5b34306d202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d1b5b376d201b5b31303b31481b5b6d1b5b33376d1b5b34306d202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d1b5b376d201b5b31313b31481b5b6d1b5b33376d1b5b34306d202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d1b5b376d201b5b31323b31481b5b6d1b5b33376d1b5b34306d202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d1b5b376d201b5b31333b31481b5b6d1b5b33376d1b5b34306d202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b6d1b5b33376d1b5b34306d1b5b376d201b5b31343b383048201b5b31353b383048201b5b31363b383048201b5b31373b383048201b5b31383b383048201b5b31393b383048201b5b32303b383048201b5b32313b383048201b5b32323b31481b5b6d1b5b6d1b5b33376d1b5b34306d202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020201b5b6d1b5b33376d1b5b34306d1b5b376d201b5b32333b31481b5b6d1b5b33306d1b5b34376d204c302020202020202020203130302520202020202020202020202020202020202020202020202020202020201b5b33316d1b5b34376d202020201b5b33306d1b5b34376d3f3a566965772048656c702020201b5b316d201b5b6d1b5b33306d1b5b34376d202020202020202020202020202020201b5b32343b31481b5b6d1b5b6d1b5b33376d1b5b34306d3220726f77287329206d6174636865641b5b3633430820081b5b3468201b5b346c0d +# write 71 +read 1b5b3f313030306c1b5b3f313030326c1b5b6d1b5b4b1b5b32343b31481b5b324a1b5b3f34376c1b380d1b5b3f316c1b3e