commit d500f691800eb3566fdb335a4491b2694f90f8e8
author Lalit Maganti <lalitm@google.com> Wed May 24 21:02:51 2023 +0100
committer Lalit Maganti <lalitm@google.com> Wed May 24 21:02:51 2023 +0100
tree 49fc2a162ab9bf94bb3c4b879e9248895999c48c
parent 1aa1c7184de1df1938de932a806719eb883fbae1

tp: introduce tokenizer for SQLite statements

This CL adds an "SqliteTokenizer" class which allows tokenizing SQLite
SQL statements into tokens. The main function is just a minified copy
of the |sqlite3GetToken| function (i.e. with keyword specific parsing
removed to reduce the numbeer of constants copied).

The class layered on top of this function allows for C++ friendly use.

Change-Id: Icc10bf0aea60b16c013a38c8afa29bf4d41ebb64

BUILD

diff --git a/BUILD b/BUILD
index a62edb8..f819fce 100644
--- a/BUILD
+++ b/BUILD
@@ -2174,6 +2174,8 @@
         "src/trace_processor/sqlite/sqlite_engine.h",
         "src/trace_processor/sqlite/sqlite_table.cc",
         "src/trace_processor/sqlite/sqlite_table.h",
+        "src/trace_processor/sqlite/sqlite_tokenizer.cc",
+        "src/trace_processor/sqlite/sqlite_tokenizer.h",
         "src/trace_processor/sqlite/sqlite_utils.cc",
         "src/trace_processor/sqlite/sqlite_utils.h",
         "src/trace_processor/sqlite/stats_table.cc",