tp: move tokenizer into its own folder
In prep for LALR based parsing of PerfettoSQL.
Change-Id: I7113adb1e8eda9b66d12cba95e222a4cdf91d814
diff --git a/Android.bp b/Android.bp
index a45d4ff..2d05823 100644
--- a/Android.bp
+++ b/Android.bp
@@ -2512,6 +2512,7 @@
":perfetto_src_trace_processor_perfetto_sql_parser_parser",
":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+ ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer",
":perfetto_src_trace_processor_sorter_sorter",
":perfetto_src_trace_processor_sqlite_bindings_bindings",
":perfetto_src_trace_processor_sqlite_sqlite",
@@ -13430,6 +13431,14 @@
name: "perfetto_src_trace_processor_perfetto_sql_parser_test_utils",
}
+// GN: //src/trace_processor/perfetto_sql/parser:unittests
+filegroup {
+ name: "perfetto_src_trace_processor_perfetto_sql_parser_unittests",
+ srcs: [
+ "src/trace_processor/perfetto_sql/parser/perfetto_sql_parser_unittest.cc",
+ ],
+}
+
// GN: //src/trace_processor/perfetto_sql/preprocessor:grammar
filegroup {
name: "perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
@@ -13602,6 +13611,22 @@
],
}
+// GN: //src/trace_processor/perfetto_sql/tokenizer:tokenizer
+filegroup {
+ name: "perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer",
+ srcs: [
+ "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc",
+ ],
+}
+
+// GN: //src/trace_processor/perfetto_sql/tokenizer:unittests
+filegroup {
+ name: "perfetto_src_trace_processor_perfetto_sql_tokenizer_unittests",
+ srcs: [
+ "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer_unittest.cc",
+ ],
+}
+
// GN: //src/trace_processor/rpc:httpd
filegroup {
name: "perfetto_src_trace_processor_rpc_httpd",
@@ -13666,7 +13691,6 @@
"src/trace_processor/sqlite/sql_source.cc",
"src/trace_processor/sqlite/sql_stats_table.cc",
"src/trace_processor/sqlite/sqlite_engine.cc",
- "src/trace_processor/sqlite/sqlite_tokenizer.cc",
"src/trace_processor/sqlite/sqlite_utils.cc",
"src/trace_processor/sqlite/stats_table.cc",
],
@@ -13678,7 +13702,6 @@
srcs: [
"src/trace_processor/sqlite/db_sqlite_table_unittest.cc",
"src/trace_processor/sqlite/sql_source_unittest.cc",
- "src/trace_processor/sqlite/sqlite_tokenizer_unittest.cc",
"src/trace_processor/sqlite/sqlite_utils_unittest.cc",
],
}
@@ -15570,9 +15593,12 @@
":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
":perfetto_src_trace_processor_perfetto_sql_parser_parser",
":perfetto_src_trace_processor_perfetto_sql_parser_test_utils",
+ ":perfetto_src_trace_processor_perfetto_sql_parser_unittests",
":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
":perfetto_src_trace_processor_perfetto_sql_preprocessor_unittests",
+ ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer",
+ ":perfetto_src_trace_processor_perfetto_sql_tokenizer_unittests",
":perfetto_src_trace_processor_rpc_rpc",
":perfetto_src_trace_processor_rpc_unittests",
":perfetto_src_trace_processor_sorter_sorter",
@@ -16621,6 +16647,7 @@
":perfetto_src_trace_processor_perfetto_sql_parser_parser",
":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+ ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer",
":perfetto_src_trace_processor_rpc_httpd",
":perfetto_src_trace_processor_rpc_rpc",
":perfetto_src_trace_processor_rpc_stdiod",
@@ -17035,6 +17062,7 @@
":perfetto_src_trace_processor_perfetto_sql_parser_parser",
":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+ ":perfetto_src_trace_processor_perfetto_sql_tokenizer_tokenizer",
":perfetto_src_trace_processor_sorter_sorter",
":perfetto_src_trace_processor_sqlite_bindings_bindings",
":perfetto_src_trace_processor_sqlite_sqlite",
diff --git a/BUILD b/BUILD
index 6ce4ad9..1d59aae 100644
--- a/BUILD
+++ b/BUILD
@@ -270,6 +270,7 @@
":src_trace_processor_perfetto_sql_parser_parser",
":src_trace_processor_perfetto_sql_preprocessor_grammar",
":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+ ":src_trace_processor_perfetto_sql_tokenizer_tokenizer",
":src_trace_processor_rpc_rpc",
":src_trace_processor_sorter_sorter",
":src_trace_processor_sqlite_bindings_bindings",
@@ -3013,6 +3014,15 @@
namespace = "stdlib",
)
+# GN target: //src/trace_processor/perfetto_sql/tokenizer:tokenizer
+perfetto_filegroup(
+ name = "src_trace_processor_perfetto_sql_tokenizer_tokenizer",
+ srcs = [
+ "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc",
+ "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h",
+ ],
+)
+
# GN target: //src/trace_processor/rpc:httpd
perfetto_filegroup(
name = "src_trace_processor_rpc_httpd",
@@ -3083,8 +3093,6 @@
"src/trace_processor/sqlite/sql_stats_table.h",
"src/trace_processor/sqlite/sqlite_engine.cc",
"src/trace_processor/sqlite/sqlite_engine.h",
- "src/trace_processor/sqlite/sqlite_tokenizer.cc",
- "src/trace_processor/sqlite/sqlite_tokenizer.h",
"src/trace_processor/sqlite/sqlite_utils.cc",
"src/trace_processor/sqlite/sqlite_utils.h",
"src/trace_processor/sqlite/stats_table.cc",
@@ -6329,6 +6337,7 @@
":src_trace_processor_perfetto_sql_parser_parser",
":src_trace_processor_perfetto_sql_preprocessor_grammar",
":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+ ":src_trace_processor_perfetto_sql_tokenizer_tokenizer",
":src_trace_processor_sorter_sorter",
":src_trace_processor_sqlite_bindings_bindings",
":src_trace_processor_sqlite_sqlite",
@@ -6523,6 +6532,7 @@
":src_trace_processor_perfetto_sql_parser_parser",
":src_trace_processor_perfetto_sql_preprocessor_grammar",
":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+ ":src_trace_processor_perfetto_sql_tokenizer_tokenizer",
":src_trace_processor_rpc_httpd",
":src_trace_processor_rpc_rpc",
":src_trace_processor_rpc_stdiod",
@@ -6774,6 +6784,7 @@
":src_trace_processor_perfetto_sql_parser_parser",
":src_trace_processor_perfetto_sql_preprocessor_grammar",
":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
+ ":src_trace_processor_perfetto_sql_tokenizer_tokenizer",
":src_trace_processor_sorter_sorter",
":src_trace_processor_sqlite_bindings_bindings",
":src_trace_processor_sqlite_sqlite",
diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn
index 4539b5a..3834b57 100644
--- a/src/trace_processor/BUILD.gn
+++ b/src/trace_processor/BUILD.gn
@@ -327,7 +327,9 @@
"perfetto_sql/intrinsics/functions:unittests",
"perfetto_sql/intrinsics/operators:unittests",
"perfetto_sql/intrinsics/table_functions:unittests",
+ "perfetto_sql/parser:unittests",
"perfetto_sql/preprocessor:unittests",
+ "perfetto_sql/tokenizer:unittests",
"sqlite:unittests",
]
}
diff --git a/src/trace_processor/perfetto_sql/engine/BUILD.gn b/src/trace_processor/perfetto_sql/engine/BUILD.gn
index b257681..adb6b6d 100644
--- a/src/trace_processor/perfetto_sql/engine/BUILD.gn
+++ b/src/trace_processor/perfetto_sql/engine/BUILD.gn
@@ -46,6 +46,7 @@
"../../util:stdlib",
"../parser",
"../preprocessor",
+ "../tokenizer",
]
}
diff --git a/src/trace_processor/perfetto_sql/parser/BUILD.gn b/src/trace_processor/perfetto_sql/parser/BUILD.gn
index 831e3c8..f31cae2 100644
--- a/src/trace_processor/perfetto_sql/parser/BUILD.gn
+++ b/src/trace_processor/perfetto_sql/parser/BUILD.gn
@@ -32,6 +32,7 @@
"../../util",
"../../util:sql_argument",
"../preprocessor",
+ "../tokenizer",
]
}
diff --git a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc
index 8f30a94..4af83ea 100644
--- a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc
+++ b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc
@@ -30,8 +30,8 @@
#include "perfetto/ext/base/string_utils.h"
#include "src/trace_processor/perfetto_sql/parser/function_util.h"
#include "src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
#include "src/trace_processor/sqlite/sql_source.h"
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
#include "src/trace_processor/util/sql_argument.h"
namespace perfetto {
diff --git a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h
index 3252b9a..8321bb6 100644
--- a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h
+++ b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h
@@ -26,8 +26,8 @@
#include "perfetto/ext/base/flat_hash_map.h"
#include "src/trace_processor/perfetto_sql/parser/function_util.h"
#include "src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
#include "src/trace_processor/sqlite/sql_source.h"
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
#include "src/trace_processor/util/sql_argument.h"
namespace perfetto {
diff --git a/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn b/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn
index 7b3e7bc..7314069 100644
--- a/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn
+++ b/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn
@@ -27,6 +27,7 @@
"../../../base",
"../../sqlite",
"../../util",
+ "../tokenizer",
]
}
diff --git a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc
index 31a2b0c..e664c25 100644
--- a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc
+++ b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc
@@ -36,8 +36,8 @@
#include "perfetto/ext/base/flat_hash_map.h"
#include "perfetto/ext/base/string_utils.h"
#include "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar_interface.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
#include "src/trace_processor/sqlite/sql_source.h"
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
namespace perfetto::trace_processor {
namespace {
diff --git a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h
index 669d964..fdbee3e 100644
--- a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h
+++ b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h
@@ -24,8 +24,8 @@
#include "perfetto/base/status.h"
#include "perfetto/ext/base/flat_hash_map.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
#include "src/trace_processor/sqlite/sql_source.h"
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
namespace perfetto::trace_processor {
diff --git a/src/trace_processor/perfetto_sql/tokenizer/BUILD.gn b/src/trace_processor/perfetto_sql/tokenizer/BUILD.gn
new file mode 100644
index 0000000..1fbbd8f
--- /dev/null
+++ b/src/trace_processor/perfetto_sql/tokenizer/BUILD.gn
@@ -0,0 +1,45 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("../../../../gn/test.gni")
+
+assert(enable_perfetto_trace_processor_sqlite)
+
+source_set("tokenizer") {
+ sources = [
+ "sqlite_tokenizer.cc",
+ "sqlite_tokenizer.h",
+ ]
+ deps = [
+ "../../../../gn:default_deps",
+ "../../../../gn:sqlite",
+ "../../../base",
+ "../../sqlite",
+ ]
+}
+
+perfetto_unittest_source_set("unittests") {
+ testonly = true
+ sources = [ "sqlite_tokenizer_unittest.cc" ]
+ deps = [
+ ":tokenizer",
+ "../../../../gn:default_deps",
+ "../../../../gn:gtest_and_gmock",
+ "../../../../gn:sqlite",
+ "../../../base",
+ "../../../base:test_support",
+ "../../sqlite",
+ "../../util",
+ ]
+}
diff --git a/src/trace_processor/sqlite/sqlite_tokenizer.cc b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc
similarity index 99%
rename from src/trace_processor/sqlite/sqlite_tokenizer.cc
rename to src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc
index 972857b..fb1b3f2 100644
--- a/src/trace_processor/sqlite/sqlite_tokenizer.cc
+++ b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.cc
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
#include <ctype.h>
#include <sqlite3.h>
diff --git a/src/trace_processor/sqlite/sqlite_tokenizer.h b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h
similarity index 95%
rename from src/trace_processor/sqlite/sqlite_tokenizer.h
rename to src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h
index c496b4c..64019fb 100644
--- a/src/trace_processor/sqlite/sqlite_tokenizer.h
+++ b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h
@@ -14,8 +14,8 @@
* limitations under the License.
*/
-#ifndef SRC_TRACE_PROCESSOR_SQLITE_SQLITE_TOKENIZER_H_
-#define SRC_TRACE_PROCESSOR_SQLITE_SQLITE_TOKENIZER_H_
+#ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_
+#define SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_
#include <optional>
#include <string_view>
@@ -166,4 +166,4 @@
} // namespace trace_processor
} // namespace perfetto
-#endif // SRC_TRACE_PROCESSOR_SQLITE_SQLITE_TOKENIZER_H_
+#endif // SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_
diff --git a/src/trace_processor/sqlite/sqlite_tokenizer_unittest.cc b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer_unittest.cc
similarity index 97%
rename from src/trace_processor/sqlite/sqlite_tokenizer_unittest.cc
rename to src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer_unittest.cc
index e7aa4cf..9fc950b 100644
--- a/src/trace_processor/sqlite/sqlite_tokenizer_unittest.cc
+++ b/src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer_unittest.cc
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-#include "src/trace_processor/sqlite/sqlite_tokenizer.h"
+#include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
#include <vector>
#include "perfetto/base/logging.h"
diff --git a/src/trace_processor/sqlite/BUILD.gn b/src/trace_processor/sqlite/BUILD.gn
index fda30ba..06150e0 100644
--- a/src/trace_processor/sqlite/BUILD.gn
+++ b/src/trace_processor/sqlite/BUILD.gn
@@ -28,8 +28,6 @@
"sql_stats_table.h",
"sqlite_engine.cc",
"sqlite_engine.h",
- "sqlite_tokenizer.cc",
- "sqlite_tokenizer.h",
"sqlite_utils.cc",
"sqlite_utils.h",
"stats_table.cc",
@@ -64,7 +62,6 @@
sources = [
"db_sqlite_table_unittest.cc",
"sql_source_unittest.cc",
- "sqlite_tokenizer_unittest.cc",
"sqlite_utils_unittest.cc",
]
deps = [
