diff --git a/Android.bp b/Android.bp
index 1b7b970..76c8a42 100644
--- a/Android.bp
+++ b/Android.bp
@@ -2509,6 +2509,7 @@
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_table_functions_interface",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_table_functions_table_functions",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
+        ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
         ":perfetto_src_trace_processor_sorter_sorter",
         ":perfetto_src_trace_processor_sqlite_bindings_bindings",
@@ -13422,6 +13423,14 @@
     name: "perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
 }
 
+// GN: //src/trace_processor/perfetto_sql/preprocessor:grammar
+filegroup {
+    name: "perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
+    srcs: [
+        "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.c",
+    ],
+}
+
 // GN: //src/trace_processor/perfetto_sql/preprocessor:preprocessor
 filegroup {
     name: "perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
@@ -13533,8 +13542,6 @@
         "src/trace_processor/perfetto_sql/stdlib/linux/memory/process.sql",
         "src/trace_processor/perfetto_sql/stdlib/linux/perf/samples.sql",
         "src/trace_processor/perfetto_sql/stdlib/linux/threads.sql",
-        "src/trace_processor/perfetto_sql/stdlib/metasql/column_list.sql",
-        "src/trace_processor/perfetto_sql/stdlib/metasql/table_list.sql",
         "src/trace_processor/perfetto_sql/stdlib/pkvm/hypervisor.sql",
         "src/trace_processor/perfetto_sql/stdlib/prelude/casts.sql",
         "src/trace_processor/perfetto_sql/stdlib/prelude/slices.sql",
@@ -15555,6 +15562,7 @@
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_table_functions_table_functions",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_table_functions_unittests",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
+        ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_unittests",
         ":perfetto_src_trace_processor_rpc_rpc",
@@ -16602,6 +16610,7 @@
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_table_functions_interface",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_table_functions_table_functions",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
+        ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
         ":perfetto_src_trace_processor_rpc_httpd",
         ":perfetto_src_trace_processor_rpc_rpc",
@@ -17014,6 +17023,7 @@
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_table_functions_interface",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_table_functions_table_functions",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
+        ":perfetto_src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":perfetto_src_trace_processor_perfetto_sql_preprocessor_preprocessor",
         ":perfetto_src_trace_processor_sorter_sorter",
         ":perfetto_src_trace_processor_sqlite_bindings_bindings",
diff --git a/BUILD b/BUILD
index f29c84a..b6937e7 100644
--- a/BUILD
+++ b/BUILD
@@ -267,6 +267,7 @@
         ":src_trace_processor_perfetto_sql_intrinsics_table_functions_table_functions",
         ":src_trace_processor_perfetto_sql_intrinsics_table_functions_tables",
         ":src_trace_processor_perfetto_sql_intrinsics_types_types",
+        ":src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
         ":src_trace_processor_rpc_rpc",
         ":src_trace_processor_sorter_sorter",
@@ -2578,6 +2579,16 @@
     ],
 )
 
+# GN target: //src/trace_processor/perfetto_sql/preprocessor:grammar
+perfetto_filegroup(
+    name = "src_trace_processor_perfetto_sql_preprocessor_grammar",
+    srcs = [
+        "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.c",
+        "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.h",
+        "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar_interface.h",
+    ],
+)
+
 # GN target: //src/trace_processor/perfetto_sql/preprocessor:preprocessor
 perfetto_filegroup(
     name = "src_trace_processor_perfetto_sql_preprocessor_preprocessor",
@@ -2837,15 +2848,6 @@
     ],
 )
 
-# GN target: //src/trace_processor/perfetto_sql/stdlib/metasql:metasql
-perfetto_filegroup(
-    name = "src_trace_processor_perfetto_sql_stdlib_metasql_metasql",
-    srcs = [
-        "src/trace_processor/perfetto_sql/stdlib/metasql/column_list.sql",
-        "src/trace_processor/perfetto_sql/stdlib/metasql/table_list.sql",
-    ],
-)
-
 # GN target: //src/trace_processor/perfetto_sql/stdlib/pkvm:pkvm
 perfetto_filegroup(
     name = "src_trace_processor_perfetto_sql_stdlib_pkvm_pkvm",
@@ -2986,7 +2988,6 @@
         ":src_trace_processor_perfetto_sql_stdlib_linux_linux",
         ":src_trace_processor_perfetto_sql_stdlib_linux_memory_memory",
         ":src_trace_processor_perfetto_sql_stdlib_linux_perf_perf",
-        ":src_trace_processor_perfetto_sql_stdlib_metasql_metasql",
         ":src_trace_processor_perfetto_sql_stdlib_pkvm_pkvm",
         ":src_trace_processor_perfetto_sql_stdlib_prelude_prelude",
         ":src_trace_processor_perfetto_sql_stdlib_sched_sched",
@@ -6317,6 +6318,7 @@
         ":src_trace_processor_perfetto_sql_intrinsics_table_functions_table_functions",
         ":src_trace_processor_perfetto_sql_intrinsics_table_functions_tables",
         ":src_trace_processor_perfetto_sql_intrinsics_types_types",
+        ":src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
         ":src_trace_processor_sorter_sorter",
         ":src_trace_processor_sqlite_bindings_bindings",
@@ -6509,6 +6511,7 @@
         ":src_trace_processor_perfetto_sql_intrinsics_table_functions_table_functions",
         ":src_trace_processor_perfetto_sql_intrinsics_table_functions_tables",
         ":src_trace_processor_perfetto_sql_intrinsics_types_types",
+        ":src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
         ":src_trace_processor_rpc_httpd",
         ":src_trace_processor_rpc_rpc",
@@ -6758,6 +6761,7 @@
         ":src_trace_processor_perfetto_sql_intrinsics_table_functions_table_functions",
         ":src_trace_processor_perfetto_sql_intrinsics_table_functions_tables",
         ":src_trace_processor_perfetto_sql_intrinsics_types_types",
+        ":src_trace_processor_perfetto_sql_preprocessor_grammar",
         ":src_trace_processor_perfetto_sql_preprocessor_preprocessor",
         ":src_trace_processor_sorter_sorter",
         ":src_trace_processor_sqlite_bindings_bindings",
diff --git a/LICENSE b/LICENSE
index cd2eba5..bbcb67c 100644
--- a/LICENSE
+++ b/LICENSE
@@ -175,6 +175,10 @@
 
    END OF TERMS AND CONDITIONS
 
+------------------
+
+Files: * except those files noted below
+
    Copyright (c) 2017, The Android Open Source Project
 
    Licensed under the Apache License, Version 2.0 (the "License");
@@ -187,6 +191,10 @@
    limitations under the License.
 
 
+------------------
+
+Files: src/trace_processor/perfetto_sql/stdlib/chromium/*, protos/third_party/chromium/*, test/trace_processor/diff_tests/stdlib/chrome/*
+
    Copyright 2015 The Chromium Authors
 
    Redistribution and use in source and binary forms, with or without
@@ -214,3 +222,14 @@
    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+------------------
+
+Files: src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.{c, h}
+
+The author disclaims copyright to this source code. In place of a legal notice, here is a blessing:
+
+May you do good and not evil.
+May you find forgiveness for yourself and forgive others.
+May you share freely, never taking more than you give.
diff --git a/PRESUBMIT.py b/PRESUBMIT.py
index c615ffa..d809fc9 100644
--- a/PRESUBMIT.py
+++ b/PRESUBMIT.py
@@ -470,7 +470,10 @@
     return input_api.FilterSourceFile(
         x,
         files_to_check=[r'.*\.gni?$'],
-        files_to_skip=['^.gn$', '^gn/.*', '^buildtools/.*'])
+        files_to_skip=[
+            '^.gn$', '^gn/.*', '^buildtools/.*',
+            'src/trace_processor/perfetto_sql/preprocessor/BUILD.gn'
+        ])
 
   error_lines = []
   for f in input_api.AffectedSourceFiles(file_filter):
diff --git a/include/perfetto/base/compiler.h b/include/perfetto/base/compiler.h
index 22ebb8c..ad78786 100644
--- a/include/perfetto/base/compiler.h
+++ b/include/perfetto/base/compiler.h
@@ -17,8 +17,9 @@
 #ifndef INCLUDE_PERFETTO_BASE_COMPILER_H_
 #define INCLUDE_PERFETTO_BASE_COMPILER_H_
 
-#include <stddef.h>
+#include <cstddef>
 #include <type_traits>
+#include <variant>
 
 #include "perfetto/public/compiler.h"
 
@@ -135,13 +136,23 @@
 // Macro for telling -Wimplicit-fallthrough that a fallthrough is intentional.
 #define PERFETTO_FALLTHROUGH [[fallthrough]]
 
-namespace perfetto {
-namespace base {
+namespace perfetto::base {
 
 template <typename... T>
 inline void ignore_result(const T&...) {}
 
-}  // namespace base
-}  // namespace perfetto
+// Given a std::variant and a type T, returns the index of the T in the variant.
+template <typename VariantType, typename T, size_t i = 0>
+constexpr size_t variant_index() {
+  static_assert(i < std::variant_size_v<VariantType>,
+                "Type not found in variant");
+  if constexpr (std::is_same_v<std::variant_alternative_t<i, VariantType>, T>) {
+    return i;
+  } else {
+    return variant_index<VariantType, T, i + 1>();
+  }
+}
+
+}  // namespace perfetto::base
 
 #endif  // INCLUDE_PERFETTO_BASE_COMPILER_H_
diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine_unittest.cc b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine_unittest.cc
index 9116e4c..90cd2df 100644
--- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine_unittest.cc
+++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine_unittest.cc
@@ -54,7 +54,7 @@
 
   res = engine_.Execute(
       SqlSource::FromExecuteQuery("creatE PeRfEttO FUNCTION foo(x INT, y LONG) "
-                                  "RETURNS INT AS select :x + :y"));
+                                  "RETURNS INT AS select $x + $y"));
   ASSERT_TRUE(res.ok()) << res.status().c_message();
 }
 
diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_test_utils.h b/src/trace_processor/perfetto_sql/engine/perfetto_sql_test_utils.h
index 78cc4dc..86c0b76 100644
--- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_test_utils.h
+++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_test_utils.h
@@ -17,6 +17,14 @@
 #ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_TEST_UTILS_H_
 #define SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_TEST_UTILS_H_
 
+#include <cstddef>
+#include <cstdint>
+#include <ostream>
+#include <string>
+#include <tuple>
+
+#include "perfetto/base/logging.h"
+#include "perfetto/ext/base/status_or.h"
 #include "src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.h"
 #include "src/trace_processor/sqlite/sql_source.h"
 #include "test/gtest_and_gmock.h"
diff --git a/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn b/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn
index 1ee774e..d790e67 100644
--- a/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn
+++ b/src/trace_processor/perfetto_sql/preprocessor/BUILD.gn
@@ -22,6 +22,7 @@
     "perfetto_sql_preprocessor.h",
   ]
   deps = [
+    ":grammar",
     "../../../../gn:default_deps",
     "../../../base",
     "../../sqlite",
@@ -29,6 +30,19 @@
   ]
 }
 
+source_set("grammar") {
+  sources = [
+    "preprocessor_grammar.c",
+    "preprocessor_grammar.h",
+    "preprocessor_grammar_interface.h",
+  ]
+  deps = [ "../../../../gn:default_deps" ]
+  visibility = [ ":preprocessor" ]
+  if (perfetto_build_standalone) {
+    configs -= [ "//gn/standalone:extra_warnings" ]
+  }
+}
+
 perfetto_unittest_source_set("unittests") {
   testonly = true
   sources = [ "perfetto_sql_preprocessor_unittest.cc" ]
diff --git a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc
index 1033fe7..31a2b0c 100644
--- a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc
+++ b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.cc
@@ -19,53 +19,126 @@
 #include <algorithm>
 #include <cstddef>
 #include <cstdint>
+#include <cstdlib>
+#include <list>
+#include <memory>
 #include <optional>
 #include <string>
-#include <unordered_map>
+#include <string_view>
 #include <unordered_set>
 #include <utility>
+#include <variant>
 #include <vector>
 
+#include "perfetto/base/compiler.h"
 #include "perfetto/base/logging.h"
 #include "perfetto/base/status.h"
 #include "perfetto/ext/base/flat_hash_map.h"
-#include "perfetto/ext/base/status_or.h"
 #include "perfetto/ext/base/string_utils.h"
+#include "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar_interface.h"
 #include "src/trace_processor/sqlite/sql_source.h"
 #include "src/trace_processor/sqlite/sqlite_tokenizer.h"
-#include "src/trace_processor/util/status_macros.h"
 
 namespace perfetto::trace_processor {
 namespace {
 
-enum IntrinsicMacro {
-  kStringify,
-  kTokenZipJoin,
-  kPrefixedTokenZipJoin,
-  kTokenApply,
-  kTokenMapJoin,
-  kTokenMapJoinWithCapture,
-  kComma,
-  kOther
+using State = PreprocessorGrammarState;
+
+struct Preprocessor {
+ public:
+  explicit Preprocessor(State* state)
+      : parser_(PreprocessorGrammarParseAlloc(malloc, state)) {}
+  ~Preprocessor() { PreprocessorGrammarParseFree(parser_, free); }
+
+  void Parse(int token_type, PreprocessorGrammarToken token) {
+    PreprocessorGrammarParse(parser_, token_type, token);
+  }
+
+ private:
+  void* parser_;
 };
 
-IntrinsicMacro MacroNameToEnum(const std::string& macro_name) {
-  if (macro_name == "__intrinsic_stringify")
-    return kStringify;
-  if (macro_name == "__intrinsic_token_zip_join")
-    return kTokenZipJoin;
-  if (macro_name == "__intrinsic_prefixed_token_zip_join")
-    return kPrefixedTokenZipJoin;
-  if (macro_name == "__intrinsic_token_apply")
-    return kTokenApply;
-  if (macro_name == "__intrinsic_token_map_join")
-    return kTokenMapJoin;
-  if (macro_name == "__intrinsic_token_map_join_with_capture")
-    return kTokenMapJoinWithCapture;
-  if (macro_name == "__intrinsic_token_comma")
-    return kComma;
+struct Stringify {
+  bool ignore_table;
+};
+struct Apply {
+  int join_token;
+  int prefix_token;
+};
+using MacroImpl =
+    std::variant<PerfettoSqlPreprocessor::Macro*, Stringify, Apply>;
 
-  return kOther;
+// Synthetic "stackframe" representing the processing of a single piece of SQL.
+struct Frame {
+  struct Root {};
+  struct Rewrite {
+    SqliteTokenizer& tokenizer;
+    SqlSource::Rewriter& rewriter;
+    SqliteTokenizer::Token start;
+    SqliteTokenizer::Token end;
+  };
+  struct Append {
+    std::vector<SqlSource>& result;
+  };
+  using Type = std::variant<Root, Rewrite, Append>;
+  struct ActiveMacro {
+    std::string name;
+    MacroImpl impl;
+    std::vector<SqlSource> args;
+    uint32_t nested_macro_count;
+    std::unordered_set<std::string> seen_variables;
+  };
+  enum VariableHandling { kLookup, kLookupOrIgnore, kIgnore };
+
+  explicit Frame(Type _type,
+                 VariableHandling _var_handling,
+                 State* s,
+                 const SqlSource& source)
+      : type(_type),
+        var_handling(_var_handling),
+        preprocessor(s),
+        tokenizer(source),
+        rewriter(source),
+        substituitions(&owned_substituitions) {}
+  Frame(const Frame&) = delete;
+  Frame& operator=(const Frame&) = delete;
+  Frame(Frame&&) = delete;
+  Frame& operator=(Frame&&) = delete;
+
+  Type type;
+  VariableHandling var_handling;
+  Preprocessor preprocessor;
+  SqliteTokenizer tokenizer;
+
+  bool seen_semicolon = false;
+  SqlSource::Rewriter rewriter;
+  bool ignore_rewrite = false;
+
+  std::optional<ActiveMacro> active_macro;
+
+  base::FlatHashMap<std::string, SqlSource> owned_substituitions;
+  base::FlatHashMap<std::string, SqlSource>* substituitions;
+};
+
+struct ErrorToken {
+  SqliteTokenizer::Token token;
+  std::string message;
+};
+
+extern "C" struct PreprocessorGrammarState {
+  std::list<Frame> stack;
+  const base::FlatHashMap<std::string, PerfettoSqlPreprocessor::Macro>& macros;
+  std::optional<ErrorToken> error;
+};
+
+extern "C" struct PreprocessorGrammarApplyList {
+  std::vector<PreprocessorGrammarTokenBounds> args;
+};
+
+SqliteTokenizer::Token GrammarTokenToTokenizerToken(
+    const PreprocessorGrammarToken& token) {
+  return SqliteTokenizer::Token{std::string_view(token.ptr, token.n),
+                                SqliteTokenType::TK_ILLEGAL};
 }
 
 base::Status ErrorAtToken(const SqliteTokenizer& tokenizer,
@@ -75,115 +148,298 @@
   return base::ErrStatus("%s%s", traceback.c_str(), error);
 }
 
-struct InvocationArg {
-  std::optional<SqlSource> arg;
-  bool has_more;
-};
-
-base::StatusOr<InvocationArg> ParseMacroInvocationArg(
-    SqliteTokenizer& tokenizer,
-    SqliteTokenizer::Token& tok,
-    bool has_prev_args) {
-  uint32_t nested_parens = 0;
-  bool seen_token_in_arg = false;
-  auto start = tokenizer.NextNonWhitespace();
-  for (tok = start;; tok = tokenizer.NextNonWhitespace()) {
-    if (tok.IsTerminal()) {
-      if (tok.token_type == SqliteTokenType::TK_SEMI) {
-        // TODO(b/290185551): add a link to macro documentation.
-        return ErrorAtToken(tokenizer, tok,
-                            "Semi-colon is not allowed in macro invocation");
-      }
-      // TODO(b/290185551): add a link to macro documentation.
-      return ErrorAtToken(tokenizer, tok, "Macro invocation not complete");
-    }
-
-    bool is_arg_terminator = tok.token_type == SqliteTokenType::TK_RP ||
-                             tok.token_type == SqliteTokenType::TK_COMMA;
-    if (nested_parens == 0 && is_arg_terminator) {
-      bool token_required =
-          has_prev_args || tok.token_type != SqliteTokenType::TK_RP;
-      if (!seen_token_in_arg && token_required) {
-        // TODO(b/290185551): add a link to macro documentation.
-        return ErrorAtToken(tokenizer, tok, "Macro arg is empty");
-      }
-      return InvocationArg{
-          seen_token_in_arg ? std::make_optional(tokenizer.Substr(start, tok))
-                            : std::optional<SqlSource>(std::nullopt),
-          tok.token_type == SqliteTokenType::TK_COMMA,
-      };
-    }
-    seen_token_in_arg = true;
-
-    if (tok.token_type == SqliteTokenType::TK_LP) {
-      nested_parens++;
-      continue;
-    }
-    if (tok.token_type == SqliteTokenType::TK_RP) {
-      nested_parens--;
-      continue;
-    }
+std::vector<std::string> SqlSourceVectorToString(
+    const std::vector<SqlSource>& vec) {
+  std::vector<std::string> pieces;
+  pieces.reserve(vec.size());
+  for (const auto& list : vec) {
+    pieces.emplace_back(list.sql());
   }
+  return pieces;
 }
 
-base::StatusOr<std::optional<SqlSource>> ExecuteStringify(
-    const SqliteTokenizer& tokenizer,
-    const SqliteTokenizer::Token& name_token,
-    const std::vector<SqlSource>& args) {
-  if (args.empty()) {
-    return ErrorAtToken(tokenizer, name_token,
-                        "stringify: stringify must not be empty");
-  }
-
-  // Track the set of variables that, even if we see during stringify, we ignore
-  // and stringify them anyway.
-  std::unordered_set<std::string> ignored_variables;
-  for (uint32_t i = 1; i < args.size(); ++i) {
-    ignored_variables.emplace(args[i].sql());
-  }
-
-  // Ensure that we don't stringifiy any SQL variables present (unless they were
-  // explcitily marked as ignored).
-  SqliteTokenizer t(args[0]);
-  for (auto tok = t.NextNonWhitespace(); !tok.IsTerminal();
-       tok = t.NextNonWhitespace()) {
-    if (tok.token_type == SqliteTokenType::TK_VARIABLE &&
-        !ignored_variables.count(std::string(tok.str.substr(1)))) {
-      return {std::nullopt};
-    }
-  }
-  std::string res = "'" + args[0].sql() + "'";
-  return {SqlSource::FromTraceProcessorImplementation(std::move(res))};
+std::string_view BoundsToStringView(const PreprocessorGrammarTokenBounds& b) {
+  return {b.start.ptr, static_cast<size_t>(b.end.ptr + b.end.n - b.start.ptr)};
 }
 
-void RewriteIntrinsicMacro(const std::string& macro_name,
-                           std::optional<SqlSource>& res,
-                           std::vector<SqlSource>& token_list,
-                           SqliteTokenizer& tokenizer,
-                           SqlSource::Rewriter& rewriter,
-                           SqliteTokenizer::Token prev,
-                           SqliteTokenizer::Token tok) {
-  if (res) {
-    tokenizer.Rewrite(rewriter, prev, tok, *std::move(res),
-                      SqliteTokenizer::EndToken::kInclusive);
+void RewriteIntrinsicMacro(Frame& frame,
+                           SqliteTokenizer::Token name,
+                           SqliteTokenizer::Token rp) {
+  const auto& macro = *frame.active_macro;
+  frame.tokenizer.Rewrite(
+      frame.rewriter, name, rp,
+      SqlSource::FromTraceProcessorImplementation(
+          macro.name + "!(" +
+          base::Join(SqlSourceVectorToString(macro.args), ", ") + ")"),
+      SqliteTokenizer::EndToken::kInclusive);
+}
+
+void ExecuteSqlMacro(State* state,
+                     Frame& frame,
+                     Frame::ActiveMacro& macro,
+                     SqliteTokenizer::Token name,
+                     SqliteTokenizer::Token rp) {
+  auto& sql_macro = std::get<PerfettoSqlPreprocessor::Macro*>(macro.impl);
+  if (macro.args.size() != sql_macro->args.size()) {
+    state->error = ErrorToken{
+        name,
+        base::ErrStatus(
+            "wrong number of macro arguments, expected %zu actual %zu",
+            sql_macro->args.size(), macro.args.size())
+            .message(),
+    };
     return;
   }
-
-  // We failed to rewrite because a variable was still present in SQL.
-  // Just readd the stringify SQL with newly expanded token list.
-  std::vector<std::string> pieces;
-  pieces.reserve(token_list.size());
-  for (const auto& list : token_list) {
-    if (base::TrimWhitespace(list.sql()) == ",") {
-      pieces.emplace_back("__intrinsic_token_comma!()");
-    } else {
-      pieces.emplace_back(list.sql());
-    }
+  state->stack.emplace_back(
+      Frame::Rewrite{frame.tokenizer, frame.rewriter, name, rp}, Frame::kLookup,
+      state, sql_macro->sql);
+  auto& macro_frame = state->stack.back();
+  for (uint32_t i = 0; i < sql_macro->args.size(); ++i) {
+    macro_frame.owned_substituitions.Insert(sql_macro->args[i],
+                                            std::move(macro.args[i]));
   }
-  tokenizer.Rewrite(rewriter, prev, tok,
-                    SqlSource::FromTraceProcessorImplementation(
-                        macro_name + "!(" + base::Join(pieces, ", ") + ")"),
-                    SqliteTokenizer::EndToken::kInclusive);
+}
+
+void ExecuteStringify(State* state,
+                      Frame& frame,
+                      Frame::ActiveMacro& macro,
+                      SqliteTokenizer::Token name,
+                      SqliteTokenizer::Token rp) {
+  auto& stringify = std::get<Stringify>(macro.impl);
+  if (macro.args.size() != 1) {
+    state->error = ErrorToken{
+        name,
+        base::ErrStatus(
+            "stringify: must specify exactly 1 argument, actual %zu",
+            macro.args.size())
+            .message(),
+    };
+    return;
+  }
+  bool can_stringify =
+      macro.seen_variables.empty() ||
+      (stringify.ignore_table && macro.seen_variables.size() == 1 &&
+       macro.seen_variables.count("table"));
+  if (!can_stringify) {
+    RewriteIntrinsicMacro(frame, name, rp);
+    return;
+  }
+  auto res = SqlSource::FromTraceProcessorImplementation(
+      "'" + macro.args[0].sql() + "'");
+  frame.tokenizer.Rewrite(frame.rewriter, name, rp, std::move(res),
+                          SqliteTokenizer::EndToken::kInclusive);
+}
+
+void ExecuteApply(State* state,
+                  Frame& frame,
+                  Frame::ActiveMacro& macro,
+                  SqliteTokenizer::Token name,
+                  SqliteTokenizer::Token rp) {
+  auto& apply = std::get<Apply>(macro.impl);
+  if (!macro.seen_variables.empty()) {
+    RewriteIntrinsicMacro(frame, name, rp);
+    return;
+  }
+  state->stack.emplace_back(
+      Frame::Rewrite{frame.tokenizer, frame.rewriter, name, rp},
+      Frame::VariableHandling::kIgnore, state,
+      SqlSource::FromTraceProcessorImplementation(
+          base::Join(SqlSourceVectorToString(macro.args), " ")));
+
+  auto& expansion_frame = state->stack.back();
+  expansion_frame.preprocessor.Parse(
+      PPTK_APPLY, PreprocessorGrammarToken{nullptr, 0, PPTK_APPLY});
+  expansion_frame.preprocessor.Parse(
+      apply.join_token, PreprocessorGrammarToken{nullptr, 0, apply.join_token});
+  expansion_frame.preprocessor.Parse(
+      apply.prefix_token,
+      PreprocessorGrammarToken{nullptr, 0, apply.prefix_token});
+  expansion_frame.ignore_rewrite = true;
+}
+
+extern "C" void OnPreprocessorSyntaxError(State* state,
+                                          PreprocessorGrammarToken* token) {
+  state->error = {GrammarTokenToTokenizerToken(*token),
+                  "preprocessor syntax error"};
+}
+
+extern "C" void OnPreprocessorApply(PreprocessorGrammarState* state,
+                                    PreprocessorGrammarToken* name,
+                                    PreprocessorGrammarToken* join,
+                                    PreprocessorGrammarToken* prefix,
+                                    PreprocessorGrammarApplyList* raw_a,
+                                    PreprocessorGrammarApplyList* raw_b) {
+  std::unique_ptr<PreprocessorGrammarApplyList> a(raw_a);
+  std::unique_ptr<PreprocessorGrammarApplyList> b(raw_b);
+  auto& frame = state->stack.back();
+  size_t size = std::min(a->args.size(), b ? b->args.size() : a->args.size());
+  if (size == 0) {
+    auto& rewrite = std::get<Frame::Rewrite>(frame.type);
+    rewrite.tokenizer.Rewrite(rewrite.rewriter, rewrite.start, rewrite.end,
+                              SqlSource::FromTraceProcessorImplementation(""),
+                              SqliteTokenizer::EndToken::kInclusive);
+    return;
+  }
+  std::string macro(name->ptr, name->n);
+  std::vector<std::string> args;
+  for (uint32_t i = 0; i < size; ++i) {
+    std::string arg = macro;
+    arg.append("!(").append(BoundsToStringView(a->args[i]));
+    if (b) {
+      arg.append(",").append(BoundsToStringView(b->args[i]));
+    }
+    arg.append(")");
+    args.emplace_back(std::move(arg));
+  }
+  std::string joiner = join->major == PPTK_AND ? " AND " : " , ";
+  std::string res = prefix->major == PPTK_TRUE ? joiner : "";
+  res.append(base::Join(args, joiner));
+  state->stack.emplace_back(
+      frame.type, Frame::VariableHandling::kLookupOrIgnore, state,
+      SqlSource::FromTraceProcessorImplementation(std::move(res)));
+}
+
+extern "C" void OnPreprocessorVariable(State* state,
+                                       PreprocessorGrammarToken* var) {
+  if (var->n == 0 || var->ptr[0] != '$') {
+    state->error = {GrammarTokenToTokenizerToken(*var),
+                    "variable must start with '$'"};
+    return;
+  }
+  auto& frame = state->stack.back();
+  if (frame.active_macro) {
+    std::string name(var->ptr + 1, var->n - 1);
+    if (!frame.substituitions->Find(name)) {
+      frame.active_macro->seen_variables.insert(name);
+    }
+    return;
+  }
+  switch (frame.var_handling) {
+    case Frame::kLookup:
+    case Frame::kLookupOrIgnore: {
+      auto* it =
+          frame.substituitions->Find(std::string(var->ptr + 1, var->n - 1));
+      if (!it) {
+        if (frame.var_handling == Frame::kLookup) {
+          state->error = {GrammarTokenToTokenizerToken(*var),
+                          "variable not defined"};
+        }
+        return;
+      }
+      frame.tokenizer.RewriteToken(frame.rewriter,
+                                   GrammarTokenToTokenizerToken(*var), *it);
+      break;
+    }
+    case Frame::kIgnore:
+      break;
+  }
+}
+
+extern "C" void OnPreprocessorMacroId(State* state,
+                                      PreprocessorGrammarToken* name_tok) {
+  auto& invocation = state->stack.back();
+  if (invocation.active_macro) {
+    invocation.active_macro->nested_macro_count++;
+    return;
+  }
+  std::string name(name_tok->ptr, name_tok->n);
+  MacroImpl impl;
+  if (name == "__intrinsic_stringify") {
+    impl = Stringify();
+  } else if (name == "__intrinsic_stringify_ignore_table") {
+    impl = Stringify{true};
+  } else if (name == "__intrinsic_token_apply") {
+    impl = Apply{PPTK_COMMA, PPTK_FALSE};
+  } else if (name == "__intrinsic_token_apply_prefix") {
+    impl = Apply{PPTK_COMMA, PPTK_TRUE};
+  } else if (name == "__intrinsic_token_apply_and") {
+    impl = Apply{PPTK_AND, PPTK_FALSE};
+  } else if (name == "__intrinsic_token_apply_and_prefix") {
+    impl = Apply{PPTK_AND, PPTK_TRUE};
+  } else {
+    auto* sql_macro = state->macros.Find(name);
+    if (!sql_macro) {
+      state->error = {GrammarTokenToTokenizerToken(*name_tok),
+                      "no such macro defined"};
+      return;
+    }
+    impl = sql_macro;
+  }
+  invocation.active_macro =
+      Frame::ActiveMacro{std::move(name), impl, {}, 0, {}};
+}
+
+extern "C" void OnPreprocessorMacroArg(State* state,
+                                       PreprocessorGrammarTokenBounds* arg) {
+  auto& frame = state->stack.back();
+  auto& macro = *frame.active_macro;
+  if (macro.nested_macro_count > 0) {
+    return;
+  }
+  auto start_token = GrammarTokenToTokenizerToken(arg->start);
+  auto end_token = GrammarTokenToTokenizerToken(arg->end);
+  state->stack.emplace_back(
+      Frame::Append{macro.args}, frame.var_handling, state,
+      frame.tokenizer.Substr(start_token, end_token,
+                             SqliteTokenizer::EndToken::kInclusive));
+
+  auto& arg_frame = state->stack.back();
+  arg_frame.substituitions = frame.substituitions;
+}
+
+extern "C" void OnPreprocessorMacroEnd(State* state,
+                                       PreprocessorGrammarToken* name,
+                                       PreprocessorGrammarToken* rp) {
+  auto& frame = state->stack.back();
+  auto& macro = *frame.active_macro;
+  if (macro.nested_macro_count > 0) {
+    --macro.nested_macro_count;
+    return;
+  }
+  switch (macro.impl.index()) {
+    case base::variant_index<MacroImpl, PerfettoSqlPreprocessor::Macro*>():
+      ExecuteSqlMacro(state, frame, macro, GrammarTokenToTokenizerToken(*name),
+                      GrammarTokenToTokenizerToken(*rp));
+      break;
+    case base::variant_index<MacroImpl, Stringify>():
+      ExecuteStringify(state, frame, macro, GrammarTokenToTokenizerToken(*name),
+                       GrammarTokenToTokenizerToken(*rp));
+      break;
+    case base::variant_index<MacroImpl, Apply>():
+      ExecuteApply(state, frame, macro, GrammarTokenToTokenizerToken(*name),
+                   GrammarTokenToTokenizerToken(*rp));
+      break;
+    default:
+      PERFETTO_FATAL("Unknown variant type");
+  }
+  frame.active_macro = std::nullopt;
+}
+
+extern "C" void OnPreprocessorEnd(State* state) {
+  auto& frame = state->stack.back();
+  PERFETTO_CHECK(!frame.active_macro);
+
+  if (frame.ignore_rewrite) {
+    return;
+  }
+  switch (frame.type.index()) {
+    case base::variant_index<Frame::Type, Frame::Append>(): {
+      auto& append = std::get<Frame::Append>(frame.type);
+      append.result.push_back(std::move(frame.rewriter).Build());
+      break;
+    }
+    case base::variant_index<Frame::Type, Frame::Rewrite>(): {
+      auto& rewrite = std::get<Frame::Rewrite>(frame.type);
+      rewrite.tokenizer.Rewrite(rewrite.rewriter, rewrite.start, rewrite.end,
+                                std::move(frame.rewriter).Build(),
+                                SqliteTokenizer::EndToken::kInclusive);
+      break;
+    }
+    case base::variant_index<Frame::Type, Frame::Root>():
+      break;
+    default:
+      PERFETTO_FATAL("Unknown frame type");
+  }
 }
 
 }  // namespace
@@ -211,398 +467,71 @@
   SqlSource stmt =
       global_tokenizer_.Substr(tok, global_tokenizer_.NextTerminal(),
                                SqliteTokenizer::EndToken::kInclusive);
-  auto stmt_or = RewriteInternal(stmt, {});
-  if (stmt_or.ok()) {
-    statement_ = std::move(*stmt_or);
-    return true;
+  State s{{}, *macros_, {}};
+  s.stack.emplace_back(Frame::Root(), Frame::kIgnore, &s, std::move(stmt));
+  for (;;) {
+    auto* frame = &s.stack.back();
+    auto& tk = frame->tokenizer;
+    SqliteTokenizer::Token t = tk.NextNonWhitespace();
+    int token_type;
+    if (t.str.empty()) {
+      token_type = frame->seen_semicolon ? 0 : PPTK_SEMI;
+      frame->seen_semicolon = true;
+    } else if (t.token_type == SqliteTokenType::TK_SEMI) {
+      token_type = PPTK_SEMI;
+      frame->seen_semicolon = true;
+    } else if (t.token_type == SqliteTokenType::TK_ILLEGAL) {
+      if (t.str.size() == 1 && t.str[0] == '!') {
+        token_type = PPTK_EXCLAIM;
+      } else {
+        status_ = ErrorAtToken(tk, t, "illegal token");
+        return false;
+      }
+    } else if (t.token_type == SqliteTokenType::TK_ID) {
+      token_type = PPTK_ID;
+    } else if (t.token_type == SqliteTokenType::TK_LP) {
+      token_type = PPTK_LP;
+    } else if (t.token_type == SqliteTokenType::TK_RP) {
+      token_type = PPTK_RP;
+    } else if (t.token_type == SqliteTokenType::TK_COMMA) {
+      token_type = PPTK_COMMA;
+    } else if (t.token_type == SqliteTokenType::TK_VARIABLE) {
+      token_type = PPTK_VARIABLE;
+    } else {
+      token_type = PPTK_OPAQUE;
+    }
+    frame->preprocessor.Parse(
+        token_type,
+        PreprocessorGrammarToken{t.str.data(), t.str.size(), token_type});
+    if (s.error) {
+      status_ = ErrorAtToken(tk, s.error->token, s.error->message.c_str());
+      return false;
+    }
+    if (token_type == 0) {
+      if (s.stack.size() == 1) {
+        statement_ = std::move(frame->rewriter).Build();
+        return true;
+      }
+      s.stack.pop_back();
+      frame = &s.stack.back();
+    }
   }
-  status_ = stmt_or.status();
-  return false;
 }
 
-base::StatusOr<SqlSource> PerfettoSqlPreprocessor::RewriteInternal(
-    const SqlSource& source,
-    const std::unordered_map<std::string, SqlSource>& arg_bindings) {
-  SqlSource::Rewriter rewriter(source);
-  SqliteTokenizer tokenizer(source);
-  for (SqliteTokenizer::Token tok = tokenizer.NextNonWhitespace(), prev;;
-       prev = tok, tok = tokenizer.NextNonWhitespace()) {
-    if (tok.IsTerminal()) {
-      break;
-    }
-    if (tok.token_type == SqliteTokenType::TK_VARIABLE &&
-        !seen_macros_.empty()) {
-      PERFETTO_CHECK(tok.str.size() >= 2);
-      if (tok.str[0] != '$') {
-        return ErrorAtToken(tokenizer, tok, "Variables must start with $");
-      }
-      auto binding_it = arg_bindings.find(std::string(tok.str.substr(1)));
-      if (binding_it == arg_bindings.end()) {
-        // TODO(lalitm): reenable making this an error once we actually pass
-        // macros around in graph_scan instead of bare-SQL.
-        // return ErrorAtToken(tokenizer, tok, "Variable not found");
-        continue;
-      }
-      tokenizer.RewriteToken(rewriter, tok, binding_it->second);
-      continue;
-    }
-    if (tok.token_type != SqliteTokenType::TK_ILLEGAL || tok.str != "!") {
-      continue;
-    }
-
-    const auto& name_token = prev;
-    if (name_token.token_type == SqliteTokenType::TK_VARIABLE) {
-      // TODO(b/290185551): add a link to macro documentation.
-      return ErrorAtToken(tokenizer, name_token,
-                          "Macro name cannot be a variable");
-    }
-    if (name_token.token_type != SqliteTokenType::TK_ID) {
-      // TODO(b/290185551): add a link to macro documentation.
-      return ErrorAtToken(tokenizer, name_token, "Macro invocation is invalid");
-    }
-
-    // Go to the opening parenthesis of the macro invocation.
-    tok = tokenizer.NextNonWhitespace();
-
-    std::string macro_name(name_token.str);
-    IntrinsicMacro macro_enum = MacroNameToEnum(macro_name);
-    ASSIGN_OR_RETURN(std::vector<SqlSource> token_list,
-                     ParseTokenList(tokenizer, tok, arg_bindings));
-
-    // Non intrinsic macro.
-    if (macro_enum == kOther) {
-      ASSIGN_OR_RETURN(SqlSource invocation,
-                       ExecuteMacroInvocation(tokenizer, prev, macro_name,
-                                              std::move(token_list)));
-      tokenizer.Rewrite(rewriter, prev, tok, std::move(invocation),
-                        SqliteTokenizer::EndToken::kInclusive);
-      continue;
-    }
-
-    // Token comma instrinsic macro requires special handling.
-    if (macro_enum == kComma) {
-      if (!token_list.empty()) {
-        return ErrorAtToken(tokenizer, name_token,
-                            "token_comma: no arguments allowd");
-      }
-      tokenizer.Rewrite(rewriter, prev, tok,
-                        SqlSource::FromTraceProcessorImplementation(","),
-                        SqliteTokenizer::EndToken::kInclusive);
-      continue;
-    }
-
-    // Intrinsic macros.
-    std::optional<SqlSource> res;
-    switch (macro_enum) {
-      case kStringify: {
-        ASSIGN_OR_RETURN(res,
-                         ExecuteStringify(tokenizer, name_token, token_list));
-        break;
-      }
-      case kTokenZipJoin: {
-        ASSIGN_OR_RETURN(
-            res, ExecuteTokenZipJoin(tokenizer, name_token, token_list, false));
-        break;
-      }
-      case kPrefixedTokenZipJoin: {
-        ASSIGN_OR_RETURN(
-            res, ExecuteTokenZipJoin(tokenizer, name_token, token_list, true));
-        break;
-      }
-      case kTokenMapJoin: {
-        ASSIGN_OR_RETURN(
-            res, ExecuteTokenMapJoin(tokenizer, name_token, token_list));
-        break;
-      }
-      case kTokenMapJoinWithCapture: {
-        ASSIGN_OR_RETURN(res, ExecuteTokenMapJoinWithCapture(
-                                  tokenizer, name_token, token_list));
-
-        break;
-      }
-
-      case kTokenApply: {
-        ASSIGN_OR_RETURN(res,
-                         ExecuteTokenApply(tokenizer, name_token, token_list));
-        break;
-      }
-      case kComma:
-      case kOther:
-        PERFETTO_FATAL("Shouldn't be reached");
-    }
-    RewriteIntrinsicMacro(macro_name, res, token_list, tokenizer, rewriter,
-                          prev, tok);
-  }
-  return std::move(rewriter).Build();
+extern "C" PreprocessorGrammarApplyList* OnPreprocessorCreateApplyList() {
+  return std::make_unique<PreprocessorGrammarApplyList>().release();
 }
 
-base::StatusOr<std::vector<SqlSource>> PerfettoSqlPreprocessor::ParseTokenList(
-    SqliteTokenizer& tokenizer,
-    SqliteTokenizer::Token& tok,
-    const std::unordered_map<std::string, SqlSource>& bindings) {
-  if (tok.token_type != SqliteTokenType::TK_LP) {
-    return ErrorAtToken(tokenizer, tok, "( expected to open token list");
-  }
-  std::vector<SqlSource> tokens;
-  bool has_more = true;
-  while (has_more) {
-    ASSIGN_OR_RETURN(InvocationArg invocation_arg,
-                     ParseMacroInvocationArg(tokenizer, tok, !tokens.empty()));
-    if (invocation_arg.arg) {
-      ASSIGN_OR_RETURN(SqlSource res,
-                       RewriteInternal(invocation_arg.arg.value(), bindings));
-      tokens.emplace_back(std::move(res));
-    }
-    has_more = invocation_arg.has_more;
-  }
-  return tokens;
+extern "C" PreprocessorGrammarApplyList* OnPreprocessorAppendApplyList(
+    PreprocessorGrammarApplyList* list,
+    PreprocessorGrammarTokenBounds* bounds) {
+  list->args.push_back(*bounds);
+  return list;
 }
 
-base::StatusOr<SqlSource> PerfettoSqlPreprocessor::ExecuteMacroInvocation(
-    const SqliteTokenizer& tokenizer,
-    const SqliteTokenizer::Token& name_token,
-    const std::string& macro_name,
-    std::vector<SqlSource> token_list) {
-  Macro* macro = macros_->Find(macro_name);
-  if (!macro) {
-    // TODO(b/290185551): add a link to macro documentation.
-    base::StackString<1024> err("Macro %s does not exist", macro_name.c_str());
-    return ErrorAtToken(tokenizer, name_token, err.c_str());
-  }
-  if (seen_macros_.count(macro_name)) {
-    // TODO(b/290185551): add a link to macro documentation.
-    return ErrorAtToken(tokenizer, name_token,
-                        "Macros cannot be recursive or mutually recursive");
-  }
-  if (token_list.size() < macro->args.size()) {
-    // TODO(lalitm): add a link to macro documentation.
-    return ErrorAtToken(tokenizer, name_token,
-                        "Macro invoked with too few args");
-  }
-  if (token_list.size() > macro->args.size()) {
-    // TODO(lalitm): add a link to macro documentation.
-    return ErrorAtToken(tokenizer, name_token,
-                        "Macro invoked with too many args");
-  }
-  std::unordered_map<std::string, SqlSource> inner_bindings;
-  for (auto& t : token_list) {
-    inner_bindings.emplace(macro->args[inner_bindings.size()], std::move(t));
-  }
-  PERFETTO_CHECK(inner_bindings.size() == macro->args.size());
-
-  seen_macros_.emplace(macro->name);
-  ASSIGN_OR_RETURN(SqlSource res, RewriteInternal(macro->sql, inner_bindings));
-  seen_macros_.erase(macro->name);
-  return res;
-}
-
-base::StatusOr<std::optional<SqlSource>>
-PerfettoSqlPreprocessor::ExecuteTokenZipJoin(
-    const SqliteTokenizer& tokenizer,
-    const SqliteTokenizer::Token& name_token,
-    std::vector<SqlSource> token_list,
-    bool prefixed) {
-  if (token_list.size() != 4) {
-    return ErrorAtToken(tokenizer, name_token,
-                        "token_zip_join: must have exactly four args");
-  }
-
-  SqliteTokenizer first_tokenizer(std::move(token_list[0]));
-  SqliteTokenizer::Token inner_tok = first_tokenizer.NextNonWhitespace();
-  if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-    return {std::nullopt};
-  }
-  ASSIGN_OR_RETURN(std::vector<SqlSource> first_sources,
-                   ParseTokenList(first_tokenizer, inner_tok, {}));
-
-  SqliteTokenizer second_tokenizer(std::move(token_list[1]));
-  inner_tok = second_tokenizer.NextNonWhitespace();
-  if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-    return {std::nullopt};
-  }
-  ASSIGN_OR_RETURN(std::vector<SqlSource> second_sources,
-                   ParseTokenList(second_tokenizer, inner_tok, {}));
-
-  SqliteTokenizer name_tokenizer(token_list[2]);
-  inner_tok = name_tokenizer.NextNonWhitespace();
-  if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-    return {std::nullopt};
-  }
-
-  size_t zip_count = std::min(first_sources.size(), second_sources.size());
-  std::vector<std::string> res;
-  for (uint32_t i = 0; i < zip_count; ++i) {
-    ASSIGN_OR_RETURN(
-        SqlSource invocation_res,
-        ExecuteMacroInvocation(tokenizer, name_token, token_list[2].sql(),
-                               {first_sources[i], second_sources[i]}));
-    res.push_back(invocation_res.sql());
-  }
-
-  if (res.empty()) {
-    return {SqlSource::FromTraceProcessorImplementation("")};
-  }
-
-  std::string zipped = base::Join(res, " " + token_list[3].sql() + " ");
-  if (prefixed) {
-    zipped = " " + token_list[3].sql() + " " + zipped;
-  }
-  return {SqlSource::FromTraceProcessorImplementation(zipped)};
-}
-
-base::StatusOr<std::optional<SqlSource>>
-PerfettoSqlPreprocessor::ExecuteTokenApply(
-    const SqliteTokenizer& tokenizer,
-    const SqliteTokenizer::Token& name_token,
-    std::vector<SqlSource> token_list) {
-  if (token_list.size() != 3) {
-    return ErrorAtToken(tokenizer, name_token,
-                        "token_apply: must have exactly three args");
-  }
-
-  SqliteTokenizer arg_list_tokenizer(token_list[0]);
-  SqliteTokenizer::Token inner_tok = arg_list_tokenizer.NextNonWhitespace();
-  if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-    return {std::nullopt};
-  }
-  ASSIGN_OR_RETURN(std::vector<SqlSource> arg_list_sources,
-                   ParseTokenList(arg_list_tokenizer, inner_tok, {}));
-
-  SqliteTokenizer name_tokenizer(token_list[1]);
-  inner_tok = name_tokenizer.NextNonWhitespace();
-  if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-    return {std::nullopt};
-  }
-
-  std::vector<std::string> res;
-  for (const auto& arg_list_source : arg_list_sources) {
-    SqliteTokenizer args_tokenizer(arg_list_source);
-    inner_tok = args_tokenizer.NextNonWhitespace();
-    if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-      return {std::nullopt};
-    }
-
-    ASSIGN_OR_RETURN(std::vector<SqlSource> args_sources,
-                     ParseTokenList(args_tokenizer, inner_tok, {}));
-
-    ASSIGN_OR_RETURN(SqlSource invocation_res,
-                     ExecuteMacroInvocation(tokenizer, name_token,
-                                            token_list[1].sql(), args_sources));
-    res.push_back(invocation_res.sql());
-  }
-
-  if (res.empty()) {
-    return {SqlSource::FromTraceProcessorImplementation("")};
-  }
-
-  std::string zipped = base::Join(res, " " + token_list[2].sql() + " ");
-  return {SqlSource::FromTraceProcessorImplementation(zipped)};
-}
-
-base::StatusOr<std::optional<SqlSource>>
-PerfettoSqlPreprocessor::ExecuteTokenMapJoin(
-    const SqliteTokenizer& tokenizer,
-    const SqliteTokenizer::Token& name_token,
-    std::vector<SqlSource> token_list) {
-  if (token_list.size() != 3) {
-    return ErrorAtToken(tokenizer, name_token,
-                        "token_map_join: must have exactly three args");
-  }
-
-  SqliteTokenizer arg_list_tokenizer(token_list[0]);
-  SqliteTokenizer::Token inner_tok = arg_list_tokenizer.NextNonWhitespace();
-  if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-    return {std::nullopt};
-  }
-  ASSIGN_OR_RETURN(std::vector<SqlSource> arg_list_sources,
-                   ParseTokenList(arg_list_tokenizer, inner_tok, {}));
-
-  SqliteTokenizer name_tokenizer(token_list[1]);
-  inner_tok = name_tokenizer.NextNonWhitespace();
-  if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-    return {std::nullopt};
-  }
-
-  std::vector<std::string> res;
-  for (const auto& arg_list_source : arg_list_sources) {
-    SqliteTokenizer args_tokenizer(arg_list_source);
-    inner_tok = args_tokenizer.NextNonWhitespace();
-    if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-      return {std::nullopt};
-    }
-
-    ASSIGN_OR_RETURN(
-        SqlSource invocation_res,
-        ExecuteMacroInvocation(tokenizer, name_token, token_list[1].sql(),
-                               {arg_list_source}));
-    res.push_back(invocation_res.sql());
-  }
-
-  if (res.empty()) {
-    return {SqlSource::FromTraceProcessorImplementation("")};
-  }
-
-  std::string zipped = base::Join(res, " " + token_list[2].sql() + " ");
-  return {SqlSource::FromTraceProcessorImplementation(zipped)};
-}
-
-base::StatusOr<std::optional<SqlSource>>
-PerfettoSqlPreprocessor::ExecuteTokenMapJoinWithCapture(
-    const SqliteTokenizer& tokenizer,
-    const SqliteTokenizer::Token& name_token,
-    std::vector<SqlSource> token_list) {
-  if (token_list.size() != 4) {
-    return ErrorAtToken(
-        tokenizer, name_token,
-        "token_map_join_with_capture: must have exactly four args");
-  }
-
-  SqliteTokenizer arg_list_tokenizer(token_list[0]);
-  SqliteTokenizer::Token inner_tok = arg_list_tokenizer.NextNonWhitespace();
-  if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-    return {std::nullopt};
-  }
-  ASSIGN_OR_RETURN(std::vector<SqlSource> arg_list_sources,
-                   ParseTokenList(arg_list_tokenizer, inner_tok, {}));
-
-  SqliteTokenizer name_tokenizer(token_list[1]);
-  inner_tok = name_tokenizer.NextNonWhitespace();
-  if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-    return {std::nullopt};
-  }
-
-  SqliteTokenizer capture_tokenizer(token_list[2]);
-  inner_tok = capture_tokenizer.NextNonWhitespace();
-  if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-    return {std::nullopt};
-  }
-  ASSIGN_OR_RETURN(std::vector<SqlSource> captured_args,
-                   ParseTokenList(capture_tokenizer, inner_tok, {}));
-
-  std::vector<std::string> res;
-  for (const auto& arg_list_source : arg_list_sources) {
-    SqliteTokenizer args_tokenizer(arg_list_source);
-    inner_tok = args_tokenizer.NextNonWhitespace();
-    if (inner_tok.token_type == SqliteTokenType::TK_VARIABLE) {
-      return {std::nullopt};
-    }
-
-    std::vector<SqlSource> macro_args{arg_list_source};
-    macro_args.insert(macro_args.end(), captured_args.begin(),
-                      captured_args.end());
-    ASSIGN_OR_RETURN(
-        SqlSource invocation_res,
-        ExecuteMacroInvocation(tokenizer, name_token, token_list[1].sql(),
-                               std::move(macro_args)));
-    res.push_back(invocation_res.sql());
-  }
-
-  if (res.empty()) {
-    return {SqlSource::FromTraceProcessorImplementation("")};
-  }
-
-  std::string zipped = base::Join(res, " " + token_list[3].sql() + " ");
-  return {SqlSource::FromTraceProcessorImplementation(zipped)};
+extern "C" void OnPreprocessorFreeApplyList(
+    PreprocessorGrammarApplyList* list) {
+  delete list;
 }
 
 }  // namespace perfetto::trace_processor
diff --git a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h
index 4991724..669d964 100644
--- a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h
+++ b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h
@@ -19,13 +19,11 @@
 
 #include <optional>
 #include <string>
-#include <unordered_map>
 #include <unordered_set>
 #include <vector>
 
 #include "perfetto/base/status.h"
 #include "perfetto/ext/base/flat_hash_map.h"
-#include "perfetto/ext/base/status_or.h"
 #include "src/trace_processor/sqlite/sql_source.h"
 #include "src/trace_processor/sqlite/sqlite_tokenizer.h"
 
@@ -68,42 +66,6 @@
   SqlSource& statement() { return *statement_; }
 
  private:
-  base::StatusOr<SqlSource> RewriteInternal(
-      const SqlSource&,
-      const std::unordered_map<std::string, SqlSource>& arg_bindings);
-
-  base::StatusOr<std::vector<SqlSource>> ParseTokenList(
-      SqliteTokenizer& tokenizer,
-      SqliteTokenizer::Token& token,
-      const std::unordered_map<std::string, SqlSource>& arg_bindings);
-
-  base::StatusOr<SqlSource> ExecuteMacroInvocation(
-      const SqliteTokenizer& tokenizer,
-      const SqliteTokenizer::Token& name_token,
-      const std::string& macro_name,
-      std::vector<SqlSource> token_list);
-
-  base::StatusOr<std::optional<SqlSource>> ExecuteTokenZipJoin(
-      const SqliteTokenizer& tokenizer,
-      const SqliteTokenizer::Token& name_token,
-      std::vector<SqlSource> token_list,
-      bool prefixed);
-
-  base::StatusOr<std::optional<SqlSource>> ExecuteTokenApply(
-      const SqliteTokenizer& tokenizer,
-      const SqliteTokenizer::Token& name_token,
-      std::vector<SqlSource> token_list);
-
-  base::StatusOr<std::optional<SqlSource>> ExecuteTokenMapJoin(
-      const SqliteTokenizer& tokenizer,
-      const SqliteTokenizer::Token& name_token,
-      std::vector<SqlSource> token_list);
-
-  base::StatusOr<std::optional<SqlSource>> ExecuteTokenMapJoinWithCapture(
-      const SqliteTokenizer& tokenizer,
-      const SqliteTokenizer::Token& name_token,
-      std::vector<SqlSource> token_list);
-
   SqliteTokenizer global_tokenizer_;
   const base::FlatHashMap<std::string, Macro>* macros_ = nullptr;
   std::unordered_set<std::string> seen_macros_;
diff --git a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor_unittest.cc b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor_unittest.cc
index 9c24cb1..b59b429 100644
--- a/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor_unittest.cc
+++ b/src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor_unittest.cc
@@ -97,7 +97,7 @@
   auto source = SqlSource::FromExecuteQuery(
       "foo!((select s.ts + r.dur from s, r), 1234); SELECT 1");
   PerfettoSqlPreprocessor preprocessor(source, macros_);
-  ASSERT_TRUE(preprocessor.NextStatement());
+  ASSERT_TRUE(preprocessor.NextStatement()) << preprocessor.status().message();
   ASSERT_EQ(preprocessor.statement().AsTraceback(0),
             "Fully expanded statement\n"
             "  SELECT (select s.ts + r.dur from s, r) + 1234;\n"
@@ -143,7 +143,7 @@
 
   auto bar = SqlSource::FromExecuteQuery(
       "CREATE PERFETTO MACRO bar(a, b) Returns Expr AS "
-      "tfoo!($a, $b) + foo!($b, $a)");
+      "foo!($a, $b) + foo!($b, $a)");
   macros_.Insert("bar", Macro{
                             false,
                             "bar",
@@ -154,11 +154,11 @@
   auto source = SqlSource::FromExecuteQuery(
       "SELECT bar!((select s.ts + r.dur from s, r), 1234); SELECT 1");
   PerfettoSqlPreprocessor preprocessor(source, macros_);
-  ASSERT_TRUE(preprocessor.NextStatement());
+  ASSERT_TRUE(preprocessor.NextStatement()) << preprocessor.status().message();
   ASSERT_EQ(preprocessor.statement().sql(),
             "SELECT (select s.ts + r.dur from s, r) + 1234 + 1234 + "
             "(select s.ts + r.dur from s, r);");
-  ASSERT_TRUE(preprocessor.NextStatement());
+  ASSERT_TRUE(preprocessor.NextStatement()) << preprocessor.status().message();
   ASSERT_EQ(preprocessor.statement().sql(), "SELECT 1");
 }
 
@@ -229,196 +229,7 @@
               "  File \"stdin\" line 1 col 1\n"
               "    __intrinsic_stringify!()\n"
               "    ^\n"
-              "stringify: stringify must not be empty");
-  }
-}
-
-TEST_F(PerfettoSqlPreprocessorUnittest, ZipJoin) {
-  auto foo = SqlSource::FromExecuteQuery(
-      "CREATE PERFETTO MACRO G(a Expr, b Expr) Returns Expr AS $a AS $b");
-  macros_.Insert("G", Macro{
-                          false,
-                          "G",
-                          {"a", "b"},
-                          FindSubstr(foo, "$a AS $b"),
-                      });
-
-  auto zj = SqlSource::FromExecuteQuery(
-      "CREATE PERFETTO MACRO ZJ(a Expr, b Expr, c Expr, d Expr) Returns Expr "
-      "AS __intrinsic_token_zip_join!($a, $b, $c, $d)");
-  macros_.Insert(
-      "ZJ", Macro{
-                false,
-                "ZJ",
-                {"a", "b", "c", "d"},
-                FindSubstr(zj, "__intrinsic_token_zip_join!($a, $b, $c, $d)"),
-            });
-  {
-    auto source = SqlSource::FromExecuteQuery(
-        "__intrinsic_token_zip_join!((foo, bar), (baz, bat), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "foo AS baz AND bar AS bat");
-    ASSERT_FALSE(preprocessor.NextStatement());
-  }
-  {
-    auto source = SqlSource::FromExecuteQuery(
-        "__intrinsic_token_zip_join!((foo, bar), (baz, bat, bada), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "foo AS baz AND bar AS bat");
-    ASSERT_FALSE(preprocessor.NextStatement());
-  }
-  {
-    auto source = SqlSource::FromExecuteQuery(
-        "__intrinsic_token_zip_join!((foo, bar), (baz, bat, bada), G, "
-        "__intrinsic_token_comma!())");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "foo AS baz , bar AS bat");
-    ASSERT_FALSE(preprocessor.NextStatement());
-  }
-  {
-    auto source = SqlSource::FromExecuteQuery(
-        "ZJ!((foo, bar), (baz, bat, bada), G, __intrinsic_token_comma!())");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "foo AS baz , bar AS bat");
-    ASSERT_FALSE(preprocessor.NextStatement());
-  }
-}
-
-TEST_F(PerfettoSqlPreprocessorUnittest, TokenApply) {
-  auto foo = SqlSource::FromExecuteQuery(
-      "CREATE PERFETTO MACRO G(a Expr, b Expr) Returns Expr AS $a AS $b");
-  macros_.Insert("G", Macro{
-                          false,
-                          "G",
-                          {"a", "b"},
-                          FindSubstr(foo, "$a AS $b"),
-                      });
-
-  auto tp = SqlSource::FromExecuteQuery(
-      "CREATE PERFETTO MACRO TokApply(a Expr, b Expr, c Expr) Returns Expr "
-      "AS __intrinsic_token_apply!($a, $b, $c)");
-  macros_.Insert("TokApply",
-                 Macro{
-                     false,
-                     "TokApply",
-                     {"a", "b", "c"},
-                     FindSubstr(tp, "__intrinsic_token_apply!($a, $b, $c)"),
-                 });
-  {
-    auto source =
-        SqlSource::FromExecuteQuery("__intrinsic_token_apply!((), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "");
-    ASSERT_FALSE(preprocessor.NextStatement());
-  }
-  {
-    auto source = SqlSource::FromExecuteQuery(
-        "__intrinsic_token_apply!(((foo, bar)), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "foo AS bar");
-    ASSERT_FALSE(preprocessor.NextStatement());
-  }
-  {
-    auto source = SqlSource::FromExecuteQuery(
-        "__intrinsic_token_apply!(((foo, bar), (baz, bat)), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "foo AS bar AND baz AS bat");
-    ASSERT_FALSE(preprocessor.NextStatement());
-  }
-  {
-    auto source = SqlSource::FromExecuteQuery(
-        "__intrinsic_token_apply!(((foo, bar), (baz, bat, bada)), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_FALSE(preprocessor.NextStatement());
-    ASSERT_THAT(preprocessor.status().message(), HasSubstr("too many args"));
-  }
-  {
-    auto source = SqlSource::FromExecuteQuery(
-        "__intrinsic_token_apply!(((foo, bar), (baz)), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_FALSE(preprocessor.NextStatement());
-    ASSERT_THAT(preprocessor.status().message(), HasSubstr("too few args"));
-  }
-  {
-    auto source = SqlSource::FromExecuteQuery(
-        "TokApply!(((foo, bar), (baz, bat)), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "foo AS bar AND baz AS bat");
-    ASSERT_FALSE(preprocessor.NextStatement());
-  }
-}
-
-TEST_F(PerfettoSqlPreprocessorUnittest, TokenMapJoin) {
-  auto foo = SqlSource::FromExecuteQuery(
-      "CREATE PERFETTO MACRO G(a Expr) Returns Expr AS baza.$a");
-  macros_.Insert("G", Macro{
-                          false,
-                          "G",
-                          {"a"},
-                          FindSubstr(foo, "baza.$a"),
-                      });
-
-  auto tp = SqlSource::FromExecuteQuery(
-      "CREATE PERFETTO MACRO TokMapJoin(a Expr, b Expr, c Expr) Returns Expr "
-      "AS __intrinsic_token_map_join!($a, $b, $c)");
-  macros_.Insert("TokMapJoin",
-                 Macro{
-                     false,
-                     "TokMapJoin",
-                     {"a", "b", "c"},
-                     FindSubstr(tp, "__intrinsic_token_map_join!($a, $b, $c)"),
-                 });
-  {
-    auto source =
-        SqlSource::FromExecuteQuery("__intrinsic_token_map_join!((), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "");
-    ASSERT_FALSE(preprocessor.NextStatement());
-  }
-  {
-    auto source = SqlSource::FromExecuteQuery(
-        "__intrinsic_token_map_join!((foo), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "baza.foo");
-    ASSERT_FALSE(preprocessor.NextStatement());
-  }
-  {
-    auto source = SqlSource::FromExecuteQuery(
-        "__intrinsic_token_map_join!((foo, baz), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "baza.foo AND baza.baz");
-    ASSERT_FALSE(preprocessor.NextStatement());
-  }
-  {
-    auto source =
-        SqlSource::FromExecuteQuery("TokMapJoin!((foo, bar), G, AND)");
-    PerfettoSqlPreprocessor preprocessor(source, macros_);
-    ASSERT_TRUE(preprocessor.NextStatement())
-        << preprocessor.status().message();
-    ASSERT_EQ(preprocessor.statement().sql(), "baza.foo AND baza.bar");
-    ASSERT_FALSE(preprocessor.NextStatement());
+              "stringify: must specify exactly 1 argument, actual 0");
   }
 }
 
diff --git a/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.c b/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.c
new file mode 100644
index 0000000..968abb7
--- /dev/null
+++ b/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.c
@@ -0,0 +1,1365 @@
+/* This file is automatically generated by Lemon from input grammar
+** source file "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.y".
+*/
+/*
+** 2000-05-29
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Driver template for the LEMON parser generator.
+**
+** The "lemon" program processes an LALR(1) input grammar file, then uses
+** this template to construct a parser.  The "lemon" program inserts text
+** at each "%%" line.  Also, any "P-a-r-s-e" identifer prefix (without the
+** interstitial "-" characters) contained in this template is changed into
+** the value of the %name directive from the grammar.  Otherwise, the content
+** of this template is copied straight through into the generate parser
+** source file.
+**
+** The following is the concatenation of all %include directives from the
+** input grammar file:
+*/
+/************ Begin %include sections from the grammar ************************/
+#include "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar_interface.h"
+
+#define YYNOERRORRECOVERY 1
+#define YYPARSEFREENEVERNULL 1
+/**************** End of %include directives **********************************/
+/* These constants specify the various numeric values for terminal symbols.
+***************** Begin token definitions *************************************/
+#ifndef PPTK_SEMI
+#define PPTK_SEMI                            1
+#define PPTK_APPLY                           2
+#define PPTK_COMMA                           3
+#define PPTK_AND                             4
+#define PPTK_TRUE                            5
+#define PPTK_FALSE                           6
+#define PPTK_ID                              7
+#define PPTK_LP                              8
+#define PPTK_RP                              9
+#define PPTK_OPAQUE                         10
+#define PPTK_EXCLAIM                        11
+#define PPTK_VARIABLE                       12
+#endif
+/**************** End token definitions ***************************************/
+
+/* The next sections is a series of control #defines.
+** various aspects of the generated parser.
+**    YYCODETYPE         is the data type used to store the integer codes
+**                       that represent terminal and non-terminal symbols.
+**                       "unsigned char" is used if there are fewer than
+**                       256 symbols.  Larger types otherwise.
+**    YYNOCODE           is a number of type YYCODETYPE that is not used for
+**                       any terminal or nonterminal symbol.
+**    YYFALLBACK         If defined, this indicates that one or more tokens
+**                       (also known as: "terminal symbols") have fall-back
+**                       values which should be used if the original symbol
+**                       would not parse.  This permits keywords to sometimes
+**                       be used as identifiers, for example.
+**    YYACTIONTYPE       is the data type used for "action codes" - numbers
+**                       that indicate what to do in response to the next
+**                       token.
+**    PreprocessorGrammarParseTOKENTYPE     is the data type used for minor type for terminal
+**                       symbols.  Background: A "minor type" is a semantic
+**                       value associated with a terminal or non-terminal
+**                       symbols.  For example, for an "ID" terminal symbol,
+**                       the minor type might be the name of the identifier.
+**                       Each non-terminal can have a different minor type.
+**                       Terminal symbols all have the same minor type, though.
+**                       This macros defines the minor type for terminal 
+**                       symbols.
+**    YYMINORTYPE        is the data type used for all minor types.
+**                       This is typically a union of many types, one of
+**                       which is PreprocessorGrammarParseTOKENTYPE.  The entry in the union
+**                       for terminal symbols is called "yy0".
+**    YYSTACKDEPTH       is the maximum depth of the parser's stack.  If
+**                       zero the stack is dynamically sized using realloc()
+**    PreprocessorGrammarParseARG_SDECL     A static variable declaration for the %extra_argument
+**    PreprocessorGrammarParseARG_PDECL     A parameter declaration for the %extra_argument
+**    PreprocessorGrammarParseARG_PARAM     Code to pass %extra_argument as a subroutine parameter
+**    PreprocessorGrammarParseARG_STORE     Code to store %extra_argument into yypParser
+**    PreprocessorGrammarParseARG_FETCH     Code to extract %extra_argument from yypParser
+**    PreprocessorGrammarParseCTX_*         As PreprocessorGrammarParseARG_ except for %extra_context
+**    YYERRORSYMBOL      is the code number of the error symbol.  If not
+**                       defined, then do no error processing.
+**    YYNSTATE           the combined number of states.
+**    YYNRULE            the number of rules in the grammar
+**    YYNTOKEN           Number of terminal symbols
+**    YY_MAX_SHIFT       Maximum value for shift actions
+**    YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions
+**    YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions
+**    YY_ERROR_ACTION    The yy_action[] code for syntax error
+**    YY_ACCEPT_ACTION   The yy_action[] code for accept
+**    YY_NO_ACTION       The yy_action[] code for no-op
+**    YY_MIN_REDUCE      Minimum value for reduce actions
+**    YY_MAX_REDUCE      Maximum value for reduce actions
+*/
+#ifndef INTERFACE
+# define INTERFACE 1
+#endif
+/************* Begin control #defines *****************************************/
+#define YYCODETYPE unsigned char
+#define YYNOCODE 25
+#define YYACTIONTYPE unsigned char
+#define PreprocessorGrammarParseTOKENTYPE struct PreprocessorGrammarToken
+typedef union {
+  int yyinit;
+  PreprocessorGrammarParseTOKENTYPE yy0;
+  struct PreprocessorGrammarApplyList* yy2;
+  struct PreprocessorGrammarTokenBounds yy32;
+} YYMINORTYPE;
+#ifndef YYSTACKDEPTH
+#define YYSTACKDEPTH 100
+#endif
+#define PreprocessorGrammarParseARG_SDECL
+#define PreprocessorGrammarParseARG_PDECL
+#define PreprocessorGrammarParseARG_PARAM
+#define PreprocessorGrammarParseARG_FETCH
+#define PreprocessorGrammarParseARG_STORE
+#define PreprocessorGrammarParseCTX_SDECL struct PreprocessorGrammarState* state;
+#define PreprocessorGrammarParseCTX_PDECL ,struct PreprocessorGrammarState* state
+#define PreprocessorGrammarParseCTX_PARAM ,state
+#define PreprocessorGrammarParseCTX_FETCH struct PreprocessorGrammarState* state=yypParser->state;
+#define PreprocessorGrammarParseCTX_STORE yypParser->state=state;
+#define YYNSTATE             26
+#define YYNRULE              26
+#define YYNRULE_WITH_ACTION  16
+#define YYNTOKEN             13
+#define YY_MAX_SHIFT         25
+#define YY_MIN_SHIFTREDUCE   42
+#define YY_MAX_SHIFTREDUCE   67
+#define YY_ERROR_ACTION      68
+#define YY_ACCEPT_ACTION     69
+#define YY_NO_ACTION         70
+#define YY_MIN_REDUCE        71
+#define YY_MAX_REDUCE        96
+/************* End control #defines *******************************************/
+#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])))
+
+/* Define the yytestcase() macro to be a no-op if is not already defined
+** otherwise.
+**
+** Applications can choose to define yytestcase() in the %include section
+** to a macro that can assist in verifying code coverage.  For production
+** code the yytestcase() macro should be turned off.  But it is useful
+** for testing.
+*/
+#ifndef yytestcase
+# define yytestcase(X)
+#endif
+
+
+/* Next are the tables used to determine what action to take based on the
+** current state and lookahead token.  These tables are used to implement
+** functions that take a state number and lookahead value and return an
+** action integer.  
+**
+** Suppose the action integer is N.  Then the action is determined as
+** follows
+**
+**   0 <= N <= YY_MAX_SHIFT             Shift N.  That is, push the lookahead
+**                                      token onto the stack and goto state N.
+**
+**   N between YY_MIN_SHIFTREDUCE       Shift to an arbitrary state then
+**     and YY_MAX_SHIFTREDUCE           reduce by rule N-YY_MIN_SHIFTREDUCE.
+**
+**   N == YY_ERROR_ACTION               A syntax error has occurred.
+**
+**   N == YY_ACCEPT_ACTION              The parser accepts its input.
+**
+**   N == YY_NO_ACTION                  No such action.  Denotes unused
+**                                      slots in the yy_action[] table.
+**
+**   N between YY_MIN_REDUCE            Reduce by rule N-YY_MIN_REDUCE
+**     and YY_MAX_REDUCE
+**
+** The action table is constructed as a single large table named yy_action[].
+** Given state S and lookahead X, the action is computed as either:
+**
+**    (A)   N = yy_action[ yy_shift_ofst[S] + X ]
+**    (B)   N = yy_default[S]
+**
+** The (A) formula is preferred.  The B formula is used instead if
+** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X.
+**
+** The formulas above are for computing the action when the lookahead is
+** a terminal symbol.  If the lookahead is a non-terminal (as occurs after
+** a reduce action) then the yy_reduce_ofst[] array is used in place of
+** the yy_shift_ofst[] array.
+**
+** The following are the tables generated in this section:
+**
+**  yy_action[]        A single table containing all actions.
+**  yy_lookahead[]     A table containing the lookahead for each entry in
+**                     yy_action.  Used to detect hash collisions.
+**  yy_shift_ofst[]    For each state, the offset into yy_action for
+**                     shifting terminals.
+**  yy_reduce_ofst[]   For each state, the offset into yy_action for
+**                     shifting non-terminals after a reduce.
+**  yy_default[]       Default action for each state.
+**
+*********** Begin parsing tables **********************************************/
+#define YY_ACTTAB_COUNT (84)
+static const YYACTIONTYPE yy_action[] = {
+ /*     0 */    69,   25,    7,   19,   15,   62,    7,    7,   24,   21,
+ /*    10 */     2,    3,   48,   11,   53,   86,   24,   22,   20,   20,
+ /*    20 */    62,   85,   24,    4,   21,    2,   51,   48,   62,   53,
+ /*    30 */    14,   14,   21,    2,   50,   48,   58,   53,   62,   89,
+ /*    40 */    89,   24,   21,    2,    5,   48,   10,   53,   86,   24,
+ /*    50 */    21,    2,    6,   48,   17,   53,    6,    6,   24,   59,
+ /*    60 */     8,   12,    9,   83,   86,   24,   44,    1,   13,    9,
+ /*    70 */    49,   86,   24,    8,   11,   23,   86,   24,   71,   16,
+ /*    80 */    95,   70,   18,   18,
+};
+static const YYCODETYPE yy_lookahead[] = {
+ /*     0 */    13,   14,   15,   16,    2,    3,   19,   20,   21,    7,
+ /*    10 */     8,    8,   10,   18,   12,   20,   21,   22,   23,   24,
+ /*    20 */     3,   20,   21,    8,    7,    8,    9,   10,    3,   12,
+ /*    30 */     3,    4,    7,    8,    9,   10,    1,   12,    3,   19,
+ /*    40 */    20,   21,    7,    8,    3,   10,   18,   12,   20,   21,
+ /*    50 */     7,    8,   15,   10,    7,   12,   19,   20,   21,    1,
+ /*    60 */     3,   17,   18,   11,   20,   21,    9,    8,   17,   18,
+ /*    70 */     9,   20,   21,    3,   18,   11,   20,   21,    0,    9,
+ /*    80 */    24,   25,    5,    6,   25,   25,   25,   25,   25,   25,
+ /*    90 */    25,   25,   25,   25,   25,   25,   13,
+};
+#define YY_SHIFT_COUNT    (25)
+#define YY_SHIFT_MIN      (0)
+#define YY_SHIFT_MAX      (78)
+static const unsigned char yy_shift_ofst[] = {
+ /*     0 */     2,   43,   17,   43,   43,   43,   25,   35,   43,   43,
+ /*    10 */    43,   43,   57,   70,   77,   27,    3,   15,   47,   58,
+ /*    20 */    41,   52,   61,   59,   64,   78,
+};
+#define YY_REDUCE_COUNT (11)
+#define YY_REDUCE_MIN   (-13)
+#define YY_REDUCE_MAX   (56)
+static const signed char yy_reduce_ofst[] = {
+ /*     0 */   -13,   -5,   37,   44,   51,   56,   20,   20,   28,    1,
+ /*    10 */     1,    1,
+};
+static const YYACTIONTYPE yy_default[] = {
+ /*     0 */    68,   94,   68,   76,   76,   68,   68,   68,   68,   75,
+ /*    10 */    74,   84,   68,   68,   68,   68,   72,   68,   68,   68,
+ /*    20 */    93,   81,   68,   68,   68,   68,
+};
+/********** End of lemon-generated parsing tables *****************************/
+
+/* The next table maps tokens (terminal symbols) into fallback tokens.  
+** If a construct like the following:
+** 
+**      %fallback ID X Y Z.
+**
+** appears in the grammar, then ID becomes a fallback token for X, Y,
+** and Z.  Whenever one of the tokens X, Y, or Z is input to the parser
+** but it does not parse, the type of the token is changed to ID and
+** the parse is retried before an error is thrown.
+**
+** This feature can be used, for example, to cause some keywords in a language
+** to revert to identifiers if they keyword does not apply in the context where
+** it appears.
+*/
+#ifdef YYFALLBACK
+static const YYCODETYPE yyFallback[] = {
+};
+#endif /* YYFALLBACK */
+
+/* The following structure represents a single element of the
+** parser's stack.  Information stored includes:
+**
+**   +  The state number for the parser at this level of the stack.
+**
+**   +  The value of the token stored at this level of the stack.
+**      (In other words, the "major" token.)
+**
+**   +  The semantic value stored at this level of the stack.  This is
+**      the information used by the action routines in the grammar.
+**      It is sometimes called the "minor" token.
+**
+** After the "shift" half of a SHIFTREDUCE action, the stateno field
+** actually contains the reduce action for the second half of the
+** SHIFTREDUCE.
+*/
+struct yyStackEntry {
+  YYACTIONTYPE stateno;  /* The state-number, or reduce action in SHIFTREDUCE */
+  YYCODETYPE major;      /* The major token value.  This is the code
+                         ** number for the token at this stack level */
+  YYMINORTYPE minor;     /* The user-supplied minor token value.  This
+                         ** is the value of the token  */
+};
+typedef struct yyStackEntry yyStackEntry;
+
+/* The state of the parser is completely contained in an instance of
+** the following structure */
+struct yyParser {
+  yyStackEntry *yytos;          /* Pointer to top element of the stack */
+#ifdef YYTRACKMAXSTACKDEPTH
+  int yyhwm;                    /* High-water mark of the stack */
+#endif
+#ifndef YYNOERRORRECOVERY
+  int yyerrcnt;                 /* Shifts left before out of the error */
+#endif
+  PreprocessorGrammarParseARG_SDECL                /* A place to hold %extra_argument */
+  PreprocessorGrammarParseCTX_SDECL                /* A place to hold %extra_context */
+#if YYSTACKDEPTH<=0
+  int yystksz;                  /* Current side of the stack */
+  yyStackEntry *yystack;        /* The parser's stack */
+  yyStackEntry yystk0;          /* First stack entry */
+#else
+  yyStackEntry yystack[YYSTACKDEPTH];  /* The parser's stack */
+  yyStackEntry *yystackEnd;            /* Last entry in the stack */
+#endif
+};
+typedef struct yyParser yyParser;
+
+#include <assert.h>
+#ifndef NDEBUG
+#include <stdio.h>
+static FILE *yyTraceFILE = 0;
+static char *yyTracePrompt = 0;
+#endif /* NDEBUG */
+
+#ifndef NDEBUG
+/* 
+** Turn parser tracing on by giving a stream to which to write the trace
+** and a prompt to preface each trace message.  Tracing is turned off
+** by making either argument NULL 
+**
+** Inputs:
+** <ul>
+** <li> A FILE* to which trace output should be written.
+**      If NULL, then tracing is turned off.
+** <li> A prefix string written at the beginning of every
+**      line of trace output.  If NULL, then tracing is
+**      turned off.
+** </ul>
+**
+** Outputs:
+** None.
+*/
+void PreprocessorGrammarParseTrace(FILE *TraceFILE, char *zTracePrompt){
+  yyTraceFILE = TraceFILE;
+  yyTracePrompt = zTracePrompt;
+  if( yyTraceFILE==0 ) yyTracePrompt = 0;
+  else if( yyTracePrompt==0 ) yyTraceFILE = 0;
+}
+#endif /* NDEBUG */
+
+#if defined(YYCOVERAGE) || !defined(NDEBUG)
+/* For tracing shifts, the names of all terminals and nonterminals
+** are required.  The following table supplies these names */
+static const char *const yyTokenName[] = { 
+  /*    0 */ "$",
+  /*    1 */ "SEMI",
+  /*    2 */ "APPLY",
+  /*    3 */ "COMMA",
+  /*    4 */ "AND",
+  /*    5 */ "TRUE",
+  /*    6 */ "FALSE",
+  /*    7 */ "ID",
+  /*    8 */ "LP",
+  /*    9 */ "RP",
+  /*   10 */ "OPAQUE",
+  /*   11 */ "EXCLAIM",
+  /*   12 */ "VARIABLE",
+  /*   13 */ "input",
+  /*   14 */ "cmd",
+  /*   15 */ "sql",
+  /*   16 */ "apply",
+  /*   17 */ "applylist",
+  /*   18 */ "tokenlist",
+  /*   19 */ "sqltoken",
+  /*   20 */ "commalesssqltoken",
+  /*   21 */ "minvocationid",
+  /*   22 */ "marglist",
+  /*   23 */ "marglistinner",
+  /*   24 */ "marg",
+};
+#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */
+
+#ifndef NDEBUG
+/* For tracing reduce actions, the names of all rules are required.
+*/
+static const char *const yyRuleName[] = {
+ /*   0 */ "input ::= cmd",
+ /*   1 */ "apply ::= APPLY COMMA|AND TRUE|FALSE ID LP applylist RP",
+ /*   2 */ "apply ::= APPLY COMMA|AND TRUE|FALSE ID LP applylist RP LP applylist RP",
+ /*   3 */ "applylist ::= applylist COMMA tokenlist",
+ /*   4 */ "applylist ::= tokenlist",
+ /*   5 */ "applylist ::=",
+ /*   6 */ "commalesssqltoken ::= OPAQUE",
+ /*   7 */ "commalesssqltoken ::= minvocationid EXCLAIM LP marglist RP",
+ /*   8 */ "commalesssqltoken ::= LP sql RP",
+ /*   9 */ "commalesssqltoken ::= LP RP",
+ /*  10 */ "commalesssqltoken ::= ID",
+ /*  11 */ "commalesssqltoken ::= VARIABLE",
+ /*  12 */ "minvocationid ::= ID",
+ /*  13 */ "marg ::= tokenlist",
+ /*  14 */ "tokenlist ::= tokenlist commalesssqltoken",
+ /*  15 */ "tokenlist ::= commalesssqltoken",
+ /*  16 */ "cmd ::= sql SEMI",
+ /*  17 */ "cmd ::= apply SEMI",
+ /*  18 */ "sql ::= sql sqltoken",
+ /*  19 */ "sql ::= sqltoken",
+ /*  20 */ "sqltoken ::= COMMA",
+ /*  21 */ "sqltoken ::= commalesssqltoken",
+ /*  22 */ "marglist ::= marglistinner",
+ /*  23 */ "marglist ::=",
+ /*  24 */ "marglistinner ::= marglistinner COMMA marg",
+ /*  25 */ "marglistinner ::= marg",
+};
+#endif /* NDEBUG */
+
+
+#if YYSTACKDEPTH<=0
+/*
+** Try to increase the size of the parser stack.  Return the number
+** of errors.  Return 0 on success.
+*/
+static int yyGrowStack(yyParser *p){
+  int newSize;
+  int idx;
+  yyStackEntry *pNew;
+
+  newSize = p->yystksz*2 + 100;
+  idx = p->yytos ? (int)(p->yytos - p->yystack) : 0;
+  if( p->yystack==&p->yystk0 ){
+    pNew = malloc(newSize*sizeof(pNew[0]));
+    if( pNew ) pNew[0] = p->yystk0;
+  }else{
+    pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
+  }
+  if( pNew ){
+    p->yystack = pNew;
+    p->yytos = &p->yystack[idx];
+#ifndef NDEBUG
+    if( yyTraceFILE ){
+      fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n",
+              yyTracePrompt, p->yystksz, newSize);
+    }
+#endif
+    p->yystksz = newSize;
+  }
+  return pNew==0; 
+}
+#endif
+
+/* Datatype of the argument to the memory allocated passed as the
+** second argument to PreprocessorGrammarParseAlloc() below.  This can be changed by
+** putting an appropriate #define in the %include section of the input
+** grammar.
+*/
+#ifndef YYMALLOCARGTYPE
+# define YYMALLOCARGTYPE size_t
+#endif
+
+/* Initialize a new parser that has already been allocated.
+*/
+void PreprocessorGrammarParseInit(void *yypRawParser PreprocessorGrammarParseCTX_PDECL){
+  yyParser *yypParser = (yyParser*)yypRawParser;
+  PreprocessorGrammarParseCTX_STORE
+#ifdef YYTRACKMAXSTACKDEPTH
+  yypParser->yyhwm = 0;
+#endif
+#if YYSTACKDEPTH<=0
+  yypParser->yytos = NULL;
+  yypParser->yystack = NULL;
+  yypParser->yystksz = 0;
+  if( yyGrowStack(yypParser) ){
+    yypParser->yystack = &yypParser->yystk0;
+    yypParser->yystksz = 1;
+  }
+#endif
+#ifndef YYNOERRORRECOVERY
+  yypParser->yyerrcnt = -1;
+#endif
+  yypParser->yytos = yypParser->yystack;
+  yypParser->yystack[0].stateno = 0;
+  yypParser->yystack[0].major = 0;
+#if YYSTACKDEPTH>0
+  yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1];
+#endif
+}
+
+#ifndef PreprocessorGrammarParse_ENGINEALWAYSONSTACK
+/* 
+** This function allocates a new parser.
+** The only argument is a pointer to a function which works like
+** malloc.
+**
+** Inputs:
+** A pointer to the function used to allocate memory.
+**
+** Outputs:
+** A pointer to a parser.  This pointer is used in subsequent calls
+** to PreprocessorGrammarParse and PreprocessorGrammarParseFree.
+*/
+void *PreprocessorGrammarParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) PreprocessorGrammarParseCTX_PDECL){
+  yyParser *yypParser;
+  yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
+  if( yypParser ){
+    PreprocessorGrammarParseCTX_STORE
+    PreprocessorGrammarParseInit(yypParser PreprocessorGrammarParseCTX_PARAM);
+  }
+  return (void*)yypParser;
+}
+#endif /* PreprocessorGrammarParse_ENGINEALWAYSONSTACK */
+
+
+/* The following function deletes the "minor type" or semantic value
+** associated with a symbol.  The symbol can be either a terminal
+** or nonterminal. "yymajor" is the symbol code, and "yypminor" is
+** a pointer to the value to be deleted.  The code used to do the 
+** deletions is derived from the %destructor and/or %token_destructor
+** directives of the input grammar.
+*/
+static void yy_destructor(
+  yyParser *yypParser,    /* The parser */
+  YYCODETYPE yymajor,     /* Type code for object to destroy */
+  YYMINORTYPE *yypminor   /* The object to be destroyed */
+){
+  PreprocessorGrammarParseARG_FETCH
+  PreprocessorGrammarParseCTX_FETCH
+  switch( yymajor ){
+    /* Here is inserted the actions which take place when a
+    ** terminal or non-terminal is destroyed.  This can happen
+    ** when the symbol is popped from the stack during a
+    ** reduce or during error processing or when a parser is 
+    ** being destroyed before it is finished parsing.
+    **
+    ** Note: during a reduce, the only symbols destroyed are those
+    ** which appear on the RHS of the rule, but which are *not* used
+    ** inside the C code.
+    */
+/********* Begin destructor definitions ***************************************/
+    case 17: /* applylist */
+{
+ OnPreprocessorFreeApplyList((yypminor->yy2)); 
+}
+      break;
+/********* End destructor definitions *****************************************/
+    default:  break;   /* If no destructor action specified: do nothing */
+  }
+}
+
+/*
+** Pop the parser's stack once.
+**
+** If there is a destructor routine associated with the token which
+** is popped from the stack, then call it.
+*/
+static void yy_pop_parser_stack(yyParser *pParser){
+  yyStackEntry *yytos;
+  assert( pParser->yytos!=0 );
+  assert( pParser->yytos > pParser->yystack );
+  yytos = pParser->yytos--;
+#ifndef NDEBUG
+  if( yyTraceFILE ){
+    fprintf(yyTraceFILE,"%sPopping %s\n",
+      yyTracePrompt,
+      yyTokenName[yytos->major]);
+  }
+#endif
+  yy_destructor(pParser, yytos->major, &yytos->minor);
+}
+
+/*
+** Clear all secondary memory allocations from the parser
+*/
+void PreprocessorGrammarParseFinalize(void *p){
+  yyParser *pParser = (yyParser*)p;
+  while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser);
+#if YYSTACKDEPTH<=0
+  if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack);
+#endif
+}
+
+#ifndef PreprocessorGrammarParse_ENGINEALWAYSONSTACK
+/* 
+** Deallocate and destroy a parser.  Destructors are called for
+** all stack elements before shutting the parser down.
+**
+** If the YYPARSEFREENEVERNULL macro exists (for example because it
+** is defined in a %include section of the input grammar) then it is
+** assumed that the input pointer is never NULL.
+*/
+void PreprocessorGrammarParseFree(
+  void *p,                    /* The parser to be deleted */
+  void (*freeProc)(void*)     /* Function used to reclaim memory */
+){
+#ifndef YYPARSEFREENEVERNULL
+  if( p==0 ) return;
+#endif
+  PreprocessorGrammarParseFinalize(p);
+  (*freeProc)(p);
+}
+#endif /* PreprocessorGrammarParse_ENGINEALWAYSONSTACK */
+
+/*
+** Return the peak depth of the stack for a parser.
+*/
+#ifdef YYTRACKMAXSTACKDEPTH
+int PreprocessorGrammarParseStackPeak(void *p){
+  yyParser *pParser = (yyParser*)p;
+  return pParser->yyhwm;
+}
+#endif
+
+/* This array of booleans keeps track of the parser statement
+** coverage.  The element yycoverage[X][Y] is set when the parser
+** is in state X and has a lookahead token Y.  In a well-tested
+** systems, every element of this matrix should end up being set.
+*/
+#if defined(YYCOVERAGE)
+static unsigned char yycoverage[YYNSTATE][YYNTOKEN];
+#endif
+
+/*
+** Write into out a description of every state/lookahead combination that
+**
+**   (1)  has not been used by the parser, and
+**   (2)  is not a syntax error.
+**
+** Return the number of missed state/lookahead combinations.
+*/
+#if defined(YYCOVERAGE)
+int PreprocessorGrammarParseCoverage(FILE *out){
+  int stateno, iLookAhead, i;
+  int nMissed = 0;
+  for(stateno=0; stateno<YYNSTATE; stateno++){
+    i = yy_shift_ofst[stateno];
+    for(iLookAhead=0; iLookAhead<YYNTOKEN; iLookAhead++){
+      if( yy_lookahead[i+iLookAhead]!=iLookAhead ) continue;
+      if( yycoverage[stateno][iLookAhead]==0 ) nMissed++;
+      if( out ){
+        fprintf(out,"State %d lookahead %s %s\n", stateno,
+                yyTokenName[iLookAhead],
+                yycoverage[stateno][iLookAhead] ? "ok" : "missed");
+      }
+    }
+  }
+  return nMissed;
+}
+#endif
+
+/*
+** Find the appropriate action for a parser given the terminal
+** look-ahead token iLookAhead.
+*/
+static YYACTIONTYPE yy_find_shift_action(
+  YYCODETYPE iLookAhead,    /* The look-ahead token */
+  YYACTIONTYPE stateno      /* Current state number */
+){
+  int i;
+
+  if( stateno>YY_MAX_SHIFT ) return stateno;
+  assert( stateno <= YY_SHIFT_COUNT );
+#if defined(YYCOVERAGE)
+  yycoverage[stateno][iLookAhead] = 1;
+#endif
+  do{
+    i = yy_shift_ofst[stateno];
+    assert( i>=0 );
+    assert( i<=YY_ACTTAB_COUNT );
+    assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD );
+    assert( iLookAhead!=YYNOCODE );
+    assert( iLookAhead < YYNTOKEN );
+    i += iLookAhead;
+    assert( i<(int)YY_NLOOKAHEAD );
+    if( yy_lookahead[i]!=iLookAhead ){
+#ifdef YYFALLBACK
+      YYCODETYPE iFallback;            /* Fallback token */
+      assert( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0]) );
+      iFallback = yyFallback[iLookAhead];
+      if( iFallback!=0 ){
+#ifndef NDEBUG
+        if( yyTraceFILE ){
+          fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n",
+             yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
+        }
+#endif
+        assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */
+        iLookAhead = iFallback;
+        continue;
+      }
+#endif
+#ifdef YYWILDCARD
+      {
+        int j = i - iLookAhead + YYWILDCARD;
+        assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) );
+        if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){
+#ifndef NDEBUG
+          if( yyTraceFILE ){
+            fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n",
+               yyTracePrompt, yyTokenName[iLookAhead],
+               yyTokenName[YYWILDCARD]);
+          }
+#endif /* NDEBUG */
+          return yy_action[j];
+        }
+      }
+#endif /* YYWILDCARD */
+      return yy_default[stateno];
+    }else{
+      assert( i>=0 && i<(int)(sizeof(yy_action)/sizeof(yy_action[0])) );
+      return yy_action[i];
+    }
+  }while(1);
+}
+
+/*
+** Find the appropriate action for a parser given the non-terminal
+** look-ahead token iLookAhead.
+*/
+static YYACTIONTYPE yy_find_reduce_action(
+  YYACTIONTYPE stateno,     /* Current state number */
+  YYCODETYPE iLookAhead     /* The look-ahead token */
+){
+  int i;
+#ifdef YYERRORSYMBOL
+  if( stateno>YY_REDUCE_COUNT ){
+    return yy_default[stateno];
+  }
+#else
+  assert( stateno<=YY_REDUCE_COUNT );
+#endif
+  i = yy_reduce_ofst[stateno];
+  assert( iLookAhead!=YYNOCODE );
+  i += iLookAhead;
+#ifdef YYERRORSYMBOL
+  if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){
+    return yy_default[stateno];
+  }
+#else
+  assert( i>=0 && i<YY_ACTTAB_COUNT );
+  assert( yy_lookahead[i]==iLookAhead );
+#endif
+  return yy_action[i];
+}
+
+/*
+** The following routine is called if the stack overflows.
+*/
+static void yyStackOverflow(yyParser *yypParser){
+   PreprocessorGrammarParseARG_FETCH
+   PreprocessorGrammarParseCTX_FETCH
+#ifndef NDEBUG
+   if( yyTraceFILE ){
+     fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt);
+   }
+#endif
+   while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser);
+   /* Here code is inserted which will execute if the parser
+   ** stack every overflows */
+/******** Begin %stack_overflow code ******************************************/
+/******** End %stack_overflow code ********************************************/
+   PreprocessorGrammarParseARG_STORE /* Suppress warning about unused %extra_argument var */
+   PreprocessorGrammarParseCTX_STORE
+}
+
+/*
+** Print tracing information for a SHIFT action
+*/
+#ifndef NDEBUG
+static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){
+  if( yyTraceFILE ){
+    if( yyNewState<YYNSTATE ){
+      fprintf(yyTraceFILE,"%s%s '%s', go to state %d\n",
+         yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major],
+         yyNewState);
+    }else{
+      fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n",
+         yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major],
+         yyNewState - YY_MIN_REDUCE);
+    }
+  }
+}
+#else
+# define yyTraceShift(X,Y,Z)
+#endif
+
+/*
+** Perform a shift action.
+*/
+static void yy_shift(
+  yyParser *yypParser,          /* The parser to be shifted */
+  YYACTIONTYPE yyNewState,      /* The new state to shift in */
+  YYCODETYPE yyMajor,           /* The major token to shift in */
+  PreprocessorGrammarParseTOKENTYPE yyMinor        /* The minor token to shift in */
+){
+  yyStackEntry *yytos;
+  yypParser->yytos++;
+#ifdef YYTRACKMAXSTACKDEPTH
+  if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
+    yypParser->yyhwm++;
+    assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) );
+  }
+#endif
+#if YYSTACKDEPTH>0 
+  if( yypParser->yytos>yypParser->yystackEnd ){
+    yypParser->yytos--;
+    yyStackOverflow(yypParser);
+    return;
+  }
+#else
+  if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){
+    if( yyGrowStack(yypParser) ){
+      yypParser->yytos--;
+      yyStackOverflow(yypParser);
+      return;
+    }
+  }
+#endif
+  if( yyNewState > YY_MAX_SHIFT ){
+    yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+  }
+  yytos = yypParser->yytos;
+  yytos->stateno = yyNewState;
+  yytos->major = yyMajor;
+  yytos->minor.yy0 = yyMinor;
+  yyTraceShift(yypParser, yyNewState, "Shift");
+}
+
+/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side
+** of that rule */
+static const YYCODETYPE yyRuleInfoLhs[] = {
+    13,  /* (0) input ::= cmd */
+    16,  /* (1) apply ::= APPLY COMMA|AND TRUE|FALSE ID LP applylist RP */
+    16,  /* (2) apply ::= APPLY COMMA|AND TRUE|FALSE ID LP applylist RP LP applylist RP */
+    17,  /* (3) applylist ::= applylist COMMA tokenlist */
+    17,  /* (4) applylist ::= tokenlist */
+    17,  /* (5) applylist ::= */
+    20,  /* (6) commalesssqltoken ::= OPAQUE */
+    20,  /* (7) commalesssqltoken ::= minvocationid EXCLAIM LP marglist RP */
+    20,  /* (8) commalesssqltoken ::= LP sql RP */
+    20,  /* (9) commalesssqltoken ::= LP RP */
+    20,  /* (10) commalesssqltoken ::= ID */
+    20,  /* (11) commalesssqltoken ::= VARIABLE */
+    21,  /* (12) minvocationid ::= ID */
+    24,  /* (13) marg ::= tokenlist */
+    18,  /* (14) tokenlist ::= tokenlist commalesssqltoken */
+    18,  /* (15) tokenlist ::= commalesssqltoken */
+    14,  /* (16) cmd ::= sql SEMI */
+    14,  /* (17) cmd ::= apply SEMI */
+    15,  /* (18) sql ::= sql sqltoken */
+    15,  /* (19) sql ::= sqltoken */
+    19,  /* (20) sqltoken ::= COMMA */
+    19,  /* (21) sqltoken ::= commalesssqltoken */
+    22,  /* (22) marglist ::= marglistinner */
+    22,  /* (23) marglist ::= */
+    23,  /* (24) marglistinner ::= marglistinner COMMA marg */
+    23,  /* (25) marglistinner ::= marg */
+};
+
+/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number
+** of symbols on the right-hand side of that rule. */
+static const signed char yyRuleInfoNRhs[] = {
+   -1,  /* (0) input ::= cmd */
+   -7,  /* (1) apply ::= APPLY COMMA|AND TRUE|FALSE ID LP applylist RP */
+  -10,  /* (2) apply ::= APPLY COMMA|AND TRUE|FALSE ID LP applylist RP LP applylist RP */
+   -3,  /* (3) applylist ::= applylist COMMA tokenlist */
+   -1,  /* (4) applylist ::= tokenlist */
+    0,  /* (5) applylist ::= */
+   -1,  /* (6) commalesssqltoken ::= OPAQUE */
+   -5,  /* (7) commalesssqltoken ::= minvocationid EXCLAIM LP marglist RP */
+   -3,  /* (8) commalesssqltoken ::= LP sql RP */
+   -2,  /* (9) commalesssqltoken ::= LP RP */
+   -1,  /* (10) commalesssqltoken ::= ID */
+   -1,  /* (11) commalesssqltoken ::= VARIABLE */
+   -1,  /* (12) minvocationid ::= ID */
+   -1,  /* (13) marg ::= tokenlist */
+   -2,  /* (14) tokenlist ::= tokenlist commalesssqltoken */
+   -1,  /* (15) tokenlist ::= commalesssqltoken */
+   -2,  /* (16) cmd ::= sql SEMI */
+   -2,  /* (17) cmd ::= apply SEMI */
+   -2,  /* (18) sql ::= sql sqltoken */
+   -1,  /* (19) sql ::= sqltoken */
+   -1,  /* (20) sqltoken ::= COMMA */
+   -1,  /* (21) sqltoken ::= commalesssqltoken */
+   -1,  /* (22) marglist ::= marglistinner */
+    0,  /* (23) marglist ::= */
+   -3,  /* (24) marglistinner ::= marglistinner COMMA marg */
+   -1,  /* (25) marglistinner ::= marg */
+};
+
+static void yy_accept(yyParser*);  /* Forward Declaration */
+
+/*
+** Perform a reduce action and the shift that must immediately
+** follow the reduce.
+**
+** The yyLookahead and yyLookaheadToken parameters provide reduce actions
+** access to the lookahead token (if any).  The yyLookahead will be YYNOCODE
+** if the lookahead token has already been consumed.  As this procedure is
+** only called from one place, optimizing compilers will in-line it, which
+** means that the extra parameters have no performance impact.
+*/
+static YYACTIONTYPE yy_reduce(
+  yyParser *yypParser,         /* The parser */
+  unsigned int yyruleno,       /* Number of the rule by which to reduce */
+  int yyLookahead,             /* Lookahead token, or YYNOCODE if none */
+  PreprocessorGrammarParseTOKENTYPE yyLookaheadToken  /* Value of the lookahead token */
+  PreprocessorGrammarParseCTX_PDECL                   /* %extra_context */
+){
+  int yygoto;                     /* The next state */
+  YYACTIONTYPE yyact;             /* The next action */
+  yyStackEntry *yymsp;            /* The top of the parser's stack */
+  int yysize;                     /* Amount to pop the stack */
+  PreprocessorGrammarParseARG_FETCH
+  (void)yyLookahead;
+  (void)yyLookaheadToken;
+  yymsp = yypParser->yytos;
+
+  switch( yyruleno ){
+  /* Beginning here are the reduction cases.  A typical example
+  ** follows:
+  **   case 0:
+  **  #line <lineno> <grammarfile>
+  **     { ... }           // User supplied code
+  **  #line <lineno> <thisfile>
+  **     break;
+  */
+/********** Begin reduce actions **********************************************/
+        YYMINORTYPE yylhsminor;
+      case 0: /* input ::= cmd */
+{
+  OnPreprocessorEnd(state);
+}
+        break;
+      case 1: /* apply ::= APPLY COMMA|AND TRUE|FALSE ID LP applylist RP */
+{
+  OnPreprocessorApply(state, &yymsp[-3].minor.yy0, &yymsp[-5].minor.yy0, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy2, 0);
+}
+        break;
+      case 2: /* apply ::= APPLY COMMA|AND TRUE|FALSE ID LP applylist RP LP applylist RP */
+{
+  OnPreprocessorApply(state, &yymsp[-6].minor.yy0, &yymsp[-8].minor.yy0, &yymsp[-7].minor.yy0, yymsp[-4].minor.yy2, yymsp[-1].minor.yy2);
+}
+        break;
+      case 3: /* applylist ::= applylist COMMA tokenlist */
+{
+  yylhsminor.yy2 = OnPreprocessorAppendApplyList(yymsp[-2].minor.yy2, &yymsp[0].minor.yy32);
+}
+  yymsp[-2].minor.yy2 = yylhsminor.yy2;
+        break;
+      case 4: /* applylist ::= tokenlist */
+{
+  yylhsminor.yy2 = OnPreprocessorAppendApplyList(OnPreprocessorCreateApplyList(), &yymsp[0].minor.yy32);
+}
+  yymsp[0].minor.yy2 = yylhsminor.yy2;
+        break;
+      case 5: /* applylist ::= */
+{
+  yymsp[1].minor.yy2 = OnPreprocessorCreateApplyList();
+}
+        break;
+      case 6: /* commalesssqltoken ::= OPAQUE */
+      case 10: /* commalesssqltoken ::= ID */ yytestcase(yyruleno==10);
+{
+  yylhsminor.yy32 = (struct PreprocessorGrammarTokenBounds) {yymsp[0].minor.yy0, yymsp[0].minor.yy0};
+}
+  yymsp[0].minor.yy32 = yylhsminor.yy32;
+        break;
+      case 7: /* commalesssqltoken ::= minvocationid EXCLAIM LP marglist RP */
+{
+  yylhsminor.yy32 = (struct PreprocessorGrammarTokenBounds) {yymsp[-4].minor.yy0, yymsp[0].minor.yy0};
+  OnPreprocessorMacroEnd(state, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0);
+}
+  yymsp[-4].minor.yy32 = yylhsminor.yy32;
+        break;
+      case 8: /* commalesssqltoken ::= LP sql RP */
+{
+  yylhsminor.yy32 = (struct PreprocessorGrammarTokenBounds) {yymsp[-2].minor.yy0, yymsp[0].minor.yy0};
+}
+  yymsp[-2].minor.yy32 = yylhsminor.yy32;
+        break;
+      case 9: /* commalesssqltoken ::= LP RP */
+{
+  yylhsminor.yy32 = (struct PreprocessorGrammarTokenBounds) {yymsp[-1].minor.yy0, yymsp[0].minor.yy0};
+}
+  yymsp[-1].minor.yy32 = yylhsminor.yy32;
+        break;
+      case 11: /* commalesssqltoken ::= VARIABLE */
+{
+  yylhsminor.yy32 = (struct PreprocessorGrammarTokenBounds) {yymsp[0].minor.yy0, yymsp[0].minor.yy0};
+  OnPreprocessorVariable(state, &yymsp[0].minor.yy0);
+}
+  yymsp[0].minor.yy32 = yylhsminor.yy32;
+        break;
+      case 12: /* minvocationid ::= ID */
+{
+  yylhsminor.yy0 = yymsp[0].minor.yy0;
+  OnPreprocessorMacroId(state, &yymsp[0].minor.yy0);
+}
+  yymsp[0].minor.yy0 = yylhsminor.yy0;
+        break;
+      case 13: /* marg ::= tokenlist */
+{
+  OnPreprocessorMacroArg(state, &yymsp[0].minor.yy32);
+}
+        break;
+      case 14: /* tokenlist ::= tokenlist commalesssqltoken */
+{
+  yylhsminor.yy32 = (struct PreprocessorGrammarTokenBounds) {yymsp[-1].minor.yy32.start, yymsp[0].minor.yy32.end};
+}
+  yymsp[-1].minor.yy32 = yylhsminor.yy32;
+        break;
+      case 15: /* tokenlist ::= commalesssqltoken */
+{
+  yylhsminor.yy32 = (struct PreprocessorGrammarTokenBounds) {yymsp[0].minor.yy32.start, yymsp[0].minor.yy32.end};
+}
+  yymsp[0].minor.yy32 = yylhsminor.yy32;
+        break;
+      default:
+      /* (16) cmd ::= sql SEMI */ yytestcase(yyruleno==16);
+      /* (17) cmd ::= apply SEMI */ yytestcase(yyruleno==17);
+      /* (18) sql ::= sql sqltoken */ yytestcase(yyruleno==18);
+      /* (19) sql ::= sqltoken (OPTIMIZED OUT) */ assert(yyruleno!=19);
+      /* (20) sqltoken ::= COMMA */ yytestcase(yyruleno==20);
+      /* (21) sqltoken ::= commalesssqltoken (OPTIMIZED OUT) */ assert(yyruleno!=21);
+      /* (22) marglist ::= marglistinner */ yytestcase(yyruleno==22);
+      /* (23) marglist ::= */ yytestcase(yyruleno==23);
+      /* (24) marglistinner ::= marglistinner COMMA marg */ yytestcase(yyruleno==24);
+      /* (25) marglistinner ::= marg (OPTIMIZED OUT) */ assert(yyruleno!=25);
+        break;
+/********** End reduce actions ************************************************/
+  };
+  assert( yyruleno<sizeof(yyRuleInfoLhs)/sizeof(yyRuleInfoLhs[0]) );
+  yygoto = yyRuleInfoLhs[yyruleno];
+  yysize = yyRuleInfoNRhs[yyruleno];
+  yyact = yy_find_reduce_action(yymsp[yysize].stateno,(YYCODETYPE)yygoto);
+
+  /* There are no SHIFTREDUCE actions on nonterminals because the table
+  ** generator has simplified them to pure REDUCE actions. */
+  assert( !(yyact>YY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) );
+
+  /* It is not possible for a REDUCE to be followed by an error */
+  assert( yyact!=YY_ERROR_ACTION );
+
+  yymsp += yysize+1;
+  yypParser->yytos = yymsp;
+  yymsp->stateno = (YYACTIONTYPE)yyact;
+  yymsp->major = (YYCODETYPE)yygoto;
+  yyTraceShift(yypParser, yyact, "... then shift");
+  return yyact;
+}
+
+/*
+** The following code executes when the parse fails
+*/
+#ifndef YYNOERRORRECOVERY
+static void yy_parse_failed(
+  yyParser *yypParser           /* The parser */
+){
+  PreprocessorGrammarParseARG_FETCH
+  PreprocessorGrammarParseCTX_FETCH
+#ifndef NDEBUG
+  if( yyTraceFILE ){
+    fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt);
+  }
+#endif
+  while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser);
+  /* Here code is inserted which will be executed whenever the
+  ** parser fails */
+/************ Begin %parse_failure code ***************************************/
+/************ End %parse_failure code *****************************************/
+  PreprocessorGrammarParseARG_STORE /* Suppress warning about unused %extra_argument variable */
+  PreprocessorGrammarParseCTX_STORE
+}
+#endif /* YYNOERRORRECOVERY */
+
+/*
+** The following code executes when a syntax error first occurs.
+*/
+static void yy_syntax_error(
+  yyParser *yypParser,           /* The parser */
+  int yymajor,                   /* The major type of the error token */
+  PreprocessorGrammarParseTOKENTYPE yyminor         /* The minor type of the error token */
+){
+  PreprocessorGrammarParseARG_FETCH
+  PreprocessorGrammarParseCTX_FETCH
+#define TOKEN yyminor
+/************ Begin %syntax_error code ****************************************/
+
+  OnPreprocessorSyntaxError(state, &yyminor);
+/************ End %syntax_error code ******************************************/
+  PreprocessorGrammarParseARG_STORE /* Suppress warning about unused %extra_argument variable */
+  PreprocessorGrammarParseCTX_STORE
+}
+
+/*
+** The following is executed when the parser accepts
+*/
+static void yy_accept(
+  yyParser *yypParser           /* The parser */
+){
+  PreprocessorGrammarParseARG_FETCH
+  PreprocessorGrammarParseCTX_FETCH
+#ifndef NDEBUG
+  if( yyTraceFILE ){
+    fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt);
+  }
+#endif
+#ifndef YYNOERRORRECOVERY
+  yypParser->yyerrcnt = -1;
+#endif
+  assert( yypParser->yytos==yypParser->yystack );
+  /* Here code is inserted which will be executed whenever the
+  ** parser accepts */
+/*********** Begin %parse_accept code *****************************************/
+/*********** End %parse_accept code *******************************************/
+  PreprocessorGrammarParseARG_STORE /* Suppress warning about unused %extra_argument variable */
+  PreprocessorGrammarParseCTX_STORE
+}
+
+/* The main parser program.
+** The first argument is a pointer to a structure obtained from
+** "PreprocessorGrammarParseAlloc" which describes the current state of the parser.
+** The second argument is the major token number.  The third is
+** the minor token.  The fourth optional argument is whatever the
+** user wants (and specified in the grammar) and is available for
+** use by the action routines.
+**
+** Inputs:
+** <ul>
+** <li> A pointer to the parser (an opaque structure.)
+** <li> The major token number.
+** <li> The minor token number.
+** <li> An option argument of a grammar-specified type.
+** </ul>
+**
+** Outputs:
+** None.
+*/
+void PreprocessorGrammarParse(
+  void *yyp,                   /* The parser */
+  int yymajor,                 /* The major token code number */
+  PreprocessorGrammarParseTOKENTYPE yyminor       /* The value for the token */
+  PreprocessorGrammarParseARG_PDECL               /* Optional %extra_argument parameter */
+){
+  YYMINORTYPE yyminorunion;
+  YYACTIONTYPE yyact;   /* The parser action. */
+#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
+  int yyendofinput;     /* True if we are at the end of input */
+#endif
+#ifdef YYERRORSYMBOL
+  int yyerrorhit = 0;   /* True if yymajor has invoked an error */
+#endif
+  yyParser *yypParser = (yyParser*)yyp;  /* The parser */
+  PreprocessorGrammarParseCTX_FETCH
+  PreprocessorGrammarParseARG_STORE
+
+  assert( yypParser->yytos!=0 );
+#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
+  yyendofinput = (yymajor==0);
+#endif
+
+  yyact = yypParser->yytos->stateno;
+#ifndef NDEBUG
+  if( yyTraceFILE ){
+    if( yyact < YY_MIN_REDUCE ){
+      fprintf(yyTraceFILE,"%sInput '%s' in state %d\n",
+              yyTracePrompt,yyTokenName[yymajor],yyact);
+    }else{
+      fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n",
+              yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE);
+    }
+  }
+#endif
+
+  while(1){ /* Exit by "break" */
+    assert( yypParser->yytos>=yypParser->yystack );
+    assert( yyact==yypParser->yytos->stateno );
+    yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact);
+    if( yyact >= YY_MIN_REDUCE ){
+      unsigned int yyruleno = yyact - YY_MIN_REDUCE; /* Reduce by this rule */
+#ifndef NDEBUG
+      assert( yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) );
+      if( yyTraceFILE ){
+        int yysize = yyRuleInfoNRhs[yyruleno];
+        if( yysize ){
+          fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n",
+            yyTracePrompt,
+            yyruleno, yyRuleName[yyruleno],
+            yyruleno<YYNRULE_WITH_ACTION ? "" : " without external action",
+            yypParser->yytos[yysize].stateno);
+        }else{
+          fprintf(yyTraceFILE, "%sReduce %d [%s]%s.\n",
+            yyTracePrompt, yyruleno, yyRuleName[yyruleno],
+            yyruleno<YYNRULE_WITH_ACTION ? "" : " without external action");
+        }
+      }
+#endif /* NDEBUG */
+
+      /* Check that the stack is large enough to grow by a single entry
+      ** if the RHS of the rule is empty.  This ensures that there is room
+      ** enough on the stack to push the LHS value */
+      if( yyRuleInfoNRhs[yyruleno]==0 ){
+#ifdef YYTRACKMAXSTACKDEPTH
+        if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
+          yypParser->yyhwm++;
+          assert( yypParser->yyhwm ==
+                  (int)(yypParser->yytos - yypParser->yystack));
+        }
+#endif
+#if YYSTACKDEPTH>0 
+        if( yypParser->yytos>=yypParser->yystackEnd ){
+          yyStackOverflow(yypParser);
+          break;
+        }
+#else
+        if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){
+          if( yyGrowStack(yypParser) ){
+            yyStackOverflow(yypParser);
+            break;
+          }
+        }
+#endif
+      }
+      yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor PreprocessorGrammarParseCTX_PARAM);
+    }else if( yyact <= YY_MAX_SHIFTREDUCE ){
+      yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor);
+#ifndef YYNOERRORRECOVERY
+      yypParser->yyerrcnt--;
+#endif
+      break;
+    }else if( yyact==YY_ACCEPT_ACTION ){
+      yypParser->yytos--;
+      yy_accept(yypParser);
+      return;
+    }else{
+      assert( yyact == YY_ERROR_ACTION );
+      yyminorunion.yy0 = yyminor;
+#ifdef YYERRORSYMBOL
+      int yymx;
+#endif
+#ifndef NDEBUG
+      if( yyTraceFILE ){
+        fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt);
+      }
+#endif
+#ifdef YYERRORSYMBOL
+      /* A syntax error has occurred.
+      ** The response to an error depends upon whether or not the
+      ** grammar defines an error token "ERROR".  
+      **
+      ** This is what we do if the grammar does define ERROR:
+      **
+      **  * Call the %syntax_error function.
+      **
+      **  * Begin popping the stack until we enter a state where
+      **    it is legal to shift the error symbol, then shift
+      **    the error symbol.
+      **
+      **  * Set the error count to three.
+      **
+      **  * Begin accepting and shifting new tokens.  No new error
+      **    processing will occur until three tokens have been
+      **    shifted successfully.
+      **
+      */
+      if( yypParser->yyerrcnt<0 ){
+        yy_syntax_error(yypParser,yymajor,yyminor);
+      }
+      yymx = yypParser->yytos->major;
+      if( yymx==YYERRORSYMBOL || yyerrorhit ){
+#ifndef NDEBUG
+        if( yyTraceFILE ){
+          fprintf(yyTraceFILE,"%sDiscard input token %s\n",
+             yyTracePrompt,yyTokenName[yymajor]);
+        }
+#endif
+        yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion);
+        yymajor = YYNOCODE;
+      }else{
+        while( yypParser->yytos > yypParser->yystack ){
+          yyact = yy_find_reduce_action(yypParser->yytos->stateno,
+                                        YYERRORSYMBOL);
+          if( yyact<=YY_MAX_SHIFTREDUCE ) break;
+          yy_pop_parser_stack(yypParser);
+        }
+        if( yypParser->yytos <= yypParser->yystack || yymajor==0 ){
+          yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
+          yy_parse_failed(yypParser);
+#ifndef YYNOERRORRECOVERY
+          yypParser->yyerrcnt = -1;
+#endif
+          yymajor = YYNOCODE;
+        }else if( yymx!=YYERRORSYMBOL ){
+          yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor);
+        }
+      }
+      yypParser->yyerrcnt = 3;
+      yyerrorhit = 1;
+      if( yymajor==YYNOCODE ) break;
+      yyact = yypParser->yytos->stateno;
+#elif defined(YYNOERRORRECOVERY)
+      /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to
+      ** do any kind of error recovery.  Instead, simply invoke the syntax
+      ** error routine and continue going as if nothing had happened.
+      **
+      ** Applications can set this macro (for example inside %include) if
+      ** they intend to abandon the parse upon the first syntax error seen.
+      */
+      yy_syntax_error(yypParser,yymajor, yyminor);
+      yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
+      break;
+#else  /* YYERRORSYMBOL is not defined */
+      /* This is what we do if the grammar does not define ERROR:
+      **
+      **  * Report an error message, and throw away the input token.
+      **
+      **  * If the input token is $, then fail the parse.
+      **
+      ** As before, subsequent error messages are suppressed until
+      ** three input tokens have been successfully shifted.
+      */
+      if( yypParser->yyerrcnt<=0 ){
+        yy_syntax_error(yypParser,yymajor, yyminor);
+      }
+      yypParser->yyerrcnt = 3;
+      yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
+      if( yyendofinput ){
+        yy_parse_failed(yypParser);
+#ifndef YYNOERRORRECOVERY
+        yypParser->yyerrcnt = -1;
+#endif
+      }
+      break;
+#endif
+    }
+  }
+#ifndef NDEBUG
+  if( yyTraceFILE ){
+    yyStackEntry *i;
+    char cDiv = '[';
+    fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt);
+    for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){
+      fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]);
+      cDiv = ' ';
+    }
+    fprintf(yyTraceFILE,"]\n");
+  }
+#endif
+  return;
+}
+
+/*
+** Return the fallback token corresponding to canonical token iToken, or
+** 0 if iToken has no fallback.
+*/
+int PreprocessorGrammarParseFallback(int iToken){
+#ifdef YYFALLBACK
+  assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) );
+  return yyFallback[iToken];
+#else
+  (void)iToken;
+  return 0;
+#endif
+}
diff --git a/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.h b/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.h
new file mode 100644
index 0000000..6cb021e
--- /dev/null
+++ b/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.h
@@ -0,0 +1,12 @@
+#define PPTK_SEMI                             1
+#define PPTK_APPLY                            2
+#define PPTK_COMMA                            3
+#define PPTK_AND                              4
+#define PPTK_TRUE                             5
+#define PPTK_FALSE                            6
+#define PPTK_ID                               7
+#define PPTK_LP                               8
+#define PPTK_RP                               9
+#define PPTK_OPAQUE                          10
+#define PPTK_EXCLAIM                         11
+#define PPTK_VARIABLE                        12
diff --git a/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.y b/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.y
new file mode 100644
index 0000000..85966c9
--- /dev/null
+++ b/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.y
@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+%name PreprocessorGrammarParse
+%token_prefix PPTK_
+%start_symbol input
+
+%include {
+#include "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar_interface.h"
+
+#define YYNOERRORRECOVERY 1
+#define YYPARSEFREENEVERNULL 1
+}
+
+%token_type {struct PreprocessorGrammarToken}
+%default_type {struct PreprocessorGrammarTokenBounds}
+
+%extra_context {struct PreprocessorGrammarState* state}
+%syntax_error {
+  OnPreprocessorSyntaxError(state, &yyminor);
+}
+
+input ::= cmd.  {
+  OnPreprocessorEnd(state);
+}
+
+cmd ::= sql SEMI.
+cmd ::= apply SEMI.
+
+apply ::= APPLY COMMA|AND(J) TRUE|FALSE(P) ID(X) LP applylist(Y) RP. {
+  OnPreprocessorApply(state, &X, &J, &P, Y, 0);
+}
+apply ::= APPLY COMMA|AND(J) TRUE|FALSE(P) ID(X) LP applylist(Y) RP LP applylist(Z) RP. {
+  OnPreprocessorApply(state, &X, &J, &P, Y, Z);
+}
+
+%type applylist {struct PreprocessorGrammarApplyList*}
+%destructor applylist { OnPreprocessorFreeApplyList($$); }
+applylist(A) ::= applylist(F) COMMA tokenlist(X). {
+  A = OnPreprocessorAppendApplyList(F, &X);
+}
+applylist(A) ::= tokenlist(F). {
+  A = OnPreprocessorAppendApplyList(OnPreprocessorCreateApplyList(), &F);
+}
+applylist(A) ::=. {
+  A = OnPreprocessorCreateApplyList();
+}
+
+sql ::= sql sqltoken.
+sql ::= sqltoken.
+
+sqltoken ::= COMMA.
+sqltoken ::= commalesssqltoken.
+
+commalesssqltoken(A) ::= OPAQUE(X). {
+  A = (struct PreprocessorGrammarTokenBounds) {X, X};
+}
+commalesssqltoken(A) ::= minvocationid(S) EXCLAIM LP marglist RP(F). {
+  A = (struct PreprocessorGrammarTokenBounds) {S, F};
+  OnPreprocessorMacroEnd(state, &S, &F);
+}
+commalesssqltoken(A) ::= LP(S) sql RP(F).  {
+  A = (struct PreprocessorGrammarTokenBounds) {S, F};
+}
+commalesssqltoken(A) ::= LP(S) RP(F). {
+  A = (struct PreprocessorGrammarTokenBounds) {S, F};
+}
+commalesssqltoken(A) ::= ID(X). {
+  A = (struct PreprocessorGrammarTokenBounds) {X, X};
+}
+commalesssqltoken(A) ::= VARIABLE(X). {
+  A = (struct PreprocessorGrammarTokenBounds) {X, X};
+  OnPreprocessorVariable(state, &X);
+}
+
+%type minvocationid {struct PreprocessorGrammarToken}
+minvocationid(A) ::= ID(X). {
+  A = X;
+  OnPreprocessorMacroId(state, &X);
+}
+
+marglist ::= marglistinner.
+marglist ::=.
+
+marglistinner ::= marglistinner COMMA marg.
+marglistinner ::= marg.
+
+marg ::= tokenlist(X). {
+  OnPreprocessorMacroArg(state, &X);
+}
+
+tokenlist(A) ::= tokenlist(S) commalesssqltoken(F). {
+  A = (struct PreprocessorGrammarTokenBounds) {S.start, F.end};
+}
+tokenlist(A) ::= commalesssqltoken(X). {
+  A = (struct PreprocessorGrammarTokenBounds) {X.start, X.end};
+}
diff --git a/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar_interface.h b/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar_interface.h
new file mode 100644
index 0000000..315d97d
--- /dev/null
+++ b/src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar_interface.h
@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_PREPROCESSOR_PREPROCESSOR_GRAMMAR_INTERFACE_H_
+#define SRC_TRACE_PROCESSOR_PERFETTO_SQL_PREPROCESSOR_PREPROCESSOR_GRAMMAR_INTERFACE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stddef.h>
+#include <stdio.h>
+
+#include "src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.h"
+
+#undef NDEBUG
+
+#ifdef __cplusplus
+namespace perfetto::trace_processor {
+namespace {
+#endif
+
+struct PreprocessorGrammarState;
+
+struct PreprocessorGrammarToken {
+  const char* ptr;
+  size_t n;
+  int major;
+};
+
+struct PreprocessorGrammarTokenBounds {
+  struct PreprocessorGrammarToken start;
+  struct PreprocessorGrammarToken end;
+};
+
+struct PreprocessorGrammarApplyList;
+
+void* PreprocessorGrammarParseAlloc(void* (*)(size_t),
+                                    struct PreprocessorGrammarState*);
+void PreprocessorGrammarParse(void* parser,
+                              int,
+                              struct PreprocessorGrammarToken);
+void PreprocessorGrammarParseFree(void* parser, void (*)(void*));
+void PreprocessorGrammarParseTrace(FILE*, char*);
+
+void OnPreprocessorSyntaxError(struct PreprocessorGrammarState*,
+                               struct PreprocessorGrammarToken*);
+void OnPreprocessorApply(struct PreprocessorGrammarState*,
+                         struct PreprocessorGrammarToken* name,
+                         struct PreprocessorGrammarToken* join,
+                         struct PreprocessorGrammarToken* prefix,
+                         struct PreprocessorGrammarApplyList*,
+                         struct PreprocessorGrammarApplyList*);
+void OnPreprocessorVariable(struct PreprocessorGrammarState*,
+                            struct PreprocessorGrammarToken* var);
+void OnPreprocessorMacroId(struct PreprocessorGrammarState*,
+                           struct PreprocessorGrammarToken* name);
+void OnPreprocessorMacroArg(struct PreprocessorGrammarState*,
+                            struct PreprocessorGrammarTokenBounds*);
+void OnPreprocessorMacroEnd(struct PreprocessorGrammarState*,
+                            struct PreprocessorGrammarToken* name,
+                            struct PreprocessorGrammarToken* rp);
+void OnPreprocessorEnd(struct PreprocessorGrammarState*);
+
+struct PreprocessorGrammarApplyList* OnPreprocessorCreateApplyList();
+struct PreprocessorGrammarApplyList* OnPreprocessorAppendApplyList(
+    struct PreprocessorGrammarApplyList*,
+    struct PreprocessorGrammarTokenBounds*);
+void OnPreprocessorFreeApplyList(struct PreprocessorGrammarApplyList*);
+
+#ifdef __cplusplus
+}
+}
+}
+#endif
+
+#endif  // SRC_TRACE_PROCESSOR_PERFETTO_SQL_PREPROCESSOR_PREPROCESSOR_GRAMMAR_INTERFACE_H_
diff --git a/src/trace_processor/perfetto_sql/stdlib/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/BUILD.gn
index d351a31..e3df2f4 100644
--- a/src/trace_processor/perfetto_sql/stdlib/BUILD.gn
+++ b/src/trace_processor/perfetto_sql/stdlib/BUILD.gn
@@ -29,7 +29,6 @@
     "graphs",
     "intervals",
     "linux",
-    "metasql",
     "pkvm",
     "prelude",
     "sched",
diff --git a/src/trace_processor/perfetto_sql/stdlib/graphs/scan.sql b/src/trace_processor/perfetto_sql/stdlib/graphs/scan.sql
index 320a73b..b272af2 100644
--- a/src/trace_processor/perfetto_sql/stdlib/graphs/scan.sql
+++ b/src/trace_processor/perfetto_sql/stdlib/graphs/scan.sql
@@ -51,11 +51,10 @@
 (
   select
     c0 as id,
-    __intrinsic_token_zip_join!(
-      (c1, c2, c3, c4, c5, c6, c7),
-      $scan_columns,
+    __intrinsic_token_apply!(
       _graph_scan_select,
-      __intrinsic_token_comma!()
+      (c1, c2, c3, c4, c5, c6, c7),
+      $scan_columns
     )
   from  __intrinsic_table_ptr(__intrinsic_graph_scan(
     (
@@ -65,24 +64,22 @@
     (
       select __intrinsic_row_dataframe_agg(
         'id', init_table.id,
-        __intrinsic_token_zip_join!(
-          $scan_columns,
-          $scan_columns,
+        __intrinsic_token_apply!(
           _graph_scan_df_agg,
-          __intrinsic_token_comma!()
+          $scan_columns,
+          $scan_columns
         )
       )
       from $init_table AS init_table
     ),
-    __intrinsic_stringify!($step_query, table),
+    __intrinsic_stringify_ignore_table!($step_query),
     __intrinsic_stringify!($scan_columns)
   )) result
   where __intrinsic_table_ptr_bind(result.c0, 'id')
-    and __intrinsic_token_zip_join!(
-          (c1, c2, c3, c4, c5, c6, c7),
-          $scan_columns,
+    and __intrinsic_token_apply_and!(
           _graph_scan_bind,
-          AND
+          (c1, c2, c3, c4, c5, c6, c7),
+          $scan_columns
         )
 );
 
@@ -113,11 +110,10 @@
 (
   select
     c0 as id,
-    __intrinsic_token_zip_join!(
-      (c1, c2, c3, c4, c5, c6, c7),
-      $agg_columns,
+    __intrinsic_token_apply!(
       _graph_scan_select,
-      __intrinsic_token_comma!()
+      (c1, c2, c3, c4, c5, c6, c7),
+      $agg_columns
     )
   from  __intrinsic_table_ptr(__intrinsic_graph_aggregating_scan(
     (
@@ -127,23 +123,21 @@
     (
       select __intrinsic_row_dataframe_agg(
         'id', init_table.id,
-        __intrinsic_token_zip_join!(
-          $agg_columns,
-          $agg_columns,
+        __intrinsic_token_apply!(
           _graph_scan_df_agg,
-          __intrinsic_token_comma!()
+          $agg_columns,
+          $agg_columns
         )
       )
       from $init_table AS init_table
     ),
-    __intrinsic_stringify!($agg_query, table),
+    __intrinsic_stringify_ignore_table!($agg_query),
     __intrinsic_stringify!($agg_columns)
   )) result
   where __intrinsic_table_ptr_bind(result.c0, 'id')
-    and __intrinsic_token_zip_join!(
-          (c1, c2, c3, c4, c5, c6, c7),
-          $agg_columns,
+    and __intrinsic_token_apply_and!(
           _graph_scan_bind,
-          AND
+          (c1, c2, c3, c4, c5, c6, c7),
+          $agg_columns
         )
 );
diff --git a/src/trace_processor/perfetto_sql/stdlib/intervals/intersect.sql b/src/trace_processor/perfetto_sql/stdlib/intervals/intersect.sql
index 08d9808..c62a1a5 100644
--- a/src/trace_processor/perfetto_sql/stdlib/intervals/intersect.sql
+++ b/src/trace_processor/perfetto_sql/stdlib/intervals/intersect.sql
@@ -13,8 +13,6 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 
-INCLUDE PERFETTO MODULE metasql.table_list;
-
 CREATE PERFETTO MACRO _ii_df_agg(x Expr, y Expr)
 RETURNS Expr AS __intrinsic_stringify!($x), input.$y;
 
@@ -37,11 +35,10 @@
     input.id,
     input.ts,
     input.dur
-    __intrinsic_prefixed_token_zip_join!(
-      $agg_columns,
-      $agg_columns,
+    __intrinsic_token_apply_prefix!(
       _ii_df_agg,
-      __intrinsic_token_comma!()
+      $agg_columns,
+      $agg_columns
     )
   )
   FROM (SELECT * FROM $tab ORDER BY ts) input
@@ -57,24 +54,25 @@
     c0 AS ts,
     c1 AS dur,
     -- Columns for tables ids, in the order of provided tables.
-    __intrinsic_token_zip_join!(
-      (c2 AS id_0, c3 AS id_1, c4 AS id_2, c5 AS id_3, c6 AS id_4),
-      $tabs,
+    __intrinsic_token_apply!(
       __first_arg,
-      __intrinsic_token_comma!()
+      (c2 AS id_0, c3 AS id_1, c4 AS id_2, c5 AS id_3, c6 AS id_4),
+      $tabs
     )
-    -- Columns for partitions, one for each column with partition. Prefixed to
-    -- handle case of no partitions.
-    __intrinsic_prefixed_token_zip_join!(
-      (c7, c8, c9, c10),
-      $agg_columns,
+    -- Columns for partitions, one for each column with partition.
+    __intrinsic_token_apply_prefix!(
       _ii_df_select,
-      __intrinsic_token_comma!()
+      (c7, c8, c9, c10),
+      $agg_columns
     )
   -- Interval intersect result table.
   FROM __intrinsic_table_ptr(
     __intrinsic_interval_intersect(
-      _metasql_map_join_table_list_with_capture!($tabs, _interval_agg, ($agg_columns)),
+      __intrinsic_token_apply!(
+        _interval_agg,
+        $tabs,
+        ($agg_columns, $agg_columns, $agg_columns, $agg_columns, $agg_columns)
+      ),
       __intrinsic_stringify!($agg_columns)
     )
   )
@@ -89,12 +87,11 @@
     AND __intrinsic_table_ptr_bind(c5, 'id_3')
     AND __intrinsic_table_ptr_bind(c6, 'id_4')
 
-    -- Partition columns. Prefixed to handle case of no partitions.
-    __intrinsic_prefixed_token_zip_join!(
-      (c7, c8, c9, c10),
-      $agg_columns,
+    -- Partition columns.
+    __intrinsic_token_apply_and_prefix!(
       _ii_df_bind,
-      AND
+      (c7, c8, c9, c10),
+      $agg_columns
     )
 );
 
diff --git a/src/trace_processor/perfetto_sql/stdlib/metasql/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/metasql/BUILD.gn
deleted file mode 100644
index a82b2d2..0000000
--- a/src/trace_processor/perfetto_sql/stdlib/metasql/BUILD.gn
+++ /dev/null
@@ -1,22 +0,0 @@
-# Copyright (C) 2024 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import("../../../../../gn/perfetto_sql.gni")
-
-perfetto_sql_source_set("metasql") {
-  sources = [
-    "column_list.sql",
-    "table_list.sql",
-  ]
-}
diff --git a/src/trace_processor/perfetto_sql/stdlib/metasql/column_list.sql b/src/trace_processor/perfetto_sql/stdlib/metasql/column_list.sql
deleted file mode 100644
index c78aab5..0000000
--- a/src/trace_processor/perfetto_sql/stdlib/metasql/column_list.sql
+++ /dev/null
@@ -1,34 +0,0 @@
---
--- Copyright 2024 The Android Open Source Project
---
--- Licensed under the Apache License, Version 2.0 (the "License");
--- you may not use this file except in compliance with the License.
--- You may obtain a copy of the License at
---
---     https://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
-
-CREATE PERFETTO MACRO _col_list_id(a ColumnName)
-RETURNS _SqlFragment AS $a;
-
--- Given a list of column names, applies an arbitrary macro to each column
--- and joins the result with a comma.
-CREATE PERFETTO MACRO _metasql_map_join_column_list(
-  columns _ColumnNameList,
-  map_macro _Macro
-)
-RETURNS _SqlFragment
-AS __intrinsic_token_map_join!($columns, $map_macro, __intrinsic_token_comma!());
-
--- Given a list of column names, removes the parentheses allowing the usage
--- of these in a select statement, window function etc.
-CREATE PERFETTO MACRO _metasql_unparenthesize_column_list(
-  columns _ColumnNameList
-)
-RETURNS _SqlFragment
-AS _metasql_map_join_column_list!($columns, _col_list_id);
diff --git a/src/trace_processor/perfetto_sql/stdlib/metasql/table_list.sql b/src/trace_processor/perfetto_sql/stdlib/metasql/table_list.sql
deleted file mode 100644
index 32d327b..0000000
--- a/src/trace_processor/perfetto_sql/stdlib/metasql/table_list.sql
+++ /dev/null
@@ -1,33 +0,0 @@
---
--- Copyright 2024 The Android Open Source Project
---
--- Licensed under the Apache License, Version 2.0 (the "License");
--- you may not use this file except in compliance with the License.
--- You may obtain a copy of the License at
---
---     https://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
-
--- Given a list of table names, applies an arbitrary macro to each table
--- and joins the result with a comma.
-CREATE PERFETTO MACRO _metasql_map_join_table_list(
-  tables _TableNameList,
-  map_macro _Macro
-)
-RETURNS _SqlFragment
-AS __intrinsic_token_map_join!($tables, $map_macro, __intrinsic_token_comma!());
-
--- Given a list of table names, applies an arbitrary macro to each table
--- and joins the result with a comma.
-CREATE PERFETTO MACRO _metasql_map_join_table_list_with_capture(
-  tables _TableNameList,
-  map_macro _Macro,
-  args _ArgumentList
-)
-RETURNS _SqlFragment
-AS __intrinsic_token_map_join_with_capture!($tables, $map_macro, $args, __intrinsic_token_comma!());
diff --git a/src/trace_processor/perfetto_sql/stdlib/viz/flamegraph.sql b/src/trace_processor/perfetto_sql/stdlib/viz/flamegraph.sql
index cd796cb..3dcdc3a 100644
--- a/src/trace_processor/perfetto_sql/stdlib/viz/flamegraph.sql
+++ b/src/trace_processor/perfetto_sql/stdlib/viz/flamegraph.sql
@@ -14,7 +14,6 @@
 -- limitations under the License.
 
 INCLUDE PERFETTO MODULE graphs.scan;
-INCLUDE PERFETTO MODULE metasql.column_list;
 
 CREATE PERFETTO MACRO _viz_flamegraph_hash_coalesce(col ColumnName)
 RETURNS _SqlFragment AS IFNULL($col, 0);
@@ -41,7 +40,7 @@
     $pivot AS isPivot,
     HASH(
       name,
-      _metasql_map_join_column_list!($grouping, _viz_flamegraph_hash_coalesce)
+      __intrinsic_token_apply!(_viz_flamegraph_hash_coalesce, $grouping)
     ) AS groupingHash
   FROM $tab
   ORDER BY id
@@ -214,8 +213,8 @@
     g.parentHash,
     g.depth,
     s.name,
-    _metasql_map_join_column_list!($grouping, _viz_flamegraph_s_prefix),
-    _metasql_map_join_column_list!($grouped, _viz_flamegraph_s_prefix),
+    __intrinsic_token_apply!(_viz_flamegraph_s_prefix, $grouping),
+    __intrinsic_token_apply!(_viz_flamegraph_s_prefix, $grouped),
     f.value,
     g.cumulativeValue
   FROM _graph_scan!(
@@ -271,8 +270,8 @@
     g.parentHash,
     g.depth,
     s.name,
-    _metasql_map_join_column_list!($grouping, _viz_flamegraph_s_prefix),
-    _metasql_map_join_column_list!($grouped, _viz_flamegraph_s_prefix),
+    __intrinsic_token_apply!(_viz_flamegraph_s_prefix, $grouping),
+    __intrinsic_token_apply!(_viz_flamegraph_s_prefix, $grouped),
     f.value,
     a.cumulativeValue
   FROM _graph_scan!(
@@ -301,6 +300,9 @@
 RETURNS _SqlFragment
 AS IIF(COUNT() = 1, $col, NULL) AS $col;
 
+CREATE PERFETTO MACRO _col_list_id(a ColumnName)
+RETURNS _SqlFragment AS $a;
+
 -- Converts a table of hashes and paretn hashes into ids and parent
 -- ids, grouping all hashes together.
 CREATE PERFETTO MACRO _viz_flamegraph_merge_hashes(
@@ -323,8 +325,8 @@
     -- The grouping columns should be passed through as-is because the
     -- hash took them into account: we would not merged any nodes where
     -- the grouping columns were different.
-    _metasql_unparenthesize_column_list!($grouping),
-    _metasql_map_join_column_list!($grouped, _viz_flamegraph_merge_grouped),
+    __intrinsic_token_apply!(_col_list_id, $grouping),
+    __intrinsic_token_apply!(_viz_flamegraph_merge_grouped, $grouped),
     SUM(value) AS value,
     SUM(cumulativeValue) AS cumulativeValue
   FROM $hashed c
@@ -381,8 +383,8 @@
     s.id,
     IFNULL(s.parentId, -1) AS parentId,
     IIF(s.name = '', 'unknown', s.name) AS name,
-    _metasql_map_join_column_list!($grouping, _viz_flamegraph_s_prefix),
-    _metasql_map_join_column_list!($grouped, _viz_flamegraph_s_prefix),
+    __intrinsic_token_apply!(_viz_flamegraph_s_prefix, $grouping),
+    __intrinsic_token_apply!(_viz_flamegraph_s_prefix, $grouped),
     s.value AS selfValue,
     s.cumulativeValue,
     p.cumulativeValue AS parentCumulativeValue,
diff --git a/src/trace_processor/sqlite/sqlite_tokenizer.cc b/src/trace_processor/sqlite/sqlite_tokenizer.cc
index 9b56c52..972857b 100644
--- a/src/trace_processor/sqlite/sqlite_tokenizer.cc
+++ b/src/trace_processor/sqlite/sqlite_tokenizer.cc
@@ -383,7 +383,7 @@
     case CC_X: {
       if (z[1] == '\'') {
         *tokenType = SqliteTokenType::TK_BLOB;
-        for (i = 2; isdigit(z[i]); i++) {
+        for (i = 2; isxdigit(z[i]); i++) {
         }
         if (z[i] != '\'' || i % 2) {
           *tokenType = SqliteTokenType::TK_ILLEGAL;
diff --git a/test/trace_processor/diff_tests/include_index.py b/test/trace_processor/diff_tests/include_index.py
index 14199e0..e522806 100644
--- a/test/trace_processor/diff_tests/include_index.py
+++ b/test/trace_processor/diff_tests/include_index.py
@@ -126,8 +126,6 @@
 from diff_tests.stdlib.linux.cpu import LinuxCpu
 from diff_tests.stdlib.linux.memory import Memory
 from diff_tests.stdlib.linux.tests import LinuxTests
-from diff_tests.stdlib.metasql.column_list import ColumnListTests
-from diff_tests.stdlib.metasql.table_list import TableListTests
 from diff_tests.stdlib.pkvm.tests import Pkvm
 from diff_tests.stdlib.prelude.math_functions_tests import PreludeMathFunctions
 from diff_tests.stdlib.prelude.pprof_functions_tests import PreludePprofFunctions
@@ -303,10 +301,6 @@
                               'StdlibCounterIntervals').fetch(),
       *DynamicTables(index_path, 'stdlib/dynamic_tables',
                      'DynamicTables').fetch(),
-      *ColumnListTests(index_path, 'stdlib/column_list',
-                       'ColumnListTests').fetch(),
-      *TableListTests(index_path, 'stdlib/table_list',
-                      'TableListTests').fetch(),
       *Memory(index_path, 'stdlib/linux', 'Memory').fetch(),
       *PreludeMathFunctions(index_path, 'stdlib/prelude',
                             'PreludeMathFunctions').fetch(),
diff --git a/test/trace_processor/diff_tests/stdlib/metasql/column_list.py b/test/trace_processor/diff_tests/stdlib/metasql/column_list.py
deleted file mode 100644
index 5c4fe6c..0000000
--- a/test/trace_processor/diff_tests/stdlib/metasql/column_list.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (C) 2024 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License a
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from python.generators.diff_tests.testing import Csv, TextProto
-from python.generators.diff_tests.testing import DiffTestBlueprint
-from python.generators.diff_tests.testing import TestSuite
-
-
-class ColumnListTests(TestSuite):
-
-  def test_column_list_result_columns(self):
-    return DiffTestBlueprint(
-        trace=TextProto(''),
-        query="""
-        INCLUDE PERFETTO MODULE metasql.column_list;
-
-        WITH data(foo, bar) AS (
-          VALUES (0, 1)
-        )
-        SELECT _metasql_unparenthesize_column_list!((foo, bar))
-        FROM data
-        """,
-        out=Csv("""
-        "foo","bar"
-        0,1
-        """))
diff --git a/test/trace_processor/diff_tests/stdlib/metasql/table_list.py b/test/trace_processor/diff_tests/stdlib/metasql/table_list.py
deleted file mode 100644
index 503f84c..0000000
--- a/test/trace_processor/diff_tests/stdlib/metasql/table_list.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (C) 2024 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License a
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from python.generators.diff_tests.testing import Csv, TextProto
-from python.generators.diff_tests.testing import DiffTestBlueprint
-from python.generators.diff_tests.testing import TestSuite
-
-
-class TableListTests(TestSuite):
-
-  def test_table_list_result_columns(self):
-    return DiffTestBlueprint(
-        trace=TextProto(''),
-        query="""
-        INCLUDE PERFETTO MODULE metasql.table_list;
-
-        CREATE PERFETTO MACRO mac(t TableOrSubquery)
-        RETURNS TableOrSubquery AS
-        (SELECT * FROM $t);
-
-        WITH foo AS (
-          SELECT 0 AS a
-        ),
-        bar AS (
-          SELECT 1 AS b
-        ),
-        baz AS (
-          SELECT 2 AS c
-        )
-        SELECT a + b + c
-        FROM _metasql_map_join_table_list!((foo, bar, baz), mac);
-        """,
-        out=Csv("""
-        "a + b + c"
-        3
-        """))
-
-  def test_table_list_with_capture(self):
-    return DiffTestBlueprint(
-        trace=TextProto(''),
-        query="""
-        INCLUDE PERFETTO MODULE metasql.table_list;
-
-        CREATE PERFETTO MACRO mac(t TableOrSubquery, x Expr)
-        RETURNS TableOrSubquery AS
-        (SELECT *, $x AS bla FROM $t);
-
-        WITH foo AS (
-          SELECT 0 AS a
-        ),
-        bar AS (
-          SELECT 1 AS b
-        ),
-        baz AS (
-          SELECT 2 AS c
-        )
-        SELECT
-          a + b + c
-        FROM _metasql_map_join_table_list_with_capture!(
-          (foo, bar, baz),
-          mac,
-          (3)
-        );
-        """,
-        out=Csv("""
-        "a + b + c"
-        3
-        """))
diff --git a/tools/fix_include_guards b/tools/fix_include_guards
index 1848eda..f336b1a 100755
--- a/tools/fix_include_guards
+++ b/tools/fix_include_guards
@@ -22,6 +22,10 @@
 from codecs import open
 from compat import xrange
 
+EXCLUDED_FILES = [
+  'src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.h',
+]
+
 
 def fix_guards(fpath, checkonly):
   with open(fpath, 'r', encoding='utf-8') as f:
@@ -74,6 +78,8 @@
         if not name.endswith('.h'):
           continue
         fpath = os.path.join(root, name)
+        if fpath in EXCLUDED_FILES:
+          continue
         num_files_changed += fix_guards(fpath, checkonly)
   if checkonly:
     return 0 if num_files_changed == 0 else 1
diff --git a/tools/update_sql_parsers.py b/tools/update_sql_parsers.py
new file mode 100755
index 0000000..5c82e56
--- /dev/null
+++ b/tools/update_sql_parsers.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import subprocess
+import shutil
+import sys
+import tempfile
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--lemon',
+      default=os.path.join(
+          os.path.normpath('buildtools/sqlite_src/tool/lemon.c')))
+  parser.add_argument(
+      '--lemon-template',
+      default=os.path.join(
+          os.path.normpath('buildtools/sqlite_src/tool/lempar.c')))
+  parser.add_argument(
+      '--clang',
+      default=os.path.join(
+          os.path.normpath('buildtools/linux64/clang/bin/clang')))
+  parser.add_argument(
+      '--preprocessor-grammar',
+      default=os.path.join(
+          os.path.normpath(
+              'src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.y'
+          )),
+  )
+  args = parser.parse_args()
+
+  with tempfile.TemporaryDirectory() as tmp:
+    subprocess.check_call([
+        args.clang,
+        os.path.join(args.lemon), '-o',
+        os.path.join(tmp, 'lemon')
+    ])
+    shutil.copy(args.lemon_template, tmp)
+    subprocess.check_call([
+        os.path.join(tmp, 'lemon'),
+        args.preprocessor_grammar,
+        '-q',
+        '-l',
+        '-s',
+    ])
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())