src/trace_processor/sqlite/sqlite_engine.cc - third_party/perfetto - Git at Google


            
            /*

            
             * Copyright (C) 2023 The Android Open Source Project

            
             *

            
             * Licensed under the Apache License, Version 2.0 (the "License");

            
             * you may not use this file except in compliance with the License.

            
             * You may obtain a copy of the License at

            
             *

            
             *      http://www.apache.org/licenses/LICENSE-2.0

            
             *

            
             * Unless required by applicable law or agreed to in writing, software

            
             * distributed under the License is distributed on an "AS IS" BASIS,

            
             * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

            
             * See the License for the specific language governing permissions and

            
             * limitations under the License.

            
             */


            #include "src/trace_processor/sqlite/sqlite_engine.h"


            #include <sqlite3.h>

            
            #include <cstdint>

            
            #include <optional>

            
            #include <string>

            
            #include <utility>


            #include "perfetto/base/build_config.h"

            
            #include "perfetto/base/logging.h"

            
            #include "perfetto/base/status.h"

            
            #include "perfetto/public/compiler.h"

            
            #include "src/trace_processor/sqlite/scoped_db.h"

            
            #include "src/trace_processor/sqlite/sql_source.h"

            
            #include "src/trace_processor/tp_metatrace.h"


            #include "protos/perfetto/trace_processor/metatrace_categories.pbzero.h"


            // In Android and Chromium tree builds, we don't have the percentile module.

            
            // Just don't include it.

            
            #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)

            
            // defined in sqlite_src/ext/misc/percentile.c

            
            extern "C" int sqlite3_percentile_init(sqlite3* db,

            
                                                   char** error,

            
                                                   const sqlite3_api_routines* api);

            
            #endif  // PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)


            namespace perfetto::trace_processor {

            
            namespace {


            void EnsureSqliteInitialized() {

            
              // sqlite3_initialize isn't actually thread-safe in standalone builds because

            
              // we build with SQLITE_THREADSAFE=0. Ensure it's only called from a single

            
              // thread.

            
              static bool init_once = [] {

            
                // Enabling memstatus causes a lock to be taken on every malloc/free in

            
                // SQLite to update the memory statistics. This can cause massive contention

            
                // in trace processor when multiple instances are used in parallel.

            
                // Fix this by disabling the memstatus API which we don't make use of in

            
                // any case. See b/335019324 for more info on this.

            
                int ret = sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0);


                // As much as it is painful, we need to catch instances of SQLITE_MISUSE

            
                // here against all the advice of the SQLite developers and lalitm@'s

            
                // intuition: SQLITE_MISUSE for sqlite3_config really means: that someone

            
                // else has already initialized SQLite. As we are an embeddable library,

            
                // it's very possible that the process embedding us has initialized SQLite

            
                // in a different way to what we want to do and, if so, we should respect

            
                // their choice.

            
                //

            
                // TODO(lalitm): ideally we would have an sqlite3_is_initialized API we

            
                // could use to gate the above check but that doesn't exist: report this

            
                // issue to SQLite developers and see if such an API could be added. If so

            
                // we can remove this check.

            
                if (ret == SQLITE_MISUSE) {

            
                  return true;

            
                }


                PERFETTO_CHECK(ret == SQLITE_OK);

            
                return sqlite3_initialize() == SQLITE_OK;

            
              }();

            
              PERFETTO_CHECK(init_once);

            
            }


            void InitializeSqlite(sqlite3* db) {

            
              char* error = nullptr;

            
              sqlite3_exec(db, "PRAGMA temp_store=2", nullptr, nullptr, &error);

            
              if (error) {

            
                PERFETTO_FATAL("Error setting pragma temp_store: %s", error);

            
              }

            
            // In Android tree builds, we don't have the percentile module.

            
            #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)

            
              sqlite3_percentile_init(db, &error, nullptr);

            
              if (error) {

            
                PERFETTO_ELOG("Error initializing: %s", error);

            
                sqlite3_free(error);

            
              }

            
            #endif

            
            }


            std::optional<uint32_t> GetErrorOffsetDb(sqlite3* db) {

            
              int offset = sqlite3_error_offset(db);

            
              return offset == -1 ? std::nullopt

            
                                  : std::make_optional(static_cast<uint32_t>(offset));

            
            }


            }  // namespace


            SqliteEngine::SqliteEngine() {

            
              sqlite3* db = nullptr;

            
              EnsureSqliteInitialized();


              // Ensure that we open the database with mutexes disabled: this is because

            
              // trace processor as a whole cannot be used from multiple threads so there is

            
              // no point paying the (potentially significant) cost of mutexes at the SQLite

            
              // level.

            
              static constexpr int kSqliteOpenFlags =

            
                  SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX;

            
              PERFETTO_CHECK(sqlite3_open_v2(":memory:", &db, kSqliteOpenFlags, nullptr) ==

            
                             SQLITE_OK);

            
              InitializeSqlite(db);

            
              db_.reset(db);

            
            }


            SqliteEngine::~SqliteEngine() {

            
              // It is important to unregister any functions that have been registered with

            
              // the database before destroying it. This is because functions can hold onto

            
              // prepared statements, which must be finalized before database destruction.

            
              for (auto it = fn_ctx_.GetIterator(); it; ++it) {

            
                int ret = sqlite3_create_function_v2(db_.get(), it.key().first.c_str(),

            
                                                     it.key().second, SQLITE_UTF8, nullptr,

            
                                                     nullptr, nullptr, nullptr, nullptr);

            
                if (PERFETTO_UNLIKELY(ret != SQLITE_OK)) {

            
                  PERFETTO_FATAL("Failed to drop function: '%s'", it.key().first.c_str());

            
                }

            
              }

            
              fn_ctx_.Clear();

            
            }


            SqliteEngine::PreparedStatement SqliteEngine::PrepareStatement(SqlSource sql) {

            
              PERFETTO_TP_TRACE(metatrace::Category::QUERY_DETAILED, "QUERY_PREPARE");

            
              sqlite3_stmt* raw_stmt = nullptr;

            
              int err =

            
                  sqlite3_prepare_v2(db_.get(), sql.sql().c_str(), -1, &raw_stmt, nullptr);

            
              PreparedStatement statement{ScopedStmt(raw_stmt), std::move(sql)};

            
              if (err != SQLITE_OK) {

            
                const char* errmsg = sqlite3_errmsg(db_.get());

            
                std::string frame =

            
                    statement.sql_source_.AsTracebackForSqliteOffset(GetErrorOffset());

            
                base::Status status = base::ErrStatus("%s%s", frame.c_str(), errmsg);

            
                status.SetPayload("perfetto.dev/has_traceback", "true");


                statement.status_ = std::move(status);

            
                return statement;

            
              }

            
              if (!raw_stmt) {

            
                statement.status_ = base::ErrStatus("No SQL to execute");

            
              }

            
              return statement;

            
            }


            base::Status SqliteEngine::RegisterFunction(const char* name,

            
                                                        int argc,

            
                                                        Fn* fn,

            
                                                        void* ctx,

            
                                                        FnCtxDestructor* destructor,

            
                                                        bool deterministic) {

            
              int flags = SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0);

            
              int ret =

            
                  sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc), flags,

            
                                             ctx, fn, nullptr, nullptr, destructor);

            
              if (ret != SQLITE_OK) {

            
                return base::ErrStatus("Unable to register function with name %s", name);

            
              }

            
              *fn_ctx_.Insert(std::make_pair(name, argc), ctx).first = ctx;

            
              return base::OkStatus();

            
            }


            base::Status SqliteEngine::RegisterAggregateFunction(

            
                const char* name,

            
                int argc,

            
                AggregateFnStep* step,

            
                AggregateFnFinal* final,

            
                void* ctx,

            
                FnCtxDestructor* destructor,

            
                bool deterministic) {

            
              int flags = SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0);

            
              int ret =

            
                  sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc), flags,

            
                                             ctx, nullptr, step, final, destructor);

            
              if (ret != SQLITE_OK) {

            
                return base::ErrStatus("Unable to register function with name %s", name);

            
              }

            
              return base::OkStatus();

            
            }


            base::Status SqliteEngine::RegisterWindowFunction(const char* name,

            
                                                              int argc,

            
                                                              WindowFnStep* step,

            
                                                              WindowFnInverse* inverse,

            
                                                              WindowFnValue* value,

            
                                                              WindowFnFinal* final,

            
                                                              void* ctx,

            
                                                              FnCtxDestructor* destructor,

            
                                                              bool deterministic) {

            
              int flags = SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0);

            
              int ret = sqlite3_create_window_function(

            
                  db_.get(), name, static_cast<int>(argc), flags, ctx, step, final, value,

            
                  inverse, destructor);

            
              if (ret != SQLITE_OK) {

            
                return base::ErrStatus("Unable to register function with name %s", name);

            
              }

            
              return base::OkStatus();

            
            }


            base::Status SqliteEngine::UnregisterFunction(const char* name, int argc) {

            
              int ret = sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc),

            
                                                   SQLITE_UTF8, nullptr, nullptr, nullptr,

            
                                                   nullptr, nullptr);

            
              if (ret != SQLITE_OK) {

            
                return base::ErrStatus("Unable to unregister function with name %s", name);

            
              }

            
              fn_ctx_.Erase({name, argc});

            
              return base::OkStatus();

            
            }


            base::Status SqliteEngine::DeclareVirtualTable(const std::string& create_stmt) {

            
              int res = sqlite3_declare_vtab(db_.get(), create_stmt.c_str());

            
              if (res != SQLITE_OK) {

            
                return base::ErrStatus("Declare vtab failed: %s",

            
                                       sqlite3_errmsg(db_.get()));

            
              }

            
              return base::OkStatus();

            
            }


            void* SqliteEngine::GetFunctionContext(const std::string& name, int argc) {

            
              auto* res = fn_ctx_.Find(std::make_pair(name, argc));

            
              return res ? *res : nullptr;

            
            }


            std::optional<uint32_t> SqliteEngine::GetErrorOffset() const {

            
              return GetErrorOffsetDb(db_.get());

            
            }


            SqliteEngine::PreparedStatement::PreparedStatement(ScopedStmt stmt,

            
                                                               SqlSource source)

            
                : stmt_(std::move(stmt)),

            
                  expanded_sql_(sqlite3_expanded_sql(stmt_.get())),

            
                  sql_source_(std::move(source)) {}


            bool SqliteEngine::PreparedStatement::Step() {

            
              PERFETTO_TP_TRACE(metatrace::Category::QUERY_DETAILED, "STMT_STEP",

            
                                [this](metatrace::Record* record) {

            
                                  record->AddArg("Original SQL", original_sql());

            
                                  record->AddArg("Executed SQL", sql());

            
                                });


              // Now step once into |cur_stmt| so that when we prepare the next statment

            
              // we will have executed any dependent bytecode in this one.

            
              int err = sqlite3_step(stmt_.get());

            
              if (err == SQLITE_ROW) {

            
                return true;

            
              }

            
              if (err == SQLITE_DONE) {

            
                return false;

            
              }

            
              sqlite3* db = sqlite3_db_handle(stmt_.get());

            
              std::string frame =

            
                  sql_source_.AsTracebackForSqliteOffset(GetErrorOffsetDb(db));

            
              const char* errmsg = sqlite3_errmsg(db);

            
              status_ = base::ErrStatus("%s%s", frame.c_str(), errmsg);

            
              return false;

            
            }


            bool SqliteEngine::PreparedStatement::IsDone() const {

            
              return !sqlite3_stmt_busy(stmt_.get());

            
            }


            const char* SqliteEngine::PreparedStatement::original_sql() const {

            
              return sql_source_.original_sql().c_str();

            
            }


            const char* SqliteEngine::PreparedStatement::sql() const {

            
              return expanded_sql_.get();

            
            }


            }  // namespace perfetto::trace_processor
	/*
	* Copyright (C) 2023 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#include "src/trace_processor/sqlite/sqlite_engine.h"

	#include <sqlite3.h>
	#include <cstdint>
	#include <optional>
	#include <string>
	#include <utility>

	#include "perfetto/base/build_config.h"
	#include "perfetto/base/logging.h"
	#include "perfetto/base/status.h"
	#include "perfetto/public/compiler.h"
	#include "src/trace_processor/sqlite/scoped_db.h"
	#include "src/trace_processor/sqlite/sql_source.h"
	#include "src/trace_processor/tp_metatrace.h"

	#include "protos/perfetto/trace_processor/metatrace_categories.pbzero.h"

	// In Android and Chromium tree builds, we don't have the percentile module.
	// Just don't include it.
	#if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
	// defined in sqlite_src/ext/misc/percentile.c
	extern "C" int sqlite3_percentile_init(sqlite3* db,
	char** error,
	const sqlite3_api_routines* api);
	#endif // PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)

	namespace perfetto::trace_processor {
	namespace {

	void EnsureSqliteInitialized() {
	// sqlite3_initialize isn't actually thread-safe in standalone builds because
	// we build with SQLITE_THREADSAFE=0. Ensure it's only called from a single
	// thread.
	static bool init_once = [] {
	// Enabling memstatus causes a lock to be taken on every malloc/free in
	// SQLite to update the memory statistics. This can cause massive contention
	// in trace processor when multiple instances are used in parallel.
	// Fix this by disabling the memstatus API which we don't make use of in
	// any case. See b/335019324 for more info on this.
	int ret = sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0);

	// As much as it is painful, we need to catch instances of SQLITE_MISUSE
	// here against all the advice of the SQLite developers and lalitm@'s
	// intuition: SQLITE_MISUSE for sqlite3_config really means: that someone
	// else has already initialized SQLite. As we are an embeddable library,
	// it's very possible that the process embedding us has initialized SQLite
	// in a different way to what we want to do and, if so, we should respect
	// their choice.
	//
	// TODO(lalitm): ideally we would have an sqlite3_is_initialized API we
	// could use to gate the above check but that doesn't exist: report this
	// issue to SQLite developers and see if such an API could be added. If so
	// we can remove this check.
	if (ret == SQLITE_MISUSE) {
	return true;
	}

	PERFETTO_CHECK(ret == SQLITE_OK);
	return sqlite3_initialize() == SQLITE_OK;
	}();
	PERFETTO_CHECK(init_once);
	}

	void InitializeSqlite(sqlite3* db) {
	char* error = nullptr;
	sqlite3_exec(db, "PRAGMA temp_store=2", nullptr, nullptr, &error);
	if (error) {
	PERFETTO_FATAL("Error setting pragma temp_store: %s", error);
	}
	// In Android tree builds, we don't have the percentile module.
	#if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
	sqlite3_percentile_init(db, &error, nullptr);
	if (error) {
	PERFETTO_ELOG("Error initializing: %s", error);
	sqlite3_free(error);
	}
	#endif
	}

	std::optional<uint32_t> GetErrorOffsetDb(sqlite3* db) {
	int offset = sqlite3_error_offset(db);
	return offset == -1 ? std::nullopt
	: std::make_optional(static_cast<uint32_t>(offset));
	}

	} // namespace

	SqliteEngine::SqliteEngine() {
	sqlite3* db = nullptr;
	EnsureSqliteInitialized();

	// Ensure that we open the database with mutexes disabled: this is because
	// trace processor as a whole cannot be used from multiple threads so there is
	// no point paying the (potentially significant) cost of mutexes at the SQLite
	// level.
	static constexpr int kSqliteOpenFlags =
	SQLITE_OPEN_READWRITE \| SQLITE_OPEN_CREATE \| SQLITE_OPEN_NOMUTEX;
	PERFETTO_CHECK(sqlite3_open_v2(":memory:", &db, kSqliteOpenFlags, nullptr) ==
	SQLITE_OK);
	InitializeSqlite(db);
	db_.reset(db);
	}

	SqliteEngine::~SqliteEngine() {
	// It is important to unregister any functions that have been registered with
	// the database before destroying it. This is because functions can hold onto
	// prepared statements, which must be finalized before database destruction.
	for (auto it = fn_ctx_.GetIterator(); it; ++it) {
	int ret = sqlite3_create_function_v2(db_.get(), it.key().first.c_str(),
	it.key().second, SQLITE_UTF8, nullptr,
	nullptr, nullptr, nullptr, nullptr);
	if (PERFETTO_UNLIKELY(ret != SQLITE_OK)) {
	PERFETTO_FATAL("Failed to drop function: '%s'", it.key().first.c_str());
	}
	}
	fn_ctx_.Clear();
	}

	SqliteEngine::PreparedStatement SqliteEngine::PrepareStatement(SqlSource sql) {
	PERFETTO_TP_TRACE(metatrace::Category::QUERY_DETAILED, "QUERY_PREPARE");
	sqlite3_stmt* raw_stmt = nullptr;
	int err =
	sqlite3_prepare_v2(db_.get(), sql.sql().c_str(), -1, &raw_stmt, nullptr);
	PreparedStatement statement{ScopedStmt(raw_stmt), std::move(sql)};
	if (err != SQLITE_OK) {
	const char* errmsg = sqlite3_errmsg(db_.get());
	std::string frame =
	statement.sql_source_.AsTracebackForSqliteOffset(GetErrorOffset());
	base::Status status = base::ErrStatus("%s%s", frame.c_str(), errmsg);
	status.SetPayload("perfetto.dev/has_traceback", "true");

	statement.status_ = std::move(status);
	return statement;
	}
	if (!raw_stmt) {
	statement.status_ = base::ErrStatus("No SQL to execute");
	}
	return statement;
	}

	base::Status SqliteEngine::RegisterFunction(const char* name,
	int argc,
	Fn* fn,
	void* ctx,
	FnCtxDestructor* destructor,
	bool deterministic) {
	int flags = SQLITE_UTF8 \| (deterministic ? SQLITE_DETERMINISTIC : 0);
	int ret =
	sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc), flags,
	ctx, fn, nullptr, nullptr, destructor);
	if (ret != SQLITE_OK) {
	return base::ErrStatus("Unable to register function with name %s", name);
	}
	*fn_ctx_.Insert(std::make_pair(name, argc), ctx).first = ctx;
	return base::OkStatus();
	}

	base::Status SqliteEngine::RegisterAggregateFunction(
	const char* name,
	int argc,
	AggregateFnStep* step,
	AggregateFnFinal* final,
	void* ctx,
	FnCtxDestructor* destructor,
	bool deterministic) {
	int flags = SQLITE_UTF8 \| (deterministic ? SQLITE_DETERMINISTIC : 0);
	int ret =
	sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc), flags,
	ctx, nullptr, step, final, destructor);
	if (ret != SQLITE_OK) {
	return base::ErrStatus("Unable to register function with name %s", name);
	}
	return base::OkStatus();
	}

	base::Status SqliteEngine::RegisterWindowFunction(const char* name,
	int argc,
	WindowFnStep* step,
	WindowFnInverse* inverse,
	WindowFnValue* value,
	WindowFnFinal* final,
	void* ctx,
	FnCtxDestructor* destructor,
	bool deterministic) {
	int flags = SQLITE_UTF8 \| (deterministic ? SQLITE_DETERMINISTIC : 0);
	int ret = sqlite3_create_window_function(
	db_.get(), name, static_cast<int>(argc), flags, ctx, step, final, value,
	inverse, destructor);
	if (ret != SQLITE_OK) {
	return base::ErrStatus("Unable to register function with name %s", name);
	}
	return base::OkStatus();
	}

	base::Status SqliteEngine::UnregisterFunction(const char* name, int argc) {
	int ret = sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc),
	SQLITE_UTF8, nullptr, nullptr, nullptr,
	nullptr, nullptr);
	if (ret != SQLITE_OK) {
	return base::ErrStatus("Unable to unregister function with name %s", name);
	}
	fn_ctx_.Erase({name, argc});
	return base::OkStatus();
	}

	base::Status SqliteEngine::DeclareVirtualTable(const std::string& create_stmt) {
	int res = sqlite3_declare_vtab(db_.get(), create_stmt.c_str());
	if (res != SQLITE_OK) {
	return base::ErrStatus("Declare vtab failed: %s",
	sqlite3_errmsg(db_.get()));
	}
	return base::OkStatus();
	}

	void* SqliteEngine::GetFunctionContext(const std::string& name, int argc) {
	auto* res = fn_ctx_.Find(std::make_pair(name, argc));
	return res ? *res : nullptr;
	}

	std::optional<uint32_t> SqliteEngine::GetErrorOffset() const {
	return GetErrorOffsetDb(db_.get());
	}

	SqliteEngine::PreparedStatement::PreparedStatement(ScopedStmt stmt,
	SqlSource source)
	: stmt_(std::move(stmt)),
	expanded_sql_(sqlite3_expanded_sql(stmt_.get())),
	sql_source_(std::move(source)) {}

	bool SqliteEngine::PreparedStatement::Step() {
	PERFETTO_TP_TRACE(metatrace::Category::QUERY_DETAILED, "STMT_STEP",
	[this](metatrace::Record* record) {
	record->AddArg("Original SQL", original_sql());
	record->AddArg("Executed SQL", sql());
	});

	// Now step once into \|cur_stmt\| so that when we prepare the next statment
	// we will have executed any dependent bytecode in this one.
	int err = sqlite3_step(stmt_.get());
	if (err == SQLITE_ROW) {
	return true;
	}
	if (err == SQLITE_DONE) {
	return false;
	}
	sqlite3* db = sqlite3_db_handle(stmt_.get());
	std::string frame =
	sql_source_.AsTracebackForSqliteOffset(GetErrorOffsetDb(db));
	const char* errmsg = sqlite3_errmsg(db);
	status_ = base::ErrStatus("%s%s", frame.c_str(), errmsg);
	return false;
	}

	bool SqliteEngine::PreparedStatement::IsDone() const {
	return !sqlite3_stmt_busy(stmt_.get());
	}

	const char* SqliteEngine::PreparedStatement::original_sql() const {
	return sql_source_.original_sql().c_str();
	}

	const char* SqliteEngine::PreparedStatement::sql() const {
	return expanded_sql_.get();
	}

	} // namespace perfetto::trace_processor