tp: FileBuffer -> TraceBlobViewReader and use it in ProtoTokenizer
Change-Id: Ie8e8dc6968a956f020f2d6b8b07d043a55949278
diff --git a/Android.bp b/Android.bp
index daef6ec..e8eb8a3 100644
--- a/Android.bp
+++ b/Android.bp
@@ -2447,7 +2447,6 @@
":perfetto_src_trace_processor_util_build_id",
":perfetto_src_trace_processor_util_bump_allocator",
":perfetto_src_trace_processor_util_descriptors",
- ":perfetto_src_trace_processor_util_file_buffer",
":perfetto_src_trace_processor_util_glob",
":perfetto_src_trace_processor_util_gzip",
":perfetto_src_trace_processor_util_interned_message_view",
@@ -2460,6 +2459,7 @@
":perfetto_src_trace_processor_util_regex",
":perfetto_src_trace_processor_util_sql_argument",
":perfetto_src_trace_processor_util_stdlib",
+ ":perfetto_src_trace_processor_util_trace_blob_view_reader",
":perfetto_src_trace_processor_util_trace_type",
":perfetto_src_trace_processor_util_util",
":perfetto_src_trace_processor_util_zip_reader",
@@ -13498,14 +13498,6 @@
],
}
-// GN: //src/trace_processor/util:file_buffer
-filegroup {
- name: "perfetto_src_trace_processor_util_file_buffer",
- srcs: [
- "src/trace_processor/util/file_buffer.cc",
- ],
-}
-
// GN: //src/trace_processor/util:glob
filegroup {
name: "perfetto_src_trace_processor_util_glob",
@@ -13595,6 +13587,14 @@
name: "perfetto_src_trace_processor_util_stdlib",
}
+// GN: //src/trace_processor/util:trace_blob_view_reader
+filegroup {
+ name: "perfetto_src_trace_processor_util_trace_blob_view_reader",
+ srcs: [
+ "src/trace_processor/util/trace_blob_view_reader.cc",
+ ],
+}
+
// GN: //src/trace_processor/util:trace_type
filegroup {
name: "perfetto_src_trace_processor_util_trace_type",
@@ -13609,7 +13609,6 @@
srcs: [
"src/trace_processor/util/bump_allocator_unittest.cc",
"src/trace_processor/util/debug_annotation_parser_unittest.cc",
- "src/trace_processor/util/file_buffer_unittest.cc",
"src/trace_processor/util/glob_unittest.cc",
"src/trace_processor/util/gzip_utils_unittest.cc",
"src/trace_processor/util/proto_profiler_unittest.cc",
@@ -13618,6 +13617,7 @@
"src/trace_processor/util/protozero_to_text_unittests.cc",
"src/trace_processor/util/sql_argument_unittest.cc",
"src/trace_processor/util/streaming_line_reader_unittest.cc",
+ "src/trace_processor/util/trace_blob_view_reader_unittest.cc",
"src/trace_processor/util/zip_reader_unittest.cc",
],
}
@@ -15197,7 +15197,6 @@
":perfetto_src_trace_processor_util_build_id",
":perfetto_src_trace_processor_util_bump_allocator",
":perfetto_src_trace_processor_util_descriptors",
- ":perfetto_src_trace_processor_util_file_buffer",
":perfetto_src_trace_processor_util_glob",
":perfetto_src_trace_processor_util_gzip",
":perfetto_src_trace_processor_util_interned_message_view",
@@ -15210,6 +15209,7 @@
":perfetto_src_trace_processor_util_regex",
":perfetto_src_trace_processor_util_sql_argument",
":perfetto_src_trace_processor_util_stdlib",
+ ":perfetto_src_trace_processor_util_trace_blob_view_reader",
":perfetto_src_trace_processor_util_trace_type",
":perfetto_src_trace_processor_util_unittests",
":perfetto_src_trace_processor_util_util",
@@ -16214,7 +16214,6 @@
":perfetto_src_trace_processor_util_build_id",
":perfetto_src_trace_processor_util_bump_allocator",
":perfetto_src_trace_processor_util_descriptors",
- ":perfetto_src_trace_processor_util_file_buffer",
":perfetto_src_trace_processor_util_glob",
":perfetto_src_trace_processor_util_gzip",
":perfetto_src_trace_processor_util_interned_message_view",
@@ -16227,6 +16226,7 @@
":perfetto_src_trace_processor_util_regex",
":perfetto_src_trace_processor_util_sql_argument",
":perfetto_src_trace_processor_util_stdlib",
+ ":perfetto_src_trace_processor_util_trace_blob_view_reader",
":perfetto_src_trace_processor_util_trace_type",
":perfetto_src_trace_processor_util_util",
":perfetto_src_trace_processor_util_zip_reader",
@@ -16427,6 +16427,7 @@
":perfetto_src_trace_processor_util_proto_to_args_parser",
":perfetto_src_trace_processor_util_protozero_to_text",
":perfetto_src_trace_processor_util_regex",
+ ":perfetto_src_trace_processor_util_trace_blob_view_reader",
":perfetto_src_trace_processor_util_trace_type",
":perfetto_src_trace_processor_util_util",
":perfetto_src_trace_redaction_trace_redaction",
@@ -16610,7 +16611,6 @@
":perfetto_src_trace_processor_util_build_id",
":perfetto_src_trace_processor_util_bump_allocator",
":perfetto_src_trace_processor_util_descriptors",
- ":perfetto_src_trace_processor_util_file_buffer",
":perfetto_src_trace_processor_util_glob",
":perfetto_src_trace_processor_util_gzip",
":perfetto_src_trace_processor_util_interned_message_view",
@@ -16623,6 +16623,7 @@
":perfetto_src_trace_processor_util_regex",
":perfetto_src_trace_processor_util_sql_argument",
":perfetto_src_trace_processor_util_stdlib",
+ ":perfetto_src_trace_processor_util_trace_blob_view_reader",
":perfetto_src_trace_processor_util_trace_type",
":perfetto_src_trace_processor_util_util",
":perfetto_src_trace_processor_util_zip_reader",
diff --git a/BUILD b/BUILD
index bf3ce33..b0337fa 100644
--- a/BUILD
+++ b/BUILD
@@ -276,7 +276,6 @@
":src_trace_processor_util_build_id",
":src_trace_processor_util_bump_allocator",
":src_trace_processor_util_descriptors",
- ":src_trace_processor_util_file_buffer",
":src_trace_processor_util_glob",
":src_trace_processor_util_gzip",
":src_trace_processor_util_interned_message_view",
@@ -289,6 +288,7 @@
":src_trace_processor_util_regex",
":src_trace_processor_util_sql_argument",
":src_trace_processor_util_stdlib",
+ ":src_trace_processor_util_trace_blob_view_reader",
":src_trace_processor_util_trace_type",
":src_trace_processor_util_util",
":src_trace_processor_util_zip_reader",
@@ -3063,15 +3063,6 @@
],
)
-# GN target: //src/trace_processor/util:file_buffer
-perfetto_filegroup(
- name = "src_trace_processor_util_file_buffer",
- srcs = [
- "src/trace_processor/util/file_buffer.cc",
- "src/trace_processor/util/file_buffer.h",
- ],
-)
-
# GN target: //src/trace_processor/util:glob
perfetto_filegroup(
name = "src_trace_processor_util_glob",
@@ -3181,6 +3172,15 @@
],
)
+# GN target: //src/trace_processor/util:trace_blob_view_reader
+perfetto_filegroup(
+ name = "src_trace_processor_util_trace_blob_view_reader",
+ srcs = [
+ "src/trace_processor/util/trace_blob_view_reader.cc",
+ "src/trace_processor/util/trace_blob_view_reader.h",
+ ],
+)
+
# GN target: //src/trace_processor/util:trace_type
perfetto_filegroup(
name = "src_trace_processor_util_trace_type",
@@ -6157,7 +6157,6 @@
":src_trace_processor_util_build_id",
":src_trace_processor_util_bump_allocator",
":src_trace_processor_util_descriptors",
- ":src_trace_processor_util_file_buffer",
":src_trace_processor_util_glob",
":src_trace_processor_util_gzip",
":src_trace_processor_util_interned_message_view",
@@ -6170,6 +6169,7 @@
":src_trace_processor_util_regex",
":src_trace_processor_util_sql_argument",
":src_trace_processor_util_stdlib",
+ ":src_trace_processor_util_trace_blob_view_reader",
":src_trace_processor_util_trace_type",
":src_trace_processor_util_util",
":src_trace_processor_util_zip_reader",
@@ -6341,7 +6341,6 @@
":src_trace_processor_util_build_id",
":src_trace_processor_util_bump_allocator",
":src_trace_processor_util_descriptors",
- ":src_trace_processor_util_file_buffer",
":src_trace_processor_util_glob",
":src_trace_processor_util_gzip",
":src_trace_processor_util_interned_message_view",
@@ -6354,6 +6353,7 @@
":src_trace_processor_util_regex",
":src_trace_processor_util_sql_argument",
":src_trace_processor_util_stdlib",
+ ":src_trace_processor_util_trace_blob_view_reader",
":src_trace_processor_util_trace_type",
":src_trace_processor_util_util",
":src_trace_processor_util_zip_reader",
@@ -6579,7 +6579,6 @@
":src_trace_processor_util_build_id",
":src_trace_processor_util_bump_allocator",
":src_trace_processor_util_descriptors",
- ":src_trace_processor_util_file_buffer",
":src_trace_processor_util_glob",
":src_trace_processor_util_gzip",
":src_trace_processor_util_interned_message_view",
@@ -6592,6 +6591,7 @@
":src_trace_processor_util_regex",
":src_trace_processor_util_sql_argument",
":src_trace_processor_util_stdlib",
+ ":src_trace_processor_util_trace_blob_view_reader",
":src_trace_processor_util_trace_type",
":src_trace_processor_util_util",
":src_trace_processor_util_zip_reader",
diff --git a/include/perfetto/base/status.h b/include/perfetto/base/status.h
index 506bb6e..ee39593 100644
--- a/include/perfetto/base/status.h
+++ b/include/perfetto/base/status.h
@@ -109,7 +109,7 @@
return Status();
}
-PERFETTO_PRINTF_FORMAT(1, 2) Status ErrStatus(const char* format, ...);
+Status ErrStatus(const char* format, ...) PERFETTO_PRINTF_FORMAT(1, 2);
} // namespace base
} // namespace perfetto
diff --git a/src/trace_processor/importers/perf/BUILD.gn b/src/trace_processor/importers/perf/BUILD.gn
index da4153a..5fd5d25 100644
--- a/src/trace_processor/importers/perf/BUILD.gn
+++ b/src/trace_processor/importers/perf/BUILD.gn
@@ -83,7 +83,7 @@
"../../tables:tables_python",
"../../types",
"../../util:build_id",
- "../../util:file_buffer",
+ "../../util:trace_blob_view_reader",
"../../util:util",
"../common:common",
"../proto:minimal",
diff --git a/src/trace_processor/importers/perf/perf_data_tokenizer.cc b/src/trace_processor/importers/perf/perf_data_tokenizer.cc
index 75e4b50..1c48a4a 100644
--- a/src/trace_processor/importers/perf/perf_data_tokenizer.cc
+++ b/src/trace_processor/importers/perf/perf_data_tokenizer.cc
@@ -239,7 +239,7 @@
base::StatusOr<PerfDataTokenizer::ParsingResult>
PerfDataTokenizer::ParseRecords() {
- while (buffer_.file_offset() < header_.data.end()) {
+ while (buffer_.start_offset() < header_.data.end()) {
Record record;
if (auto res = ParseRecord(record);
@@ -260,7 +260,7 @@
Record& record) {
record.session = perf_session_;
std::optional<TraceBlobView> tbv =
- buffer_.SliceOff(buffer_.file_offset(), sizeof(record.header));
+ buffer_.SliceOff(buffer_.start_offset(), sizeof(record.header));
if (!tbv) {
return ParsingResult::kMoreDataNeeded;
}
@@ -270,7 +270,7 @@
return base::ErrStatus("Invalid record size: %" PRIu16, record.header.size);
}
- tbv = buffer_.SliceOff(buffer_.file_offset() + sizeof(record.header),
+ tbv = buffer_.SliceOff(buffer_.start_offset() + sizeof(record.header),
record.header.size - sizeof(record.header));
if (!tbv) {
return ParsingResult::kMoreDataNeeded;
@@ -332,7 +332,7 @@
base::StatusOr<PerfDataTokenizer::ParsingResult>
PerfDataTokenizer::ParseFeatureSections() {
- PERFETTO_CHECK(buffer_.file_offset() == header_.data.end());
+ PERFETTO_CHECK(buffer_.start_offset() == header_.data.end());
auto tbv = buffer_.SliceOff(feature_headers_section_.offset,
feature_headers_section_.size);
if (!tbv) {
diff --git a/src/trace_processor/importers/perf/perf_data_tokenizer.h b/src/trace_processor/importers/perf/perf_data_tokenizer.h
index 61c1308..f02efba 100644
--- a/src/trace_processor/importers/perf/perf_data_tokenizer.h
+++ b/src/trace_processor/importers/perf/perf_data_tokenizer.h
@@ -30,7 +30,7 @@
#include "src/trace_processor/importers/common/chunked_trace_reader.h"
#include "src/trace_processor/importers/perf/perf_file.h"
#include "src/trace_processor/importers/perf/perf_session.h"
-#include "src/trace_processor/util/file_buffer.h"
+#include "src/trace_processor/util/trace_blob_view_reader.h"
namespace perfetto {
namespace trace_processor {
@@ -90,7 +90,7 @@
RefPtr<PerfSession> perf_session_;
- util::FileBuffer buffer_;
+ util::TraceBlobViewReader buffer_;
int64_t latest_timestamp_ = 0;
};
diff --git a/src/trace_processor/importers/proto/BUILD.gn b/src/trace_processor/importers/proto/BUILD.gn
index 937fd1d..68d8963 100644
--- a/src/trace_processor/importers/proto/BUILD.gn
+++ b/src/trace_processor/importers/proto/BUILD.gn
@@ -100,6 +100,7 @@
"../../util:build_id",
"../../util:gzip",
"../../util:profiler_util",
+ "../../util:trace_blob_view_reader",
"../common",
"../common:parser_types",
"../etw:minimal",
diff --git a/src/trace_processor/importers/proto/proto_trace_tokenizer.h b/src/trace_processor/importers/proto/proto_trace_tokenizer.h
index 2526900..1d37791 100644
--- a/src/trace_processor/importers/proto/proto_trace_tokenizer.h
+++ b/src/trace_processor/importers/proto/proto_trace_tokenizer.h
@@ -17,23 +17,26 @@
#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_
#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_
+#include <algorithm>
+#include <cstddef>
#include <cstdint>
-#include <vector>
+#include <optional>
+#include <utility>
+#include "perfetto/base/logging.h"
#include "perfetto/base/status.h"
+#include "perfetto/protozero/field.h"
#include "perfetto/protozero/proto_utils.h"
#include "perfetto/public/compiler.h"
-#include "perfetto/trace_processor/status.h"
-#include "perfetto/trace_processor/trace_blob.h"
#include "perfetto/trace_processor/trace_blob_view.h"
+#include "protos/perfetto/trace/trace_packet.pbzero.h"
#include "src/trace_processor/util/gzip_utils.h"
-#include "src/trace_processor/util/status_macros.h"
#include "protos/perfetto/trace/trace.pbzero.h"
-#include "protos/perfetto/trace/trace_packet.pbzero.h"
+#include "src/trace_processor/util/status_macros.h"
+#include "src/trace_processor/util/trace_blob_view_reader.h"
-namespace perfetto {
-namespace trace_processor {
+namespace perfetto::trace_processor {
// Reads a protobuf trace in chunks and extracts boundaries of trace packets
// (or subfields, for the case of ftrace) with their timestamps.
@@ -41,116 +44,79 @@
public:
ProtoTraceTokenizer();
- template <typename Callback = util::Status(TraceBlobView)>
- util::Status Tokenize(TraceBlobView blob, Callback callback) {
- const uint8_t* data = blob.data();
- size_t size = blob.size();
- if (!partial_buf_.empty()) {
- // It takes ~5 bytes for a proto preamble + the varint size.
- const size_t kHeaderBytes = 5;
- if (PERFETTO_UNLIKELY(partial_buf_.size() < kHeaderBytes)) {
- size_t missing_len = std::min(kHeaderBytes - partial_buf_.size(), size);
- partial_buf_.insert(partial_buf_.end(), &data[0], &data[missing_len]);
- if (partial_buf_.size() < kHeaderBytes)
- return util::OkStatus();
- data += missing_len;
- size -= missing_len;
+ template <typename Callback = base::Status(TraceBlobView)>
+ base::Status Tokenize(TraceBlobView tbv, Callback callback) {
+ reader_.PushBack(std::move(tbv));
+
+ for (;;) {
+ size_t start_offset = reader_.start_offset();
+ size_t avail = reader_.avail();
+
+ // The header must be at least 2 bytes (1 byte for tag, 1 byte for
+ // size) and can be at most 6 bytes (1 byte for tag + 5 bytes for
+ // size).
+ const size_t kMinHeaderBytes = 2;
+ const size_t kMaxHeaderBytes = 6;
+ std::optional<TraceBlobView> header = reader_.SliceOff(
+ start_offset,
+ std::min(std::max(avail, kMinHeaderBytes), kMaxHeaderBytes));
+
+ // This means that kMinHeaderBytes was not available. Just wait for the
+ // next round.
+ if (PERFETTO_UNLIKELY(!header)) {
+ return base::OkStatus();
}
- // At this point we have enough data in |partial_buf_| to read at least
- // the field header and know the size of the next TracePacket.
- const uint8_t* pos = &partial_buf_[0];
- uint8_t proto_field_tag = *pos;
- uint64_t field_size = 0;
- // We cannot do &partial_buf_[partial_buf_.size()] because that crashes
- // on MSVC STL debug builds, so does &*partial_buf_.end().
- const uint8_t* next = protozero::proto_utils::ParseVarInt(
- ++pos, &partial_buf_.front() + partial_buf_.size(), &field_size);
- bool parse_failed = next == pos;
- pos = next;
- if (proto_field_tag != kTracePacketTag || field_size == 0 ||
- parse_failed) {
- return util::ErrStatus(
- "Failed parsing a TracePacket from the partial buffer");
+ uint64_t field_size;
+ const uint8_t* size_start = header->data() + 1;
+ const uint8_t* size_end = protozero::proto_utils::ParseVarInt(
+ size_start, header->data() + header->size(), &field_size);
+
+ // If we had less than the maximum number of header bytes, it's possible
+ // that we just need more to actually parse. Otherwise, this is an error.
+ if (PERFETTO_UNLIKELY(size_start == size_end)) {
+ return header->size() < kMaxHeaderBytes
+ ? base::OkStatus()
+ : base::ErrStatus("Failed to parse TracePacket size");
}
- // At this point we know how big the TracePacket is.
- size_t hdr_size = static_cast<size_t>(pos - &partial_buf_[0]);
- size_t size_incl_header = static_cast<size_t>(field_size + hdr_size);
- PERFETTO_DCHECK(size_incl_header > partial_buf_.size());
-
- // There is a good chance that between the |partial_buf_| and the new
- // |data| of the current call we have enough bytes to parse a TracePacket.
- if (partial_buf_.size() + size >= size_incl_header) {
- // Create a new buffer for the whole TracePacket and copy into that:
- // 1) The beginning of the TracePacket (including the proto header) from
- // the partial buffer.
- // 2) The rest of the TracePacket from the current |data| buffer (note
- // that we might have consumed already a few bytes form |data|
- // earlier in this function, hence we need to keep |off| into
- // account).
- TraceBlob glued = TraceBlob::Allocate(size_incl_header);
- memcpy(glued.data(), partial_buf_.data(), partial_buf_.size());
- // |size_missing| is the number of bytes for the rest of the TracePacket
- // in |data|.
- size_t size_missing = size_incl_header - partial_buf_.size();
- memcpy(glued.data() + partial_buf_.size(), &data[0], size_missing);
- data += size_missing;
- size -= size_missing;
- partial_buf_.clear();
- RETURN_IF_ERROR(
- ParseInternal(TraceBlobView(std::move(glued)), callback));
- } else {
- partial_buf_.insert(partial_buf_.end(), data, &data[size]);
- return util::OkStatus();
+ // Empty packets can legitimately happen if the producer ends up emitting
+ // no data: just ignore them.
+ auto hdr_size = static_cast<size_t>(size_end - header->data());
+ if (PERFETTO_UNLIKELY(field_size == 0)) {
+ PERFETTO_CHECK(reader_.PopFrontBytes(hdr_size));
+ continue;
}
- }
- return ParseInternal(blob.slice(data, size), callback);
- }
- private:
- static constexpr uint8_t kTracePacketTag =
- protozero::proto_utils::MakeTagLengthDelimited(
- protos::pbzero::Trace::kPacketFieldNumber);
-
- template <typename Callback = util::Status(TraceBlobView)>
- util::Status ParseInternal(TraceBlobView whole_buf, Callback callback) {
- static constexpr auto kLengthDelimited =
- protozero::proto_utils::ProtoWireType::kLengthDelimited;
- const uint8_t* const start = whole_buf.data();
- protos::pbzero::Trace::Decoder decoder(whole_buf.data(), whole_buf.size());
- for (auto it = decoder.packet(); it; ++it) {
- if (PERFETTO_UNLIKELY(it->type() != kLengthDelimited)) {
- return base::ErrStatus("Failed to parse TracePacket bounds");
+ // If there's no enough bytes in the reader, then we cannot do anymore.
+ size_t size_incl_header = hdr_size + field_size;
+ if (size_incl_header > avail) {
+ return base::OkStatus();
}
- protozero::ConstBytes packet = *it;
- TraceBlobView sliced = whole_buf.slice(packet.data, packet.size);
- RETURN_IF_ERROR(ParsePacket(std::move(sliced), callback));
- }
- const size_t bytes_left = decoder.bytes_left();
- if (bytes_left > 0) {
- PERFETTO_DCHECK(partial_buf_.empty());
- partial_buf_.insert(partial_buf_.end(), &start[decoder.read_offset()],
- &start[decoder.read_offset() + bytes_left]);
- }
- return util::OkStatus();
- }
+ uint8_t proto_field_tag = *header->data();
+ if (PERFETTO_UNLIKELY(proto_field_tag != kTracePacketTag)) {
+ return base::ErrStatus("Invalid TracePacket tag or size");
+ }
- template <typename Callback = util::Status(TraceBlobView)>
- util::Status ParsePacket(TraceBlobView packet, Callback callback) {
- protos::pbzero::TracePacket::Decoder decoder(packet.data(),
- packet.length());
- if (decoder.has_compressed_packets()) {
+ auto packet = reader_.SliceOff(start_offset + hdr_size, field_size);
+ PERFETTO_CHECK(packet);
+ PERFETTO_CHECK(reader_.PopFrontBytes(hdr_size + field_size));
+ protos::pbzero::TracePacket::Decoder decoder(packet->data(),
+ packet->length());
+ if (!decoder.has_compressed_packets()) {
+ RETURN_IF_ERROR(callback(std::move(*packet)));
+ continue;
+ }
+
if (!util::IsGzipSupported()) {
- return util::Status(
+ return base::ErrStatus(
"Cannot decode compressed packets. Zlib not enabled");
}
protozero::ConstBytes field = decoder.compressed_packets();
- TraceBlobView compressed_packets = packet.slice(field.data, field.size);
+ TraceBlobView compressed_packets = packet->slice(field.data, field.size);
TraceBlobView packets;
-
RETURN_IF_ERROR(Decompress(std::move(compressed_packets), &packets));
const uint8_t* start = packets.data();
@@ -158,35 +124,41 @@
const uint8_t* ptr = start;
while ((end - ptr) > 2) {
const uint8_t* packet_outer = ptr;
- if (PERFETTO_UNLIKELY(*ptr != kTracePacketTag))
- return util::ErrStatus("Expected TracePacket tag");
+ if (PERFETTO_UNLIKELY(*ptr != kTracePacketTag)) {
+ return base::ErrStatus("Expected TracePacket tag");
+ }
uint64_t packet_size = 0;
ptr = protozero::proto_utils::ParseVarInt(++ptr, end, &packet_size);
const uint8_t* packet_start = ptr;
ptr += packet_size;
- if (PERFETTO_UNLIKELY((ptr - packet_outer) < 2 || ptr > end))
- return util::ErrStatus("Invalid packet size");
-
+ if (PERFETTO_UNLIKELY((ptr - packet_outer) < 2 || ptr > end)) {
+ return base::ErrStatus("Invalid packet size");
+ }
TraceBlobView sliced =
packets.slice(packet_start, static_cast<size_t>(packet_size));
- RETURN_IF_ERROR(ParsePacket(std::move(sliced), callback));
+ RETURN_IF_ERROR(callback(std::move(sliced)));
}
- return util::OkStatus();
}
- return callback(std::move(packet));
}
- util::Status Decompress(TraceBlobView input, TraceBlobView* output);
+ private:
+ static constexpr uint8_t kTracePacketTag =
+ protozero::proto_utils::MakeTagLengthDelimited(
+ protos::pbzero::Trace::kPacketFieldNumber);
+ static constexpr uint16_t kCompresedPacketsTag =
+ protozero::proto_utils::MakeTagLengthDelimited(
+ protos::pbzero::TracePacket::kCompressedPacketsFieldNumber);
+
+ base::Status Decompress(TraceBlobView input, TraceBlobView* output);
// Used to glue together trace packets that span across two (or more)
// Parse() boundaries.
- std::vector<uint8_t> partial_buf_;
+ util::TraceBlobViewReader reader_;
// Allows support for compressed trace packets.
util::GzipDecompressor decompressor_;
};
-} // namespace trace_processor
-} // namespace perfetto
+} // namespace perfetto::trace_processor
#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_
diff --git a/src/trace_processor/util/BUILD.gn b/src/trace_processor/util/BUILD.gn
index 6c40069..ee4ee7c 100644
--- a/src/trace_processor/util/BUILD.gn
+++ b/src/trace_processor/util/BUILD.gn
@@ -254,10 +254,10 @@
]
}
-source_set("file_buffer") {
+source_set("trace_blob_view_reader") {
sources = [
- "file_buffer.cc",
- "file_buffer.h",
+ "trace_blob_view_reader.cc",
+ "trace_blob_view_reader.h",
]
deps = [
"../../../gn:default_deps",
@@ -281,7 +281,6 @@
sources = [
"bump_allocator_unittest.cc",
"debug_annotation_parser_unittest.cc",
- "file_buffer_unittest.cc",
"glob_unittest.cc",
"proto_profiler_unittest.cc",
"proto_to_args_parser_unittest.cc",
@@ -289,13 +288,13 @@
"protozero_to_text_unittests.cc",
"sql_argument_unittest.cc",
"streaming_line_reader_unittest.cc",
+ "trace_blob_view_reader_unittest.cc",
"zip_reader_unittest.cc",
]
testonly = true
deps = [
":bump_allocator",
":descriptors",
- ":file_buffer",
":glob",
":gzip",
":proto_profiler",
@@ -303,6 +302,7 @@
":protozero_to_json",
":protozero_to_text",
":sql_argument",
+ ":trace_blob_view_reader",
":zip_reader",
"..:gen_cc_test_messages_descriptor",
"../../../gn:default_deps",
diff --git a/src/trace_processor/util/file_buffer.cc b/src/trace_processor/util/file_buffer.cc
deleted file mode 100644
index 1dbd10b..0000000
--- a/src/trace_processor/util/file_buffer.cc
+++ /dev/null
@@ -1,119 +0,0 @@
-
-/*
- * Copyright (C) 2024 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "src/trace_processor/util/file_buffer.h"
-
-#include <algorithm>
-#include <cstddef>
-#include <cstdint>
-#include <cstring>
-#include <iterator>
-#include <optional>
-
-#include "perfetto/base/logging.h"
-#include "perfetto/trace_processor/trace_blob.h"
-#include "perfetto/trace_processor/trace_blob_view.h"
-
-namespace perfetto::trace_processor::util {
-
-void FileBuffer::PushBack(TraceBlobView data) {
- if (data.size() == 0) {
- return;
- }
- const size_t size = data.size();
- data_.emplace_back(Entry{end_offset_, std::move(data)});
- end_offset_ += size;
-}
-
-bool FileBuffer::PopFrontUntil(const size_t target_offset) {
- PERFETTO_CHECK(file_offset() <= target_offset);
- while (!data_.empty()) {
- Entry& entry = data_.front();
- if (target_offset == entry.file_offset) {
- return true;
- }
- const size_t bytes_to_pop = target_offset - entry.file_offset;
- if (entry.data.size() > bytes_to_pop) {
- entry.data =
- entry.data.slice_off(bytes_to_pop, entry.data.size() - bytes_to_pop);
- entry.file_offset += bytes_to_pop;
- return true;
- }
- data_.pop_front();
- }
-
- return target_offset == end_offset_;
-}
-
-std::optional<TraceBlobView> FileBuffer::SliceOff(size_t start_offset,
- size_t length) const {
- if (length == 0) {
- return TraceBlobView();
- }
-
- if (start_offset + length > end_offset_) {
- return std::nullopt;
- }
-
- Iterator it = FindEntryWithOffset(start_offset);
- if (it == end()) {
- return std::nullopt;
- }
-
- const size_t offset_from_entry_start = start_offset - it->file_offset;
- const size_t bytes_in_entry = it->data.size() - offset_from_entry_start;
- TraceBlobView first_blob = it->data.slice_off(
- offset_from_entry_start, std::min(bytes_in_entry, length));
-
- if (first_blob.size() == length) {
- return std::move(first_blob);
- }
-
- auto buffer = TraceBlob::Allocate(length);
- uint8_t* ptr = buffer.data();
-
- memcpy(ptr, first_blob.data(), first_blob.size());
- ptr += first_blob.size();
- length -= first_blob.size();
- ++it;
-
- while (length != 0) {
- PERFETTO_DCHECK(it != end());
- const size_t bytes_to_copy = std::min(length, it->data.size());
- memcpy(ptr, it->data.data(), bytes_to_copy);
- ptr += bytes_to_copy;
- length -= bytes_to_copy;
- ++it;
- }
-
- return TraceBlobView(std::move(buffer));
-}
-
-FileBuffer::Iterator FileBuffer::FindEntryWithOffset(size_t offset) const {
- if (offset >= end_offset_) {
- return end();
- }
-
- auto it = std::upper_bound(
- data_.begin(), data_.end(), offset,
- [](size_t offset, const Entry& rhs) { return offset < rhs.file_offset; });
- // This can only happen if too much data was popped.
- PERFETTO_CHECK(it != data_.begin());
- return std::prev(it);
-}
-
-} // namespace perfetto::trace_processor::util
diff --git a/src/trace_processor/util/file_buffer.h b/src/trace_processor/util/file_buffer.h
deleted file mode 100644
index 35e7384..0000000
--- a/src/trace_processor/util/file_buffer.h
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright (C) 2024 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef SRC_TRACE_PROCESSOR_UTIL_FILE_BUFFER_H_
-#define SRC_TRACE_PROCESSOR_UTIL_FILE_BUFFER_H_
-
-#include <cstddef>
-#include <optional>
-
-#include "perfetto/ext/base/circular_queue.h"
-#include "perfetto/trace_processor/trace_blob_view.h"
-
-namespace perfetto::trace_processor::util {
-
-// Helper class that exposes a window into the contents of a file. Data can be
-// appended to the end of the buffer (increasing the size of the window) or
-// removed from the front (decreasing the size of the window).
-//
-// TraceProcessor reads trace files in chunks and streams those to the
-// `ChunkedTraceReader` instance. But sometimes the reader needs to look into
-// the future (i.e. some data that has not yet arrived) before being able to
-// process the current data. In such a case the reader would have to buffer data
-// until the "future" data arrives. This class encapsulates that functionality.
-class FileBuffer {
- public:
- // Trivial empty ctor.
- FileBuffer() = default;
-
- bool empty() const { return data_.empty(); }
-
- // Returns the offset to the start of the buffered window of data.
- size_t file_offset() const {
- return data_.empty() ? end_offset_ : data_.front().file_offset;
- }
-
- // Adds a `TraceBlobView` at the back.
- void PushBack(TraceBlobView view);
-
- // Shrinks the buffer by dropping data from the front of the buffer until the
- // given offset is reached. If not enough data is present as much data as
- // possible will be dropped and `false` will be returned.
- // ATTENTION: If `offset` < 'file_offset()' (i.e. you try to access data
- // previously popped) this method will CHECK fail.
- bool PopFrontUntil(size_t offset);
-
- // Shrinks the buffer by dropping `bytes` from the front of the buffer. If not
- // enough data is present as much data as possible will be dropped and `false`
- // will be returned.
- bool PopFrontBytes(size_t bytes) {
- return PopFrontUntil(file_offset() + bytes);
- }
-
- // Similar to `TraceBlobView::slice_off`, creates a slice with data starting
- // at `offset` and of the given `length`. This method might need to allocate a
- // new buffer and copy data into it (if the requested data spans multiple
- // TraceBlobView instances). If not enough data is present `std::nullopt` is
- // returned.
- //
- // ATTENTION: If `offset` < 'file_offset()' (i.e. you try to access data
- // previously popped) this method will CHECK fail.
- std::optional<TraceBlobView> SliceOff(size_t offset, size_t length) const;
-
- private:
- struct Entry {
- // File offset of the first byte in `data`.
- size_t file_offset;
- TraceBlobView data;
- };
- using Iterator = base::CircularQueue<Entry>::Iterator;
- // Finds the `TraceBlobView` at `offset` and returns a slice starting at that
- // offset and spanning the rest of the `TraceBlobView`. It also returns an
- // iterator to the next `TraceBlobView` instance (which might be `end()`).
- Iterator FindEntryWithOffset(size_t offset) const;
-
- Iterator end() const { return data_.end(); }
-
- // CircularQueue has no const_iterator, so mutable is needed to access it from
- // const methods.
- // CircularQueue has no const_iterator, so mutable is needed to access it from
- // const methods.
- mutable base::CircularQueue<Entry> data_;
- size_t end_offset_ = 0;
-};
-
-} // namespace perfetto::trace_processor::util
-
-#endif // SRC_TRACE_PROCESSOR_UTIL_FILE_BUFFER_H_
diff --git a/src/trace_processor/util/trace_blob_view_reader.cc b/src/trace_processor/util/trace_blob_view_reader.cc
new file mode 100644
index 0000000..4ba5b13
--- /dev/null
+++ b/src/trace_processor/util/trace_blob_view_reader.cc
@@ -0,0 +1,122 @@
+
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/trace_processor/util/trace_blob_view_reader.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <optional>
+#include <utility>
+
+#include "perfetto/base/logging.h"
+#include "perfetto/public/compiler.h"
+#include "perfetto/trace_processor/trace_blob.h"
+#include "perfetto/trace_processor/trace_blob_view.h"
+
+namespace perfetto::trace_processor::util {
+
+void TraceBlobViewReader::PushBack(TraceBlobView data) {
+ if (data.size() == 0) {
+ return;
+ }
+ const size_t size = data.size();
+ data_.emplace_back(Entry{end_offset_, std::move(data)});
+ end_offset_ += size;
+}
+
+bool TraceBlobViewReader::PopFrontUntil(const size_t target_offset) {
+ PERFETTO_CHECK(start_offset() <= target_offset);
+ while (!data_.empty()) {
+ Entry& entry = data_.front();
+ if (target_offset == entry.start_offset) {
+ return true;
+ }
+ const size_t bytes_to_pop = target_offset - entry.start_offset;
+ if (entry.data.size() > bytes_to_pop) {
+ entry.data =
+ entry.data.slice_off(bytes_to_pop, entry.data.size() - bytes_to_pop);
+ entry.start_offset += bytes_to_pop;
+ return true;
+ }
+ data_.pop_front();
+ }
+ return target_offset == end_offset_;
+}
+
+std::optional<TraceBlobView> TraceBlobViewReader::SliceOff(
+ size_t offset,
+ size_t length) const {
+ PERFETTO_DCHECK(offset >= start_offset());
+
+ // Fast path: the slice fits entirely inside the first TBV, we can just slice
+ // that directly without doing any searching. This will happen most of the
+ // time when this class is used so optimize for it.
+ bool is_fast_path =
+ !data_.empty() &&
+ offset + length <= data_.front().start_offset + data_.front().data.size();
+ if (PERFETTO_LIKELY(is_fast_path)) {
+ return data_.front().data.slice_off(offset - data_.front().start_offset,
+ length);
+ }
+
+ // If the length is zero, then a zero-sized blob view is always approrpriate.
+ if (PERFETTO_UNLIKELY(length == 0)) {
+ return TraceBlobView();
+ }
+
+ // If we don't have any TBVs or the end of the slice does not fit, then we
+ // cannot possibly return a full slice.
+ if (PERFETTO_UNLIKELY(data_.empty() || offset + length > end_offset_)) {
+ return std::nullopt;
+ }
+
+ // Find the first block finishes *after* start_offset i.e. there is at least
+ // one byte in that block which will end up in the slice. We know this *must*
+ // exist because of the above check.
+ auto rit = std::upper_bound(
+ data_.begin(), data_.end(), offset, [](size_t offset, const Entry& rhs) {
+ return offset < rhs.start_offset + rhs.data.size();
+ });
+ PERFETTO_CHECK(rit != data_.end());
+
+ // If the slice fits entirely in the block we found, then just slice that
+ // block avoiding any copies.
+ size_t rel_off = offset - rit->start_offset;
+ if (rel_off + length <= rit->data.size()) {
+ return rit->data.slice_off(rel_off, length);
+ }
+
+ // Otherwise, allocate some memory and make a copy.
+ auto buffer = TraceBlob::Allocate(length);
+ uint8_t* ptr = buffer.data();
+ uint8_t* end = buffer.data() + buffer.size();
+
+ // Copy all bytes in this block which overlap with the slice.
+ memcpy(ptr, rit->data.data() + rel_off, rit->data.length() - rel_off);
+ ptr += rit->data.length() - rel_off;
+
+ for (auto it = rit + 1; ptr != end; ++it) {
+ auto len = std::min(static_cast<size_t>(end - ptr), it->data.size());
+ memcpy(ptr, it->data.data(), len);
+ ptr += len;
+ }
+ return TraceBlobView(std::move(buffer));
+}
+
+} // namespace perfetto::trace_processor::util
diff --git a/src/trace_processor/util/trace_blob_view_reader.h b/src/trace_processor/util/trace_blob_view_reader.h
new file mode 100644
index 0000000..c39ffab
--- /dev/null
+++ b/src/trace_processor/util/trace_blob_view_reader.h
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_TRACE_PROCESSOR_UTIL_TRACE_BLOB_VIEW_READER_H_
+#define SRC_TRACE_PROCESSOR_UTIL_TRACE_BLOB_VIEW_READER_H_
+
+#include <cstddef>
+#include <optional>
+
+#include "perfetto/ext/base/circular_queue.h"
+#include "perfetto/trace_processor/trace_blob_view.h"
+
+namespace perfetto::trace_processor::util {
+
+// Helper class which handles all the complexity of reading pieces of data which
+// span across multiple TraceBlobView chunks. It takes care of:
+// 1) Buffering data until it can be read.
+// 2) Stitching together the cross-chunk spanning pieces.
+// 3) Dropping data when it is no longer necessary to be buffered.
+class TraceBlobViewReader {
+ public:
+ // Adds a `TraceBlobView` at the back.
+ void PushBack(TraceBlobView);
+
+ // Shrinks the buffer by dropping data from the front of the buffer until the
+ // given offset is reached. If not enough data is present as much data as
+ // possible will be dropped and `false` will be returned.
+ //
+ // NOTE: If `offset` < 'file_offset()' this method will CHECK fail.
+ bool PopFrontUntil(size_t offset);
+
+ // Shrinks the buffer by dropping `bytes` from the front of the buffer. If not
+ // enough data is present as much data as possible will be dropped and `false`
+ // will be returned.
+ bool PopFrontBytes(size_t bytes) {
+ return PopFrontUntil(start_offset() + bytes);
+ }
+
+ // Creates a TraceBlobView by slicing this reader starting at |offset| and
+ // spanning |length| bytes.
+ //
+ // If possible, this method will try to avoid copies and simply slice an
+ // input TraceBlobView. However, that may not be possible and it so, it will
+ // allocate a new chunk of memory and copy over the data instead.
+ //
+ // NOTE: If `offset` < 'file_offset()' this method will CHECK fail.
+ std::optional<TraceBlobView> SliceOff(size_t offset, size_t length) const;
+
+ // Returns the offset to the start of the available data.
+ size_t start_offset() const {
+ return data_.empty() ? end_offset_ : data_.front().start_offset;
+ }
+
+ // Returns the offset to the end of the available data.
+ size_t end_offset() const { return end_offset_; }
+
+ // Returns the number of bytes of buffered data.
+ size_t avail() const { return end_offset() - start_offset(); }
+
+ bool empty() const { return data_.empty(); }
+
+ private:
+ struct Entry {
+ // File offset of the first byte in `data`.
+ size_t start_offset;
+ TraceBlobView data;
+ };
+ using Iterator = base::CircularQueue<Entry>::Iterator;
+
+ // CircularQueue has no const_iterator, so mutable is needed to access it from
+ // const methods.
+ mutable base::CircularQueue<Entry> data_;
+ size_t end_offset_ = 0;
+};
+
+} // namespace perfetto::trace_processor::util
+
+#endif // SRC_TRACE_PROCESSOR_UTIL_TRACE_BLOB_VIEW_READER_H_
diff --git a/src/trace_processor/util/file_buffer_unittest.cc b/src/trace_processor/util/trace_blob_view_reader_unittest.cc
similarity index 86%
rename from src/trace_processor/util/file_buffer_unittest.cc
rename to src/trace_processor/util/trace_blob_view_reader_unittest.cc
index 1518fe4..583648f 100644
--- a/src/trace_processor/util/file_buffer_unittest.cc
+++ b/src/trace_processor/util/trace_blob_view_reader_unittest.cc
@@ -14,12 +14,14 @@
* limitations under the License.
*/
-#include "src/trace_processor/util/file_buffer.h"
+#include "src/trace_processor/util/trace_blob_view_reader.h"
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstring>
+#include <optional>
+#include <ostream>
#include <vector>
#include "perfetto/trace_processor/trace_blob.h"
@@ -92,19 +94,21 @@
return chunks;
}
-FileBuffer CreateFileBuffer(const std::vector<TraceBlobView>& chunks) {
- FileBuffer chunked_buffer;
+TraceBlobViewReader CreateTraceBlobViewReader(
+ const std::vector<TraceBlobView>& chunks) {
+ TraceBlobViewReader chunked_buffer;
for (const auto& chunk : chunks) {
chunked_buffer.PushBack(chunk.copy());
}
return chunked_buffer;
}
-TEST(FileBuffer, ContiguousAccessAtOffset) {
+TEST(TraceBlobViewReader, ContiguousAccessAtOffset) {
constexpr size_t kExpectedSize = 256;
constexpr size_t kChunkSize = kExpectedSize / 4;
TraceBlobView expected_data = CreateExpectedData(kExpectedSize);
- FileBuffer buffer = CreateFileBuffer(Slice(expected_data, kChunkSize));
+ TraceBlobViewReader buffer =
+ CreateTraceBlobViewReader(Slice(expected_data, kChunkSize));
for (size_t file_offset = 0; file_offset <= kExpectedSize; ++file_offset) {
EXPECT_TRUE(buffer.PopFrontUntil(file_offset));
@@ -116,12 +120,12 @@
}
}
-TEST(FileBuffer, NoCopyIfDataIsContiguous) {
+TEST(TraceBlobViewReader, NoCopyIfDataIsContiguous) {
constexpr size_t kExpectedSize = 256;
constexpr size_t kChunkSize = kExpectedSize / 4;
std::vector<TraceBlobView> chunks =
Slice(CreateExpectedData(kExpectedSize), kChunkSize);
- FileBuffer buffer = CreateFileBuffer(chunks);
+ TraceBlobViewReader buffer = CreateTraceBlobViewReader(chunks);
for (size_t i = 0; i < chunks.size(); ++i) {
for (size_t off = 0; off < kChunkSize; ++off) {
@@ -133,17 +137,18 @@
}
}
-TEST(FileBuffer, PopRemovesData) {
+TEST(TraceBlobViewReader, PopRemovesData) {
size_t expected_size = 256;
size_t expected_file_offset = 0;
const size_t kChunkSize = expected_size / 4;
TraceBlobView expected_data = CreateExpectedData(expected_size);
- FileBuffer buffer = CreateFileBuffer(Slice(expected_data, kChunkSize));
+ TraceBlobViewReader buffer =
+ CreateTraceBlobViewReader(Slice(expected_data, kChunkSize));
--expected_size;
++expected_file_offset;
buffer.PopFrontUntil(expected_file_offset);
- EXPECT_THAT(buffer.file_offset(), Eq(expected_file_offset));
+ EXPECT_THAT(buffer.start_offset(), Eq(expected_file_offset));
EXPECT_THAT(buffer.SliceOff(expected_file_offset, expected_size),
Optional(SameDataAs(expected_data.slice_off(
expected_data.size() - expected_size, expected_size))));
@@ -151,7 +156,7 @@
expected_size -= kChunkSize;
expected_file_offset += kChunkSize;
buffer.PopFrontUntil(expected_file_offset);
- EXPECT_THAT(buffer.file_offset(), Eq(expected_file_offset));
+ EXPECT_THAT(buffer.start_offset(), Eq(expected_file_offset));
EXPECT_THAT(buffer.SliceOff(expected_file_offset, expected_size),
Optional(SameDataAs(expected_data.slice_off(
expected_data.size() - expected_size, expected_size))));
