TraceProcessor: improve parsing performances to ~420 MB/s A bunch of minor changes that boost the trace processing performance from ~200 MB/s to 420 MB/s. Because fast. Perf measured on a MacBook Pro Mid 2015 - 2.8 GHz i7 using a 1GB trace (go/pft-1gb). Change-Id: I27e3517ac2177b3f21a7290b123115a3d49ec3b2
diff --git a/Android.bp b/Android.bp index aae56a8..9ad3aa6 100644 --- a/Android.bp +++ b/Android.bp
@@ -52,7 +52,6 @@ "src/protozero/message_handle.cc", "src/protozero/proto_decoder.cc", "src/protozero/proto_field_descriptor.cc", - "src/protozero/proto_utils.cc", "src/protozero/scattered_stream_null_delegate.cc", "src/protozero/scattered_stream_writer.cc", "src/traced/probes/filesystem/file_scanner.cc", @@ -175,7 +174,6 @@ "src/protozero/message_handle.cc", "src/protozero/proto_decoder.cc", "src/protozero/proto_field_descriptor.cc", - "src/protozero/proto_utils.cc", "src/protozero/scattered_stream_null_delegate.cc", "src/protozero/scattered_stream_writer.cc", "src/tracing/core/chrome_config.cc", @@ -310,7 +308,6 @@ "src/protozero/message_handle.cc", "src/protozero/proto_decoder.cc", "src/protozero/proto_field_descriptor.cc", - "src/protozero/proto_utils.cc", "src/protozero/scattered_stream_delegate_for_testing.cc", "src/protozero/scattered_stream_null_delegate.cc", "src/protozero/scattered_stream_writer.cc", @@ -3497,7 +3494,6 @@ "src/protozero/message_handle.cc", "src/protozero/proto_decoder.cc", "src/protozero/proto_field_descriptor.cc", - "src/protozero/proto_utils.cc", "src/protozero/scattered_stream_null_delegate.cc", "src/protozero/scattered_stream_writer.cc", "src/tracing/core/chrome_config.cc", @@ -3703,7 +3699,6 @@ "src/protozero/proto_decoder.cc", "src/protozero/proto_decoder_unittest.cc", "src/protozero/proto_field_descriptor.cc", - "src/protozero/proto_utils.cc", "src/protozero/proto_utils_unittest.cc", "src/protozero/scattered_stream_delegate_for_testing.cc", "src/protozero/scattered_stream_null_delegate.cc",
diff --git a/include/perfetto/base/utils.h b/include/perfetto/base/utils.h index af341a1..9b18214 100644 --- a/include/perfetto/base/utils.h +++ b/include/perfetto/base/utils.h
@@ -50,6 +50,14 @@ #define PERFETTO_WARN_UNUSED_RESULT #endif +#if defined(__clang__) +#define PERFETTO_ALWAYS_INLINE __attribute__((__always_inline__)) +#else +// GCC is too pedantic and often fails with the error: +// "always_inline function might not be inlinable" +#define PERFETTO_ALWAYS_INLINE +#endif + namespace perfetto { namespace base {
diff --git a/include/perfetto/protozero/proto_decoder.h b/include/perfetto/protozero/proto_decoder.h index 077f4c8..fcd6638 100644 --- a/include/perfetto/protozero/proto_decoder.h +++ b/include/perfetto/protozero/proto_decoder.h
@@ -75,7 +75,8 @@ }; // Creates a ProtoDecoder using the given |buffer| with size |length| bytes. - ProtoDecoder(const uint8_t* buffer, uint64_t length); + inline ProtoDecoder(const uint8_t* buffer, uint64_t length) + : buffer_(buffer), length_(length), current_position_(buffer) {} // Reads the next field from the buffer. If the full field cannot be read, // the returned struct will have id 0 which is an invalid field id. @@ -83,16 +84,28 @@ // Returns true if |length_| == |current_position_| - |buffer| and false // otherwise. - bool IsEndOfBuffer(); + inline bool IsEndOfBuffer() { + PERFETTO_DCHECK(current_position_ >= buffer_); + return length_ == static_cast<uint64_t>(current_position_ - buffer_); + } // Resets the current position to the start of the buffer. - void Reset(); + inline void Reset() { current_position_ = buffer_; } + + // Resets to the given position (must be within the buffer). + inline void Reset(const uint8_t* pos) { + PERFETTO_DCHECK(pos >= buffer_ && pos < buffer_ + length_); + current_position_ = pos; + } // Return's offset inside the buffer. - uint64_t offset() const { + inline uint64_t offset() const { return static_cast<uint64_t>(current_position_ - buffer_); } + inline const uint8_t* buffer() const { return buffer_; } + inline uint64_t length() const { return length_; } + private: const uint8_t* const buffer_; const uint64_t length_; // The outer buffer can be larger than 4GB.
diff --git a/include/perfetto/protozero/proto_utils.h b/include/perfetto/protozero/proto_utils.h index cb1a715..d73f652 100644 --- a/include/perfetto/protozero/proto_utils.h +++ b/include/perfetto/protozero/proto_utils.h
@@ -22,6 +22,9 @@ #include <type_traits> +#include "perfetto/base/logging.h" +#include "perfetto/base/utils.h" + namespace protozero { namespace proto_utils { @@ -110,9 +113,23 @@ // to the next unconsumed byte (so start < retval <= end) or |start| if the // VarInt could not be fully parsed because there was not enough space in the // buffer. -const uint8_t* ParseVarInt(const uint8_t* start, - const uint8_t* end, - uint64_t* value); +inline const uint8_t* ParseVarInt(const uint8_t* start, + const uint8_t* end, + uint64_t* value) { + const uint8_t* pos = start; + uint64_t shift = 0; + *value = 0; + do { + if (PERFETTO_UNLIKELY(pos >= end)) { + *value = 0; + return start; + } + PERFETTO_DCHECK(shift < 64ull); + *value |= static_cast<uint64_t>(*pos & 0x7f) << shift; + shift += 7; + } while (*pos++ & 0x80); + return pos; +} } // namespace proto_utils } // namespace protozero
diff --git a/src/protozero/BUILD.gn b/src/protozero/BUILD.gn index bfd0509..d2d27aa 100644 --- a/src/protozero/BUILD.gn +++ b/src/protozero/BUILD.gn
@@ -31,7 +31,6 @@ "message_handle.cc", "proto_decoder.cc", "proto_field_descriptor.cc", - "proto_utils.cc", "scattered_stream_null_delegate.cc", "scattered_stream_writer.cc", ]
diff --git a/src/protozero/proto_decoder.cc b/src/protozero/proto_decoder.cc index 2e2754e..d9cf294 100644 --- a/src/protozero/proto_decoder.cc +++ b/src/protozero/proto_decoder.cc
@@ -32,8 +32,6 @@ #error Unimplemented for big endian archs. #endif -ProtoDecoder::ProtoDecoder(const uint8_t* buffer, uint64_t length) - : buffer_(buffer), length_(length), current_position_(buffer) {} ProtoDecoder::Field ProtoDecoder::ReadField() { Field field{}; @@ -56,36 +54,22 @@ } uint64_t raw_field_id = 0; - pos = ParseVarInt(pos, end, &raw_field_id); + if (PERFETTO_LIKELY(*pos < 0x80)) { + raw_field_id = *(pos++); // Fastpath for fields with ID < 32. + } else { + pos = ParseVarInt(pos, end, &raw_field_id); + } uint32_t field_id = static_cast<uint32_t>(raw_field_id >> kFieldTypeNumBits); if (field_id == 0 || pos >= end) { return field; } + field.type = static_cast<FieldType>(raw_field_id & kFieldTypeMask); const uint8_t* new_pos = nullptr; uint64_t field_intvalue = 0; switch (field.type) { - case kFieldTypeFixed64: { - if (pos + sizeof(uint64_t) > end) { - return field; - } - memcpy(&field_intvalue, pos, sizeof(uint64_t)); - field.int_value = BYTE_SWAP_TO_LE64(field_intvalue); - pos += sizeof(uint64_t); - break; - } - case kFieldTypeFixed32: { - if (pos + sizeof(uint32_t) > end) { - return field; - } - uint32_t tmp; - memcpy(&tmp, pos, sizeof(uint32_t)); - field.int_value = BYTE_SWAP_TO_LE32(tmp); - pos += sizeof(uint32_t); - break; - } case kFieldTypeVarInt: { new_pos = ParseVarInt(pos, end, &field.int_value); @@ -117,6 +101,25 @@ pos += field_intvalue; break; } + case kFieldTypeFixed64: { + if (pos + sizeof(uint64_t) > end) { + return field; + } + memcpy(&field_intvalue, pos, sizeof(uint64_t)); + field.int_value = BYTE_SWAP_TO_LE64(field_intvalue); + pos += sizeof(uint64_t); + break; + } + case kFieldTypeFixed32: { + if (pos + sizeof(uint32_t) > end) { + return field; + } + uint32_t tmp; + memcpy(&tmp, pos, sizeof(uint32_t)); + field.int_value = BYTE_SWAP_TO_LE32(tmp); + pos += sizeof(uint32_t); + break; + } } // Set the field id to make the returned value valid and update the current // position in the buffer. @@ -125,13 +128,4 @@ return field; } -bool ProtoDecoder::IsEndOfBuffer() { - PERFETTO_DCHECK(current_position_ >= buffer_); - return length_ == static_cast<uint64_t>(current_position_ - buffer_); -} - -void ProtoDecoder::Reset() { - current_position_ = buffer_; -} - } // namespace protozero
diff --git a/src/protozero/proto_utils.cc b/src/protozero/proto_utils.cc deleted file mode 100644 index 7a3798a..0000000 --- a/src/protozero/proto_utils.cc +++ /dev/null
@@ -1,48 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "perfetto/protozero/proto_utils.h" - -#include <string.h> - -#include <limits> - -#include "perfetto/base/logging.h" -#include "perfetto/base/utils.h" - -namespace protozero { -namespace proto_utils { - -const uint8_t* ParseVarInt(const uint8_t* start, - const uint8_t* end, - uint64_t* value) { - const uint8_t* pos = start; - uint64_t shift = 0; - *value = 0; - do { - if (PERFETTO_UNLIKELY(pos >= end)) { - *value = 0; - return start; - } - PERFETTO_DCHECK(shift < 64ull); - *value |= static_cast<uint64_t>(*pos & 0x7f) << shift; - shift += 7; - } while (*pos++ & 0x80); - return pos; -} - -} // namespace proto_utils -} // namespace protozero
diff --git a/src/trace_processor/file_reader.cc b/src/trace_processor/file_reader.cc index f7418d1..0ff94b9 100644 --- a/src/trace_processor/file_reader.cc +++ b/src/trace_processor/file_reader.cc
@@ -37,7 +37,7 @@ uint32_t FileReader::Read(uint64_t offset, uint32_t len, uint8_t* dst) { ssize_t res = pread(*fd_, dst, len, static_cast<off_t>(offset)); uint64_t size_read = offset + static_cast<uint64_t>(res); - if (print_progress_ && size_read >= last_progress_bytes_ + 1E7) { + if (print_progress_ && (!res || size_read >= last_progress_bytes_ + 1E8)) { last_progress_bytes_ = size_read; double progress = size_read * 100.0 / file_size_; fprintf(stderr, "\rReading trace: %.2f%% %.1f MB / %.1f MB", progress,
diff --git a/src/trace_processor/proto_trace_parser.cc b/src/trace_processor/proto_trace_parser.cc index 5d3e378..2d43fcc 100644 --- a/src/trace_processor/proto_trace_parser.cc +++ b/src/trace_processor/proto_trace_parser.cc
@@ -35,20 +35,25 @@ using protozero::ProtoDecoder; using protozero::proto_utils::kFieldTypeLengthDelimited; +using protozero::proto_utils::ParseVarInt; +using protozero::proto_utils::MakeTagVarInt; namespace { -bool FindIntField(ProtoDecoder* decoder, - uint32_t field_id, - uint64_t* field_value) { +template <int field_id> +inline bool FindIntField(ProtoDecoder* decoder, uint64_t* field_value) { + bool res = false; for (auto f = decoder->ReadField(); f.id != 0; f = decoder->ReadField()) { if (f.id == field_id) { *field_value = f.int_value; - return true; + res = true; + break; } } - return false; + decoder->Reset(); + return res; } + } // namespace ProtoTraceParser::ProtoTraceParser(BlobReader* reader, @@ -164,12 +169,20 @@ size_t length) { ProtoDecoder decoder(data, length); uint64_t cpu = 0; - if (!FindIntField(&decoder, protos::FtraceEventBundle::kCpuFieldNumber, - &cpu)) { - PERFETTO_ELOG("CPU field not found in FtraceEventBundle"); - return; + constexpr auto kCpuFieldNumber = protos::FtraceEventBundle::kCpuFieldNumber; + constexpr auto kCpuFieldTag = MakeTagVarInt(kCpuFieldNumber); + + // Speculate on the fact that the cpu is often pushed as the 2nd last field + // And is a number < 128. + if (PERFETTO_LIKELY(length > 4 && data[length - 4] == kCpuFieldTag) && + data[length - 3] < 0x80) { + cpu = data[length - 3]; + } else { + if (!PERFETTO_LIKELY((FindIntField<kCpuFieldNumber>(&decoder, &cpu)))) { + PERFETTO_ELOG("CPU field not found in FtraceEventBundle"); + return; + } } - decoder.Reset(); for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) { switch (fld.id) { @@ -183,17 +196,33 @@ PERFETTO_DCHECK(decoder.IsEndOfBuffer()); } -void ProtoTraceParser::ParseFtraceEvent(uint32_t cpu, - const uint8_t* data, - size_t length) { +PERFETTO_ALWAYS_INLINE void ProtoTraceParser::ParseFtraceEvent( + uint32_t cpu, + const uint8_t* data, + size_t length) { ProtoDecoder decoder(data, length); - uint64_t timestamp = 0; - if (!FindIntField(&decoder, protos::FtraceEvent::kTimestampFieldNumber, - ×tamp)) { + constexpr auto kTimestampFieldNumber = + protos::FtraceEvent::kTimestampFieldNumber; + uint64_t timestamp; + bool timestamp_found = false; + + // Speculate on the fact that the timestamp is often the 1st field of the + // event. + constexpr auto timestampFieldTag = MakeTagVarInt(kTimestampFieldNumber); + if (PERFETTO_LIKELY(length > 10 && data[0] == timestampFieldTag)) { + // Fastpath. + const uint8_t* next = ParseVarInt(data + 1, data + 10, ×tamp); + timestamp_found = next != data + 1; + decoder.Reset(next); + } else { + // Slowpath. + timestamp_found = FindIntField<kTimestampFieldNumber>(&decoder, ×tamp); + } + + if (PERFETTO_UNLIKELY(!timestamp_found)) { PERFETTO_ELOG("Timestamp field not found in FtraceEvent"); return; } - decoder.Reset(); for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) { switch (fld.id) { @@ -208,12 +237,12 @@ PERFETTO_DCHECK(decoder.IsEndOfBuffer()); } -void ProtoTraceParser::ParseSchedSwitch(uint32_t cpu, - uint64_t timestamp, - const uint8_t* data, - size_t length) { +PERFETTO_ALWAYS_INLINE void ProtoTraceParser::ParseSchedSwitch( + uint32_t cpu, + uint64_t timestamp, + const uint8_t* data, + size_t length) { ProtoDecoder decoder(data, length); - uint32_t prev_pid = 0; uint32_t prev_state = 0; base::StringView prev_comm;
diff --git a/src/traced/probes/ftrace/cpu_reader.cc b/src/traced/probes/ftrace/cpu_reader.cc index d031021..6ad7cd9 100644 --- a/src/traced/probes/ftrace/cpu_reader.cc +++ b/src/traced/probes/ftrace/cpu_reader.cc
@@ -283,6 +283,10 @@ auto* filter = data_source->event_filter(); size_t evt_size = ParsePage(buffer, filter, bundle, table_, metadata); PERFETTO_DCHECK(evt_size); + + // Note: The fastpath in proto_decoder.cc speculates on the fact that + // the cpu field is exactly -4 bytes from the end of the proto message. + // If this changes, change the proto_decoder.cc accordingly. bundle->set_cpu(static_cast<uint32_t>(cpu_)); bundle->set_overwrite_count(metadata->overwrite_count); }
diff --git a/tools/trace_to_text/main.cc b/tools/trace_to_text/main.cc index a0488c3..b775326 100644 --- a/tools/trace_to_text/main.cc +++ b/tools/trace_to_text/main.cc
@@ -163,7 +163,10 @@ protos::TracePacket packet; auto res = packet.ParseFromArray(buf.get(), static_cast<int>(field_size)); - PERFETTO_CHECK(res); + if (!res) { + PERFETTO_ELOG("Skipping invalid packet"); + continue; + } f(packet); } }