Support gzip files in trace_to_text
Core change in trace_to_text:
1. Add `OnlineTraceToText` utility by refactoring the core TraceToText
code to make it easy to use, easy to test independently without much
boilerplate code and easy to compose with other streaming algorithms.
Client OnlineTraceToText can continue to `feed` it a sequence of
mem-blocks, and it will continue to write the output in a out_stream.
2. Rewrite trace_to_text driver function, which first figures out if the
input file is gzip or proto files and setup/wire the streaming
components as follows:
when gzip file:
{InputFile} --> {Streaming GzipDecompress} --> {OnlineTraceToText}
||
||
\/
{Output}
when proto file:
{InputFile} --> {OnlineTraceToText} --> {Output}
The data is streamed from one component to the next one, in a
sequence of mem-block, and finally written to the output_file/stdout.
3. Some other minor cleanup/refactoring in trace_to_text code.
4. Manual testing:
For `example_android_trace_15s.gz` attached on b/148130323:
>> ./tools/traceconv text /tmp/e_trace_15s > /tmp/p1.txt
>> ./out/default/trace_to_text text /tmp/e_trace_15s > /tmp/p2.txt
>> ./out/default/trace_to_text text /tmp/e_trace_15s.gz > /tmp/p3.txt
>> md5sum /tmp/p1.txt
ac7b8b397d3e5e543e91d2c909c35416 // [Baseline (before this CL)]
>> md5sum /tmp/p2.txt
ac7b8b397d3e5e543e91d2c909c35416 // [trace2text after this CL]
>> md5sum /tmp/p3.txt
ac7b8b397d3e5e543e91d2c909c35416 // [decompress + trace2text after
this CL]
Bug: 148130323
Change-Id: I2c8aa080aba45434e202bd9ff99c4a20de69571ediff --git a/Android.bp b/Android.bp
index 8d92551..b74dcf6 100644
--- a/Android.bp
+++ b/Android.bp
@@ -8492,6 +8492,7 @@
name: "perfetto_src_trace_processor_util_unittests",
srcs: [
"src/trace_processor/util/debug_annotation_parser_unittest.cc",
+ "src/trace_processor/util/gzip_utils_unittest.cc",
"src/trace_processor/util/proto_to_args_parser_unittest.cc",
"src/trace_processor/util/protozero_to_text_unittests.cc",
],
@@ -9181,7 +9182,6 @@
name: "perfetto_tools_trace_to_text_full",
srcs: [
"tools/trace_to_text/proto_full_utils.cc",
- "tools/trace_to_text/trace_to_text.cc",
],
}
@@ -9208,6 +9208,22 @@
],
}
+// GN: //tools/trace_to_text:trace_to_text_lib
+filegroup {
+ name: "perfetto_tools_trace_to_text_trace_to_text_lib",
+ srcs: [
+ "tools/trace_to_text/trace_to_text.cc",
+ ],
+}
+
+// GN: //tools/trace_to_text:unittests
+filegroup {
+ name: "perfetto_tools_trace_to_text_unittests",
+ srcs: [
+ "tools/trace_to_text/trace_to_text_unittest.cc",
+ ],
+}
+
// GN: //tools/trace_to_text:utils
filegroup {
name: "perfetto_tools_trace_to_text_utils",
@@ -9478,6 +9494,7 @@
":perfetto_src_profiling_perf_producer_unittests",
":perfetto_src_profiling_perf_regs_parsing",
":perfetto_src_profiling_perf_unwinding",
+ ":perfetto_src_profiling_symbolizer_symbolize_database",
":perfetto_src_profiling_symbolizer_symbolizer",
":perfetto_src_profiling_symbolizer_unittests",
":perfetto_src_profiling_unittests",
@@ -9578,6 +9595,9 @@
":perfetto_src_tracing_test_tracing_integration_test",
":perfetto_src_tracing_unittests",
":perfetto_tools_sanitizers_unittests_sanitizers_unittests",
+ ":perfetto_tools_trace_to_text_trace_to_text_lib",
+ ":perfetto_tools_trace_to_text_unittests",
+ ":perfetto_tools_trace_to_text_utils",
],
shared_libs: [
"libandroidicu",
@@ -9703,6 +9723,7 @@
"perfetto_src_traced_probes_ftrace_test_messages_cpp_gen_headers",
"perfetto_src_traced_probes_ftrace_test_messages_lite_gen_headers",
"perfetto_src_traced_probes_ftrace_test_messages_zero_gen_headers",
+ "perfetto_tools_trace_to_text_gen_cc_trace_descriptor",
],
defaults: [
"perfetto_defaults",
@@ -9979,6 +10000,7 @@
":perfetto_tools_trace_to_text_common",
":perfetto_tools_trace_to_text_full",
":perfetto_tools_trace_to_text_pprofbuilder",
+ ":perfetto_tools_trace_to_text_trace_to_text_lib",
":perfetto_tools_trace_to_text_utils",
],
static_libs: [
diff --git a/BUILD b/BUILD
index 9d48d47..ca0adb7 100644
--- a/BUILD
+++ b/BUILD
@@ -1929,7 +1929,6 @@
srcs = [
"tools/trace_to_text/proto_full_utils.cc",
"tools/trace_to_text/proto_full_utils.h",
- "tools/trace_to_text/trace_to_text.cc",
],
)
@@ -1951,6 +1950,16 @@
],
)
+# GN target: //tools/trace_to_text:trace_to_text_lib
+perfetto_filegroup(
+ name = "tools_trace_to_text_trace_to_text_lib",
+ srcs = [
+ "tools/trace_to_text/proto_full_utils.h",
+ "tools/trace_to_text/trace_to_text.cc",
+ "tools/trace_to_text/trace_to_text.h",
+ ],
+)
+
# GN target: //tools/trace_to_text:utils
perfetto_filegroup(
name = "tools_trace_to_text_utils",
@@ -4022,6 +4031,7 @@
":tools_trace_to_text_common",
":tools_trace_to_text_full",
":tools_trace_to_text_pprofbuilder",
+ ":tools_trace_to_text_trace_to_text_lib",
":tools_trace_to_text_utils",
],
visibility = [
diff --git a/gn/perfetto_unittests.gni b/gn/perfetto_unittests.gni
index 1a2b482..dbfa0fa 100644
--- a/gn/perfetto_unittests.gni
+++ b/gn/perfetto_unittests.gni
@@ -80,6 +80,7 @@
if (enable_perfetto_trace_processor) {
perfetto_unittests_targets += [ "src/trace_processor:unittests" ]
+ perfetto_unittests_targets += [ "tools/trace_to_text:unittests" ]
if (enable_perfetto_trace_processor_sqlite) {
perfetto_unittests_targets += [ "src/trace_processor/metrics:unittests" ]
diff --git a/include/perfetto/ext/base/hash.h b/include/perfetto/ext/base/hash.h
index 50669f9..158a886 100644
--- a/include/perfetto/ext/base/hash.h
+++ b/include/perfetto/ext/base/hash.h
@@ -45,11 +45,14 @@
void Update(const char* data, size_t size) {
for (size_t i = 0; i < size; i++) {
result_ ^= static_cast<uint8_t>(data[i]);
+ // Note: Arithmetic overflow of unsigned integers is well defined in C++
+ // standard unlike signed integers.
+ // https://stackoverflow.com/a/41280273
result_ *= kFnv1a64Prime;
}
}
- uint64_t digest() { return result_; }
+ uint64_t digest() const { return result_; }
private:
static constexpr uint64_t kFnv1a64OffsetBasis = 0xcbf29ce484222325;
diff --git a/src/trace_processor/util/BUILD.gn b/src/trace_processor/util/BUILD.gn
index 78ee878..82b2c7e 100644
--- a/src/trace_processor/util/BUILD.gn
+++ b/src/trace_processor/util/BUILD.gn
@@ -117,9 +117,13 @@
"proto_to_args_parser_unittest.cc",
"protozero_to_text_unittests.cc",
]
+ if (enable_perfetto_zlib) {
+ sources += [ "gzip_utils_unittest.cc" ]
+ }
testonly = true
deps = [
":descriptors",
+ ":gzip",
":proto_to_args_parser",
":protozero_to_text",
"..:gen_cc_test_messages_descriptor",
diff --git a/src/trace_processor/util/gzip_utils.cc b/src/trace_processor/util/gzip_utils.cc
index e198bd2..60431fb 100644
--- a/src/trace_processor/util/gzip_utils.cc
+++ b/src/trace_processor/util/gzip_utils.cc
@@ -93,13 +93,25 @@
GzipDecompressor::GzipDecompressor() = default;
GzipDecompressor::~GzipDecompressor() = default;
void GzipDecompressor::Reset() {}
-void GzipDecompressor::SetInput(const uint8_t*, size_t) {}
-GzipDecompressor::Result GzipDecompressor::Decompress(uint8_t*, size_t) {
+void GzipDecompressor::Feed(const uint8_t*, size_t) {}
+GzipDecompressor::Result GzipDecompressor::ExtractOutput(uint8_t*, size_t) {
Result{ResultCode::kError, 0};
}
#endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
+// static
+std::vector<uint8_t> GzipDecompressor::DecompressFully(const uint8_t* data,
+ size_t len) {
+ std::vector<uint8_t> whole_data;
+ GzipDecompressor decompressor;
+ auto decom_output_consumer = [&](const uint8_t* buf, size_t buf_len) {
+ whole_data.insert(whole_data.end(), buf, buf + buf_len);
+ };
+ decompressor.FeedAndExtract(data, len, decom_output_consumer);
+ return whole_data;
+}
+
} // namespace util
} // namespace trace_processor
} // namespace perfetto
diff --git a/src/trace_processor/util/gzip_utils.h b/src/trace_processor/util/gzip_utils.h
index 85cd217..711541f 100644
--- a/src/trace_processor/util/gzip_utils.h
+++ b/src/trace_processor/util/gzip_utils.h
@@ -18,6 +18,7 @@
#define SRC_TRACE_PROCESSOR_UTIL_GZIP_UTILS_H_
#include <memory>
+#include <vector>
struct z_stream_s;
@@ -103,6 +104,11 @@
// but without paying the cost of internal memory allocation.
void Reset();
+ // Decompress the entire mem-block and return decompressed mem-block.
+ // This is used for decompressing small strings or small files
+ // which doesn't require streaming decompression.
+ static std::vector<uint8_t> DecompressFully(const uint8_t* data, size_t len);
+
private:
std::unique_ptr<z_stream_s> z_stream_;
};
diff --git a/src/trace_processor/util/gzip_utils_unittest.cc b/src/trace_processor/util/gzip_utils_unittest.cc
new file mode 100644
index 0000000..a58332e
--- /dev/null
+++ b/src/trace_processor/util/gzip_utils_unittest.cc
@@ -0,0 +1,145 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "test/gtest_and_gmock.h"
+
+#include "src/trace_processor/util/gzip_utils.h"
+
+#include <zlib.h>
+#include <fstream>
+#include <iostream>
+#include "perfetto/base/logging.h"
+
+using std::string;
+
+namespace perfetto {
+namespace trace_processor {
+namespace util {
+
+static std::string TrivialGzipCompress(const std::string& input) {
+ constexpr auto buffer_len = 10000;
+ std::unique_ptr<char[]> output_ptr(new char[buffer_len]);
+ char* output = output_ptr.get();
+ z_stream defstream;
+ defstream.zalloc = Z_NULL;
+ defstream.zfree = Z_NULL;
+ defstream.opaque = Z_NULL;
+ defstream.avail_in = uint32_t(input.size());
+ defstream.next_in =
+ const_cast<Bytef*>(reinterpret_cast<const Bytef*>(input.data()));
+ defstream.avail_out = buffer_len;
+ defstream.next_out = reinterpret_cast<Bytef*>(output);
+ deflateInit(&defstream, Z_BEST_COMPRESSION); // GZip decompress
+ deflate(&defstream, Z_FINISH);
+ deflateEnd(&defstream);
+ PERFETTO_CHECK(defstream.avail_out > 0);
+ return std::string(output, buffer_len - defstream.avail_out);
+}
+
+// Trivially decompress using ZlibOnlineDecompress.
+// It's called 'trivial' because we are feeding the entire input in one shot.
+static std::string TrivialDecompress(const std::string& input) {
+ string output;
+ GzipDecompressor decompressor;
+ decompressor.FeedAndExtract(
+ reinterpret_cast<const uint8_t*>(input.data()), uint32_t(input.size()),
+ [&](const uint8_t* data, size_t len) {
+ output.append(reinterpret_cast<const char*>(data), len);
+ });
+ return output;
+}
+
+// Decompress a large GZip file using a in-memory buffer of 4KB, and write the
+// decompressed output in another file.
+static void DecompressGzipFileInFileOut(const std::string& input_file,
+ const std::string& output_file) {
+ std::ofstream output(output_file.c_str(), std::ios::out | std::ios::binary);
+ std::ifstream input(input_file.c_str(), std::ios::binary);
+ GzipDecompressor decompressor;
+ constexpr uint32_t buffer_sizeof = 4096;
+ char buffer[buffer_sizeof];
+ while (!input.eof()) {
+ input.read(buffer, buffer_sizeof);
+ decompressor.FeedAndExtract(
+ reinterpret_cast<const uint8_t*>(buffer), size_t(input.gcount()),
+ [&](const uint8_t* data, size_t len) {
+ output.write(reinterpret_cast<const char*>(data),
+ std::streamsize(len));
+ });
+ }
+ EXPECT_FALSE(input.bad());
+}
+
+TEST(GzipDecompressor, Basic) {
+ string input = "Abc..Def..Ghi";
+ string compressed = TrivialGzipCompress(input);
+ EXPECT_EQ(21u, compressed.size());
+ string decompressed = TrivialDecompress(compressed);
+ EXPECT_EQ(input, decompressed);
+}
+
+TEST(GzipDecompressor, Streaming) {
+ string input = "Abc..Def..Ghi";
+ string compressed = TrivialGzipCompress(input);
+ string decompressed;
+ auto consumer = [&](const uint8_t* data, size_t len) {
+ decompressed.append(reinterpret_cast<const char*>(data), len);
+ };
+ GzipDecompressor decompressor;
+ auto compressed_u8 = reinterpret_cast<const uint8_t*>(compressed.data());
+ ASSERT_GT(compressed.size(), 17u);
+ decompressor.FeedAndExtract(compressed_u8, 7, consumer);
+ decompressor.FeedAndExtract(compressed_u8 + 7, 10, consumer);
+ decompressor.FeedAndExtract(compressed_u8 + 17, compressed.size() - 17,
+ consumer);
+
+ EXPECT_EQ(input, decompressed);
+}
+
+static std::string ReadFile(const std::string& file_name) {
+ std::ifstream fd(file_name, std::ios::binary);
+ std::stringstream buffer;
+ buffer << fd.rdbuf();
+ fd.close();
+ return buffer.str();
+}
+
+static void WriteFile(const std::string& file_name,
+ const std::string& content) {
+ std::ofstream fd(file_name, std::ios::out | std::ios::binary);
+ fd.write(content.data(), std::streamsize(content.size()));
+ fd.close();
+}
+
+TEST(GzipDecompressor, DISABLED_FileInFileOut) {
+ auto big_string = []() {
+ std::string output;
+ for (int i = 0; i < 1000; i++) {
+ output += "Abc..Def..Ghi."; // len = 14
+ }
+ return output;
+ }();
+ constexpr auto gz_file = "/tmp/abc.gz";
+ constexpr auto txt_file = "/tmp/abc.txt";
+ EXPECT_EQ(size_t(1000 * 14), big_string.size());
+ WriteFile(gz_file, TrivialGzipCompress(big_string));
+ DecompressGzipFileInFileOut(gz_file, txt_file);
+ EXPECT_TRUE(ReadFile(txt_file) == big_string);
+}
+
+} // namespace util
+} // namespace trace_processor
+} // namespace perfetto
diff --git a/tools/trace_to_text/BUILD.gn b/tools/trace_to_text/BUILD.gn
index 1abb0a8..44ad2f6 100644
--- a/tools/trace_to_text/BUILD.gn
+++ b/tools/trace_to_text/BUILD.gn
@@ -15,6 +15,7 @@
import("../../gn/perfetto.gni")
import("../../gn/perfetto_cc_proto_descriptor.gni")
import("../../gn/perfetto_host_executable.gni")
+import("../../gn/test.gni")
import("../../gn/wasm.gni")
perfetto_host_executable("trace_to_text") {
@@ -83,6 +84,23 @@
public_deps = [ ":pprofbuilder" ]
}
+source_set("trace_to_text_lib") {
+ deps = [
+ ":gen_cc_trace_descriptor",
+ ":utils",
+ "../../gn:default_deps",
+ "../../include/perfetto/base",
+ "../../src/protozero:proto_ring_buffer",
+ "../../src/trace_processor:storage_minimal",
+ "../../src/trace_processor/util:gzip",
+ ]
+ sources = [
+ "proto_full_utils.h",
+ "trace_to_text.cc",
+ "trace_to_text.h",
+ ]
+}
+
# The core source files that are used both by the "full" version (the host
# executable) and by the "lite" version (the WASM module for the UI).
source_set("common") {
@@ -132,12 +150,11 @@
testonly = true
deps = [
":common",
- ":gen_cc_trace_descriptor",
+ ":trace_to_text_lib",
":utils",
"../../gn:default_deps",
"../../gn:protobuf_full",
"../../protos/perfetto/trace:zero",
- "../../src/protozero:proto_ring_buffer",
]
if (enable_perfetto_zlib) {
deps += [ "../../gn:zlib" ]
@@ -145,7 +162,6 @@
sources = [
"proto_full_utils.cc",
"proto_full_utils.h",
- "trace_to_text.cc",
]
}
@@ -163,3 +179,15 @@
descriptor_name = "trace.descriptor"
descriptor_target = "../../protos/perfetto/trace:descriptor"
}
+
+perfetto_unittest_source_set("unittests") {
+ testonly = true
+ deps = [
+ ":trace_to_text_lib",
+ "../../gn:default_deps",
+ "../../gn:gtest_and_gmock",
+ "../../include/perfetto/base",
+ "../../include/perfetto/ext/base:base",
+ ]
+ sources = [ "trace_to_text_unittest.cc" ]
+}
diff --git a/tools/trace_to_text/lite_fallbacks.cc b/tools/trace_to_text/lite_fallbacks.cc
index 82e09ff..e208a12 100644
--- a/tools/trace_to_text/lite_fallbacks.cc
+++ b/tools/trace_to_text/lite_fallbacks.cc
@@ -24,7 +24,7 @@
namespace perfetto {
namespace trace_to_text {
-int TraceToText(std::istream*, std::ostream*) {
+bool TraceToText(std::istream*, std::ostream*) {
PERFETTO_FATAL(
"The 'text' command is not available in lite builds of trace_to_text");
}
diff --git a/tools/trace_to_text/main.cc b/tools/trace_to_text/main.cc
index e4a9f1d..dfd75b6 100644
--- a/tools/trace_to_text/main.cc
+++ b/tools/trace_to_text/main.cc
@@ -187,8 +187,9 @@
return 1;
}
- if (format == "text")
- return TraceToText(input_stream, output_stream);
+ if (format == "text") {
+ return TraceToText(input_stream, output_stream) ? 0 : 1;
+ }
if (format == "profile") {
return perf_profile
diff --git a/tools/trace_to_text/trace_to_text.cc b/tools/trace_to_text/trace_to_text.cc
index 7507c75..2ddf713 100644
--- a/tools/trace_to_text/trace_to_text.cc
+++ b/tools/trace_to_text/trace_to_text.cc
@@ -31,9 +31,8 @@
#include "protos/perfetto/trace/trace.pbzero.h"
#include "protos/perfetto/trace/trace_packet.pbzero.h"
-#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
-#include <zlib.h>
-#endif
+#include "src/trace_processor/forwarding_trace_parser.h"
+#include "src/trace_processor/util/gzip_utils.h"
namespace perfetto {
namespace trace_to_text {
@@ -50,6 +49,8 @@
using google::protobuf::TextFormat;
using google::protobuf::io::OstreamOutputStream;
using google::protobuf::io::ZeroCopyOutputStream;
+using trace_processor::TraceType;
+using trace_processor::util::GzipDecompressor;
inline void WriteToZeroCopyOutput(ZeroCopyOutputStream* output,
const char* str,
@@ -83,34 +84,9 @@
Message* compressed_msg_scratch,
ZeroCopyOutputStream* output) {
#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
- uint8_t out[4096];
- std::vector<uint8_t> data;
-
- z_stream stream{};
- stream.next_in =
- const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(packets.data()));
- stream.avail_in = static_cast<unsigned int>(packets.length());
-
- if (inflateInit(&stream) != Z_OK) {
- PERFETTO_ELOG("Error when initiliazing zlib to decompress packets");
- return;
- }
-
- int ret;
- do {
- stream.next_out = out;
- stream.avail_out = sizeof(out);
- ret = inflate(&stream, Z_NO_FLUSH);
- if (ret != Z_STREAM_END && ret != Z_OK) {
- PERFETTO_ELOG("Error when decompressing packets: %s",
- (stream.msg ? stream.msg : ""));
- return;
- }
- data.insert(data.end(), out, out + (sizeof(out) - stream.avail_out));
- } while (ret != Z_STREAM_END);
- inflateEnd(&stream);
-
- protos::pbzero::Trace::Decoder decoder(data.data(), data.size());
+ std::vector<uint8_t> whole_data = GzipDecompressor::DecompressFully(
+ reinterpret_cast<const uint8_t*>(packets.data()), packets.size());
+ protos::pbzero::Trace::Decoder decoder(whole_data.data(), whole_data.size());
WriteToZeroCopyOutput(output, kCompressedPacketsPrefix,
sizeof(kCompressedPacketsPrefix) - 1);
TextFormat::Printer printer;
@@ -146,90 +122,171 @@
#endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
}
-} // namespace
-
-int TraceToText(std::istream* input, std::ostream* output) {
- DescriptorPool pool;
+// TracePacket descriptor and metadata, used to print a TracePacket proto as a
+// text proto.
+struct TracePacketProtoDescInfo {
+ TracePacketProtoDescInfo();
FileDescriptorSet desc_set;
+ DescriptorPool pool;
+ std::unique_ptr<DynamicMessageFactory> factory;
+ const Descriptor* trace_descriptor;
+ const Message* prototype;
+ const FieldDescriptor* compressed_desc;
+};
+
+TracePacketProtoDescInfo::TracePacketProtoDescInfo() {
desc_set.ParseFromArray(kTraceDescriptor.data(), kTraceDescriptor.size());
for (const auto& desc : desc_set.file()) {
pool.BuildFile(desc);
}
-
- DynamicMessageFactory factory(&pool);
- const Descriptor* trace_descriptor =
- pool.FindMessageTypeByName("perfetto.protos.TracePacket");
- const Message* prototype = factory.GetPrototype(trace_descriptor);
- std::unique_ptr<Message> msg(prototype->New());
-
- OstreamOutputStream zero_copy_output(output);
- OstreamOutputStream* zero_copy_output_ptr = &zero_copy_output;
-
- const Reflection* reflect = msg->GetReflection();
- const FieldDescriptor* compressed_desc = trace_descriptor->FindFieldByNumber(
+ factory.reset(new DynamicMessageFactory(&pool));
+ trace_descriptor = pool.FindMessageTypeByName("perfetto.protos.TracePacket");
+ prototype = factory->GetPrototype(trace_descriptor);
+ compressed_desc = trace_descriptor->FindFieldByNumber(
protos::pbzero::TracePacket::kCompressedPacketsFieldNumber);
+}
- std::unique_ptr<Message> compressed_packets_msg(prototype->New());
- std::string compressed_packets;
+// Online algorithm to covert trace binary to text format.
+// Usage:
+// - Feed the trace-binary in a sequence of memblock, and it will continue to
+// write the output in given std::ostream*.
+class OnlineTraceToText {
+ public:
+ OnlineTraceToText(std::ostream* output)
+ : zero_copy_out_stream_(output),
+ msg_(pb_desc_info_.prototype->New()),
+ compressed_packets_msg_(pb_desc_info_.prototype->New()),
+ reflect_(msg_->GetReflection()) {
+ printer_.SetInitialIndentLevel(1);
+ }
+ OnlineTraceToText(const OnlineTraceToText&) = delete;
+ OnlineTraceToText& operator=(const OnlineTraceToText&) = delete;
+ void Feed(const uint8_t* data, size_t len);
+ bool ok() const { return ok_; }
- TextFormat::Printer printer;
- printer.SetInitialIndentLevel(1);
+ private:
+ bool ok_ = true;
+ OstreamOutputStream zero_copy_out_stream_;
+ protozero::ProtoRingBuffer ring_buffer_;
+ TextFormat::Printer printer_;
+ TracePacketProtoDescInfo pb_desc_info_;
+ std::unique_ptr<Message> msg_;
+ std::unique_ptr<Message> compressed_packets_msg_;
+ const Reflection* reflect_;
+ std::string compressed_packets_;
+ size_t bytes_processed_ = 0;
+ size_t packet_ = 0;
+};
- static constexpr size_t kMaxMsgSize = protozero::ProtoRingBuffer::kMaxMsgSize;
- std::unique_ptr<char[]> data(new char[kMaxMsgSize]);
- protozero::ProtoRingBuffer ring_buffer;
-
- uint32_t packet = 0;
- size_t bytes_processed = 0;
- while (!input->eof()) {
- input->read(data.get(), kMaxMsgSize);
- if (input->bad() || (input->fail() && !input->eof())) {
- PERFETTO_ELOG("Failed while reading trace");
- return 1;
+void OnlineTraceToText::Feed(const uint8_t* data, size_t len) {
+ ring_buffer_.Append(data, static_cast<size_t>(len));
+ while (true) {
+ auto token = ring_buffer_.ReadMessage();
+ if (token.fatal_framing_error) {
+ PERFETTO_ELOG("Failed to tokenize trace packet");
+ ok_ = false;
+ return;
}
- ring_buffer.Append(data.get(), static_cast<size_t>(input->gcount()));
+ if (!token.valid()) {
+ // no need to set `ok_ = false` here because this just means
+ // we've run out of packets in the ring buffer.
+ break;
+ }
- for (;;) {
- auto token = ring_buffer.ReadMessage();
- if (token.fatal_framing_error) {
- PERFETTO_ELOG("Failed to tokenize trace packet");
- return 1;
- }
- if (!token.valid())
- break;
- bytes_processed += token.len;
-
- if (token.field_id != protos::pbzero::Trace::kPacketFieldNumber) {
- PERFETTO_ELOG("Skipping invalid field");
- continue;
- }
-
- if ((packet++ & 0x3f) == 0) {
- fprintf(stderr, "Processing trace: %8zu KB%c", bytes_processed / 1024,
- kProgressChar);
- fflush(stderr);
- }
-
- if (!msg->ParseFromArray(token.start, static_cast<int>(token.len))) {
- PERFETTO_ELOG("Skipping invalid packet");
- continue;
- }
-
- if (reflect->HasField(*msg, compressed_desc)) {
- compressed_packets = reflect->GetStringReference(*msg, compressed_desc,
- &compressed_packets);
- PrintCompressedPackets(compressed_packets, compressed_packets_msg.get(),
- zero_copy_output_ptr);
- } else {
- WriteToZeroCopyOutput(zero_copy_output_ptr, kPacketPrefix,
- sizeof(kPacketPrefix) - 1);
- printer.Print(*msg, zero_copy_output_ptr);
- WriteToZeroCopyOutput(zero_copy_output_ptr, kPacketSuffix,
- sizeof(kPacketSuffix) - 1);
- }
+ if (token.field_id != protos::pbzero::Trace::kPacketFieldNumber) {
+ PERFETTO_ELOG("Skipping invalid field");
+ continue;
+ }
+ if (!msg_->ParseFromArray(token.start, static_cast<int>(token.len))) {
+ PERFETTO_ELOG("Skipping invalid packet");
+ continue;
+ }
+ bytes_processed_ += token.len;
+ if ((packet_++ & 0x3f) == 0) {
+ fprintf(stderr, "Processing trace: %8zu KB%c", bytes_processed_ / 1024,
+ kProgressChar);
+ fflush(stderr);
+ }
+ if (reflect_->HasField(*msg_, pb_desc_info_.compressed_desc)) {
+ // TODO(mohitms): GetStringReference ignores third argument. Why are we
+ // passing ?
+ compressed_packets_ = reflect_->GetStringReference(
+ *msg_, pb_desc_info_.compressed_desc, &compressed_packets_);
+ PrintCompressedPackets(compressed_packets_, compressed_packets_msg_.get(),
+ &zero_copy_out_stream_);
+ } else {
+ WriteToZeroCopyOutput(&zero_copy_out_stream_, kPacketPrefix,
+ sizeof(kPacketPrefix) - 1);
+ printer_.Print(*msg_, &zero_copy_out_stream_);
+ WriteToZeroCopyOutput(&zero_copy_out_stream_, kPacketSuffix,
+ sizeof(kPacketSuffix) - 1);
}
}
- return 0;
+}
+
+class InputReader {
+ public:
+ InputReader(std::istream* input) : input_(input) {}
+ // Request the input-stream to read next |len_limit| bytes and load
+ // it in |data|. It also updates the |len| with actual number of bytes loaded
+ // in |data|. This can be less than requested |len_limit| if we have reached
+ // at the end of the file.
+ bool Read(uint8_t* data, uint32_t* len, uint32_t len_limit) {
+ if (input_->eof())
+ return false;
+ input_->read(reinterpret_cast<char*>(data), std::streamsize(len_limit));
+ if (input_->bad() || (input_->fail() && !input_->eof())) {
+ PERFETTO_ELOG("Failed while reading trace");
+ ok_ = false;
+ return false;
+ }
+ *len = uint32_t(input_->gcount());
+ return true;
+ }
+ bool ok() const { return ok_; }
+
+ private:
+ std::istream* input_;
+ bool ok_ = true;
+};
+
+} // namespace
+
+bool TraceToText(std::istream* input, std::ostream* output) {
+ constexpr size_t kMaxMsgSize = protozero::ProtoRingBuffer::kMaxMsgSize;
+ std::unique_ptr<uint8_t[]> buffer(new uint8_t[kMaxMsgSize]);
+ uint32_t buffer_len = 0;
+
+ InputReader input_reader(input);
+ OnlineTraceToText online_trace_to_text(output);
+
+ input_reader.Read(buffer.get(), &buffer_len, kMaxMsgSize);
+ TraceType type = trace_processor::GuessTraceType(buffer.get(), buffer_len);
+
+ if (type == TraceType::kGzipTraceType) {
+ GzipDecompressor decompressor;
+ auto consumer = [&](const uint8_t* data, size_t len) {
+ online_trace_to_text.Feed(data, len);
+ };
+ using ResultCode = GzipDecompressor::ResultCode;
+ do {
+ ResultCode code =
+ decompressor.FeedAndExtract(buffer.get(), buffer_len, consumer);
+ if (code == ResultCode::kError || !online_trace_to_text.ok())
+ return false;
+ } while (input_reader.Read(buffer.get(), &buffer_len, kMaxMsgSize));
+ return input_reader.ok();
+ } else if (type == TraceType::kProtoTraceType) {
+ do {
+ online_trace_to_text.Feed(buffer.get(), buffer_len);
+ if (!online_trace_to_text.ok())
+ return false;
+ } while (input_reader.Read(buffer.get(), &buffer_len, kMaxMsgSize));
+ return input_reader.ok();
+ } else {
+ PERFETTO_ELOG("Unrecognised file.");
+ return false;
+ }
}
} // namespace trace_to_text
diff --git a/tools/trace_to_text/trace_to_text.h b/tools/trace_to_text/trace_to_text.h
index 2be24ad..d0f35e0 100644
--- a/tools/trace_to_text/trace_to_text.h
+++ b/tools/trace_to_text/trace_to_text.h
@@ -22,7 +22,8 @@
namespace perfetto {
namespace trace_to_text {
-int TraceToText(std::istream* input, std::ostream* output);
+// Returns true in case of success.
+bool TraceToText(std::istream* input, std::ostream* output);
} // namespace trace_to_text
} // namespace perfetto
diff --git a/tools/trace_to_text/trace_to_text_unittest.cc b/tools/trace_to_text/trace_to_text_unittest.cc
new file mode 100644
index 0000000..0066c0c
--- /dev/null
+++ b/tools/trace_to_text/trace_to_text_unittest.cc
@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "tools/trace_to_text/trace_to_text.h"
+
+#include "perfetto/base/logging.h"
+#include "perfetto/ext/base/hash.h"
+#include "test/gtest_and_gmock.h"
+
+#include <fstream>
+
+using std::string;
+
+namespace perfetto {
+namespace trace_to_text {
+
+// Given a file, compute the checksum/hash of file.
+// Learn more @ base::Hash.
+// Precondition: File should exist and be accessible.
+static uint64_t FileHash(const string& filename) {
+ base::Hash hash;
+ std::ifstream input_f(filename, std::ios::binary);
+ PERFETTO_DCHECK(input_f.good());
+ char buffer[4096];
+ while (!input_f.eof()) {
+ input_f.read(buffer, sizeof(buffer));
+ if (input_f.gcount() > 0) {
+ hash.Update(buffer, size_t(input_f.gcount()));
+ }
+ }
+ return hash.digest();
+}
+
+TEST(TraceToText, DISABLED_Basic) {
+ auto tmp_file = "/tmp/trace_" + std::to_string(rand()) + ".txt";
+ auto input_file_names = {"test/data/example_android_trace_30s.pb.gz",
+ "test/data/example_android_trace_30s.pb"};
+ PERFETTO_LOG("tmp_file = %s.", tmp_file.c_str());
+ for (auto filename : input_file_names) {
+ {
+ std::ifstream input_f(filename, std::ios::binary);
+ std::ofstream output_f(tmp_file, std::ios::out | std::ios::binary);
+ EXPECT_TRUE(TraceToText(&input_f, &output_f));
+ PERFETTO_LOG("Processed %s", filename);
+ }
+ EXPECT_EQ(0xCD794377594BC7DCull, FileHash(tmp_file));
+ remove(tmp_file.c_str());
+ }
+}
+
+} // namespace trace_to_text
+} // namespace perfetto
