zstd
diff --git a/Android.bp b/Android.bp index 8aaec4d..78f658f 100644 --- a/Android.bp +++ b/Android.bp
@@ -692,8 +692,10 @@ ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ], host_supported: true, export_include_dirs: [ @@ -781,6 +783,7 @@ "liblog", "libpcre2", "libz", + "libzstd", ], static_libs: [ "perfetto_flags_c_lib", @@ -790,6 +793,7 @@ static_libs: [ "liblog", "libz", + "libzstd", ], whole_static_libs: [ "libpcre2", @@ -962,8 +966,10 @@ ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_src_tracing_system_backend", ], host_supported: true, @@ -1043,6 +1049,7 @@ ], cflags: [ "-DPERFETTO_FORCE_DISABLE_ZLIB", + "-DPERFETTO_FORCE_DISABLE_ZSTD", "-DPERFETTO_REGEX_FORCE_STD", "-DPERFETTO_SHLIB_SDK_IMPLEMENTATION", "-DZLIB_IMPLEMENTATION", @@ -1187,8 +1194,10 @@ ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_src_tracing_system_backend", ], shared_libs: [ @@ -1340,6 +1349,7 @@ ], cflags: [ "-DPERFETTO_FORCE_DISABLE_ZLIB", + "-DPERFETTO_FORCE_DISABLE_ZSTD", "-DPERFETTO_REGEX_FORCE_STD", "-DZLIB_IMPLEMENTATION", ], @@ -1981,7 +1991,10 @@ ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", + ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_test_test_helper", "test/cts/art_module/heapprofd_java_test_cts.cc", "test/cts/device_feature_test_cts.cc", @@ -1993,6 +2006,8 @@ shared_libs: [ "liblog", "libpcre2", + "libz", + "libzstd", ], static_libs: [ "libgmock", @@ -2182,6 +2197,9 @@ defaults: [ "perfetto_defaults", ], + cflags: [ + "-DZLIB_IMPLEMENTATION", + ], target: { android: { static_libs: [ @@ -2353,11 +2371,16 @@ ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", + ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_test_test_helper", ], shared_libs: [ "libpcre2", + "libz", + "libzstd", ], static_libs: [ "libgmock", @@ -2544,6 +2567,9 @@ defaults: [ "perfetto_defaults", ], + cflags: [ + "-DZLIB_IMPLEMENTATION", + ], } // GN: //gn:default_deps @@ -3100,6 +3126,7 @@ ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_decompress", ":perfetto_src_trace_processor_util_descriptors", ":perfetto_src_trace_processor_util_elf_elf", ":perfetto_src_trace_processor_util_galloping_search", @@ -3123,9 +3150,11 @@ ":perfetto_src_trace_processor_util_sql_bundle", ":perfetto_src_trace_processor_util_sql_module_doc_parser", ":perfetto_src_trace_processor_util_stdlib", + ":perfetto_src_trace_processor_util_stream_decompressor", ":perfetto_src_trace_processor_util_trace_blob_view_reader", ":perfetto_src_trace_processor_util_trace_type", ":perfetto_src_trace_processor_util_zip_reader", + ":perfetto_src_trace_processor_util_zstd", ":perfetto_src_traced_probes_android_aflags_android_aflags", ":perfetto_src_traced_probes_android_cpu_per_uid_android_cpu_per_uid", ":perfetto_src_traced_probes_android_game_intervention_list_android_game_intervention_list", @@ -3166,8 +3195,10 @@ ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_producer_relay", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_src_tracing_system_backend", ":perfetto_src_tracing_test_api_test_support", ":perfetto_src_tracing_test_client_api_integrationtests", @@ -3188,6 +3219,7 @@ "libunwindstack", "libutils", "libz", + "libzstd", ], static_libs: [ "libgmock", @@ -17296,7 +17328,7 @@ name: "perfetto_src_trace_processor_importers_archive_archive", srcs: [ "src/trace_processor/importers/archive/archive_entry.cc", - "src/trace_processor/importers/archive/gzip_trace_parser.cc", + "src/trace_processor/importers/archive/decompressing_trace_reader.cc", "src/trace_processor/importers/archive/tar_trace_reader.cc", "src/trace_processor/importers/archive/zip_trace_reader.cc", ], @@ -20606,6 +20638,7 @@ ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_decompress", ":perfetto_src_trace_processor_util_deobfuscation_deobfuscator", ":perfetto_src_trace_processor_util_descriptors", ":perfetto_src_trace_processor_util_elf_elf", @@ -20630,6 +20663,7 @@ ":perfetto_src_trace_processor_util_sql_bundle", ":perfetto_src_trace_processor_util_sql_module_doc_parser", ":perfetto_src_trace_processor_util_stdlib", + ":perfetto_src_trace_processor_util_stream_decompressor", ":perfetto_src_trace_processor_util_symbolizer_symbolize_database", ":perfetto_src_trace_processor_util_symbolizer_symbolizer", ":perfetto_src_trace_processor_util_tar_writer", @@ -20637,6 +20671,7 @@ ":perfetto_src_trace_processor_util_trace_enrichment_trace_enrichment", ":perfetto_src_trace_processor_util_trace_type", ":perfetto_src_trace_processor_util_zip_reader", + ":perfetto_src_trace_processor_util_zstd", ":perfetto_src_traceconv_lib", ":perfetto_src_traceconv_pprofbuilder", ":perfetto_src_traceconv_utils", @@ -20872,6 +20907,7 @@ "libsqlite", "libutils", "libz", + "libzstd", ], static_libs: [ "perfetto_flags_c_lib", @@ -20883,6 +20919,7 @@ "libprotobuf-cpp-full", "libsqlite_static_noicu", "libz", + "libzstd", "sqlite_ext_percentile", ], whole_static_libs: [ @@ -20992,6 +21029,14 @@ ], } +// GN: //src/trace_processor/util:decompress +filegroup { + name: "perfetto_src_trace_processor_util_decompress", + srcs: [ + "src/trace_processor/util/decompress.cc", + ], +} + // GN: //src/trace_processor/util/deobfuscation:deobfuscator filegroup { name: "perfetto_src_trace_processor_util_deobfuscation_deobfuscator", @@ -21041,7 +21086,7 @@ filegroup { name: "perfetto_src_trace_processor_util_gzip", srcs: [ - "src/trace_processor/util/gzip_utils.cc", + "src/trace_processor/util/gzip_decompressor.cc", ], } @@ -21172,6 +21217,14 @@ name: "perfetto_src_trace_processor_util_stdlib", } +// GN: //src/trace_processor/util:stream_decompressor +filegroup { + name: "perfetto_src_trace_processor_util_stream_decompressor", + srcs: [ + "src/trace_processor/util/stream_decompressor.cc", + ], +} + // GN: //src/trace_processor/util/symbolizer:symbolize_database filegroup { name: "perfetto_src_trace_processor_util_symbolizer_symbolize_database", @@ -21245,7 +21298,7 @@ "src/trace_processor/util/descriptors_unittest.cc", "src/trace_processor/util/galloping_search_unittest.cc", "src/trace_processor/util/glob_unittest.cc", - "src/trace_processor/util/gzip_utils_unittest.cc", + "src/trace_processor/util/gzip_decompressor_unittest.cc", "src/trace_processor/util/json_parser_unittest.cc", "src/trace_processor/util/json_serializer_unittest.cc", "src/trace_processor/util/json_value_unittest.cc", @@ -21261,6 +21314,7 @@ "src/trace_processor/util/tar_writer_unittest.cc", "src/trace_processor/util/trace_blob_view_reader_unittest.cc", "src/trace_processor/util/zip_reader_unittest.cc", + "src/trace_processor/util/zstd_decompressor_unittest.cc", ], } @@ -21273,6 +21327,14 @@ ], } +// GN: //src/trace_processor/util:zstd +filegroup { + name: "perfetto_src_trace_processor_util_zstd", + srcs: [ + "src/trace_processor/util/zstd_decompressor.cc", + ], +} + // GN: //src/trace_redaction:trace_redaction filegroup { name: "perfetto_src_trace_redaction_trace_redaction", @@ -22140,6 +22202,11 @@ ], } +// GN: //src/tracing/service:packet_compressor_common +filegroup { + name: "perfetto_src_tracing_service_packet_compressor_common", +} + // GN: //src/tracing/service:service filegroup { name: "perfetto_src_tracing_service_service", @@ -22161,12 +22228,12 @@ filegroup { name: "perfetto_src_tracing_service_unittests", srcs: [ + "src/tracing/service/compressor_unittest.cc", "src/tracing/service/histogram_unittest.cc", "src/tracing/service/packet_stream_validator_unittest.cc", "src/tracing/service/trace_buffer_v1_unittest.cc", "src/tracing/service/trace_buffer_v2_unittest.cc", "src/tracing/service/tracing_service_impl_unittest.cc", - "src/tracing/service/zlib_compressor_unittest.cc", ], } @@ -22178,6 +22245,14 @@ ], } +// GN: //src/tracing/service:zstd_compressor +filegroup { + name: "perfetto_src_tracing_service_zstd_compressor", + srcs: [ + "src/tracing/service/zstd_compressor.cc", + ], +} + // GN: //src/tracing:system_backend filegroup { name: "perfetto_src_tracing_system_backend", @@ -23231,6 +23306,7 @@ ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_decompress", ":perfetto_src_trace_processor_util_deobfuscation_deobfuscator", ":perfetto_src_trace_processor_util_deobfuscation_unittests", ":perfetto_src_trace_processor_util_descriptors", @@ -23256,6 +23332,7 @@ ":perfetto_src_trace_processor_util_sql_bundle", ":perfetto_src_trace_processor_util_sql_module_doc_parser", ":perfetto_src_trace_processor_util_stdlib", + ":perfetto_src_trace_processor_util_stream_decompressor", ":perfetto_src_trace_processor_util_symbolizer_symbolize_database", ":perfetto_src_trace_processor_util_symbolizer_symbolizer", ":perfetto_src_trace_processor_util_symbolizer_unittests", @@ -23265,6 +23342,7 @@ ":perfetto_src_trace_processor_util_trace_type", ":perfetto_src_trace_processor_util_unittests", ":perfetto_src_trace_processor_util_zip_reader", + ":perfetto_src_trace_processor_util_zstd", ":perfetto_src_trace_redaction_trace_redaction", ":perfetto_src_trace_redaction_unittests", ":perfetto_src_traceconv_lib", @@ -23334,9 +23412,11 @@ ":perfetto_src_tracing_ipc_producer_relay", ":perfetto_src_tracing_ipc_service_service", ":perfetto_src_tracing_ipc_unittests", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", ":perfetto_src_tracing_service_unittests", ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_src_tracing_system_backend", ":perfetto_src_tracing_test_test_support", ":perfetto_src_tracing_unittests", @@ -23354,6 +23434,7 @@ "libunwindstack", "libutils", "libz", + "libzstd", ], static_libs: [ "libgmock", @@ -23778,11 +23859,16 @@ ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", + ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_test_test_helper", ], shared_libs: [ "libpcre2", + "libz", + "libzstd", ], static_libs: [ "libgmock", @@ -23969,6 +24055,9 @@ defaults: [ "perfetto_defaults", ], + cflags: [ + "-DZLIB_IMPLEMENTATION", + ], } // GN: //src/protozero/protoc_plugin:protozero_c_plugin @@ -24242,6 +24331,7 @@ ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_decompress", ":perfetto_src_trace_processor_util_descriptors", ":perfetto_src_trace_processor_util_elf_elf", ":perfetto_src_trace_processor_util_galloping_search", @@ -24265,9 +24355,11 @@ ":perfetto_src_trace_processor_util_sql_bundle", ":perfetto_src_trace_processor_util_sql_module_doc_parser", ":perfetto_src_trace_processor_util_stdlib", + ":perfetto_src_trace_processor_util_stream_decompressor", ":perfetto_src_trace_processor_util_trace_blob_view_reader", ":perfetto_src_trace_processor_util_trace_type", ":perfetto_src_trace_processor_util_zip_reader", + ":perfetto_src_trace_processor_util_zstd", ], static_libs: [ "perfetto_src_trace_processor_demangle", @@ -24494,6 +24586,7 @@ "libsqlite", "libutils", "libz", + "libzstd", ], static_libs: [ "perfetto_flags_c_lib", @@ -24504,6 +24597,7 @@ static_libs: [ "libsqlite_static_noicu", "libz", + "libzstd", "sqlite_ext_percentile", ], whole_static_libs: [ @@ -24540,6 +24634,7 @@ "libsqlite", "libutils", "libz", + "libzstd", ], static_libs: [ "perfetto_flags_c_lib", @@ -24552,6 +24647,7 @@ "libprotobuf-cpp-full", "libsqlite_static_noicu", "libz", + "libzstd", "sqlite_ext_percentile", ], stl: "libc++_static", @@ -24917,6 +25013,7 @@ ":perfetto_src_trace_processor_util_build_id", ":perfetto_src_trace_processor_util_bump_allocator", ":perfetto_src_trace_processor_util_clock", + ":perfetto_src_trace_processor_util_decompress", ":perfetto_src_trace_processor_util_deobfuscation_deobfuscator", ":perfetto_src_trace_processor_util_descriptors", ":perfetto_src_trace_processor_util_elf_elf", @@ -24941,6 +25038,7 @@ ":perfetto_src_trace_processor_util_sql_bundle", ":perfetto_src_trace_processor_util_sql_module_doc_parser", ":perfetto_src_trace_processor_util_stdlib", + ":perfetto_src_trace_processor_util_stream_decompressor", ":perfetto_src_trace_processor_util_symbolizer_symbolize_database", ":perfetto_src_trace_processor_util_symbolizer_symbolizer", ":perfetto_src_trace_processor_util_tar_writer", @@ -24948,6 +25046,7 @@ ":perfetto_src_trace_processor_util_trace_enrichment_trace_enrichment", ":perfetto_src_trace_processor_util_trace_type", ":perfetto_src_trace_processor_util_zip_reader", + ":perfetto_src_trace_processor_util_zstd", ":perfetto_src_traceconv_lib", ":perfetto_src_traceconv_main", ":perfetto_src_traceconv_pprofbuilder", @@ -24958,6 +25057,7 @@ "libpcre2", "libsqlite_static_noicu", "libz", + "libzstd", "perfetto_src_trace_processor_demangle", "sqlite_ext_percentile", ], @@ -25242,7 +25342,10 @@ ":perfetto_src_tracing_ipc_common", ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", + ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", "src/profiling/perf/main.cc", ], shared_libs: [ @@ -25251,6 +25354,8 @@ "libpcre2", "libprocinfo", "libunwindstack", + "libz", + "libzstd", ], static_libs: [ "perfetto_flags_c_lib", @@ -25328,6 +25433,9 @@ defaults: [ "perfetto_defaults", ], + cflags: [ + "-DZLIB_IMPLEMENTATION", + ], include_dirs: [ "bionic/libc/kernel", ],
diff --git a/BUILD b/BUILD index a661a4c..ab8ccdb 100644 --- a/BUILD +++ b/BUILD
@@ -182,8 +182,10 @@ ":src_tracing_ipc_default_socket", ":src_tracing_ipc_producer_producer", ":src_tracing_ipc_service_service", + ":src_tracing_service_packet_compressor_common", ":src_tracing_service_service", ":src_tracing_service_zlib_compressor", + ":src_tracing_service_zstd_compressor", ":src_tracing_system_backend", ], hdrs = [ @@ -208,77 +210,78 @@ ], visibility = PERFETTO_CONFIG.public_visibility, deps = [ - ":perfetto_ipc", - ":protos_perfetto_common_cpp", - ":protos_perfetto_common_semantic_type_cpp", - ":protos_perfetto_common_semantic_type_zero", - ":protos_perfetto_common_zero", - ":protos_perfetto_config_android_cpp", - ":protos_perfetto_config_android_zero", - ":protos_perfetto_config_cpp", - ":protos_perfetto_config_ftrace_cpp", - ":protos_perfetto_config_ftrace_zero", - ":protos_perfetto_config_gpu_cpp", - ":protos_perfetto_config_gpu_zero", - ":protos_perfetto_config_inode_file_cpp", - ":protos_perfetto_config_inode_file_zero", - ":protos_perfetto_config_interceptors_cpp", - ":protos_perfetto_config_interceptors_zero", - ":protos_perfetto_config_linux_cpp", - ":protos_perfetto_config_linux_zero", - ":protos_perfetto_config_power_cpp", - ":protos_perfetto_config_power_zero", - ":protos_perfetto_config_priority_boost_cpp", - ":protos_perfetto_config_priority_boost_zero", - ":protos_perfetto_config_process_stats_cpp", - ":protos_perfetto_config_process_stats_zero", - ":protos_perfetto_config_profiling_cpp", - ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_protovm_cpp", - ":protos_perfetto_config_protovm_zero", - ":protos_perfetto_config_qnx_cpp", - ":protos_perfetto_config_qnx_zero", - ":protos_perfetto_config_statsd_cpp", - ":protos_perfetto_config_statsd_zero", - ":protos_perfetto_config_sys_stats_cpp", - ":protos_perfetto_config_sys_stats_zero", - ":protos_perfetto_config_system_info_cpp", - ":protos_perfetto_config_system_info_zero", - ":protos_perfetto_config_track_event_cpp", - ":protos_perfetto_config_track_event_zero", - ":protos_perfetto_config_zero", - ":protos_perfetto_ipc_cpp", - ":protos_perfetto_ipc_ipc", - ":protos_perfetto_protovm_cpp", - ":protos_perfetto_protovm_zero", - ":protos_perfetto_trace_android_zero", - ":protos_perfetto_trace_chrome_zero", - ":protos_perfetto_trace_etw_zero", - ":protos_perfetto_trace_filesystem_zero", - ":protos_perfetto_trace_ftrace_zero", - ":protos_perfetto_trace_generic_kernel_zero", - ":protos_perfetto_trace_gpu_zero", - ":protos_perfetto_trace_interned_data_zero", - ":protos_perfetto_trace_linux_zero", - ":protos_perfetto_trace_minimal_zero", - ":protos_perfetto_trace_non_minimal_zero", - ":protos_perfetto_trace_perfetto_zero", - ":protos_perfetto_trace_power_zero", - ":protos_perfetto_trace_profiling_zero", - ":protos_perfetto_trace_ps_zero", - ":protos_perfetto_trace_statsd_zero", - ":protos_perfetto_trace_sys_stats_zero", - ":protos_perfetto_trace_system_info_zero", - ":protos_perfetto_trace_track_event_cpp", - ":protos_perfetto_trace_track_event_zero", - ":protos_perfetto_trace_translation_zero", - ":protozero", - ":src_base_base", - ":src_base_clock_snapshots", - ":src_base_regex_regex", - ":src_base_version", - ":src_protovm_protovm", - ] + PERFETTO_CONFIG.deps.zlib, + ":perfetto_ipc", + ":protos_perfetto_common_cpp", + ":protos_perfetto_common_semantic_type_cpp", + ":protos_perfetto_common_semantic_type_zero", + ":protos_perfetto_common_zero", + ":protos_perfetto_config_android_cpp", + ":protos_perfetto_config_android_zero", + ":protos_perfetto_config_cpp", + ":protos_perfetto_config_ftrace_cpp", + ":protos_perfetto_config_ftrace_zero", + ":protos_perfetto_config_gpu_cpp", + ":protos_perfetto_config_gpu_zero", + ":protos_perfetto_config_inode_file_cpp", + ":protos_perfetto_config_inode_file_zero", + ":protos_perfetto_config_interceptors_cpp", + ":protos_perfetto_config_interceptors_zero", + ":protos_perfetto_config_linux_cpp", + ":protos_perfetto_config_linux_zero", + ":protos_perfetto_config_power_cpp", + ":protos_perfetto_config_power_zero", + ":protos_perfetto_config_priority_boost_cpp", + ":protos_perfetto_config_priority_boost_zero", + ":protos_perfetto_config_process_stats_cpp", + ":protos_perfetto_config_process_stats_zero", + ":protos_perfetto_config_profiling_cpp", + ":protos_perfetto_config_profiling_zero", + ":protos_perfetto_config_protovm_cpp", + ":protos_perfetto_config_protovm_zero", + ":protos_perfetto_config_qnx_cpp", + ":protos_perfetto_config_qnx_zero", + ":protos_perfetto_config_statsd_cpp", + ":protos_perfetto_config_statsd_zero", + ":protos_perfetto_config_sys_stats_cpp", + ":protos_perfetto_config_sys_stats_zero", + ":protos_perfetto_config_system_info_cpp", + ":protos_perfetto_config_system_info_zero", + ":protos_perfetto_config_track_event_cpp", + ":protos_perfetto_config_track_event_zero", + ":protos_perfetto_config_zero", + ":protos_perfetto_ipc_cpp", + ":protos_perfetto_ipc_ipc", + ":protos_perfetto_protovm_cpp", + ":protos_perfetto_protovm_zero", + ":protos_perfetto_trace_android_zero", + ":protos_perfetto_trace_chrome_zero", + ":protos_perfetto_trace_etw_zero", + ":protos_perfetto_trace_filesystem_zero", + ":protos_perfetto_trace_ftrace_zero", + ":protos_perfetto_trace_generic_kernel_zero", + ":protos_perfetto_trace_gpu_zero", + ":protos_perfetto_trace_interned_data_zero", + ":protos_perfetto_trace_linux_zero", + ":protos_perfetto_trace_minimal_zero", + ":protos_perfetto_trace_non_minimal_zero", + ":protos_perfetto_trace_perfetto_zero", + ":protos_perfetto_trace_power_zero", + ":protos_perfetto_trace_profiling_zero", + ":protos_perfetto_trace_ps_zero", + ":protos_perfetto_trace_statsd_zero", + ":protos_perfetto_trace_sys_stats_zero", + ":protos_perfetto_trace_system_info_zero", + ":protos_perfetto_trace_track_event_cpp", + ":protos_perfetto_trace_track_event_zero", + ":protos_perfetto_trace_translation_zero", + ":protozero", + ":src_base_base", + ":src_base_clock_snapshots", + ":src_base_regex_regex", + ":src_base_version", + ":src_protovm_protovm", + ] + PERFETTO_CONFIG.deps.zlib + + PERFETTO_CONFIG.deps.zstd, linkstatic = True, ) @@ -541,6 +544,7 @@ ":src_trace_processor_util_blob", ":src_trace_processor_util_bump_allocator", ":src_trace_processor_util_clock", + ":src_trace_processor_util_decompress", ":src_trace_processor_util_descriptors", ":src_trace_processor_util_elf_elf", ":src_trace_processor_util_galloping_search", @@ -564,9 +568,11 @@ ":src_trace_processor_util_sql_bundle", ":src_trace_processor_util_sql_module_doc_parser", ":src_trace_processor_util_stdlib", + ":src_trace_processor_util_stream_decompressor", ":src_trace_processor_util_trace_blob_view_reader", ":src_trace_processor_util_trace_type", ":src_trace_processor_util_zip_reader", + ":src_trace_processor_util_zstd", ], hdrs = [ ":include_perfetto_base_base", @@ -679,6 +685,7 @@ ] + PERFETTO_CONFIG.deps.sqlite + PERFETTO_CONFIG.deps.sqlite_ext_percentile + PERFETTO_CONFIG.deps.zlib + + PERFETTO_CONFIG.deps.zstd + PERFETTO_CONFIG.deps.demangle_wrapper, linkstatic = True, ) @@ -849,6 +856,7 @@ ":src_trace_processor_util_blob", ":src_trace_processor_util_bump_allocator", ":src_trace_processor_util_clock", + ":src_trace_processor_util_decompress", ":src_trace_processor_util_descriptors", ":src_trace_processor_util_elf_elf", ":src_trace_processor_util_galloping_search", @@ -872,11 +880,13 @@ ":src_trace_processor_util_sql_bundle", ":src_trace_processor_util_sql_module_doc_parser", ":src_trace_processor_util_stdlib", + ":src_trace_processor_util_stream_decompressor", ":src_trace_processor_util_tar_writer", ":src_trace_processor_util_trace_blob_view_reader", ":src_trace_processor_util_trace_enrichment_trace_enrichment", ":src_trace_processor_util_trace_type", ":src_trace_processor_util_zip_reader", + ":src_trace_processor_util_zstd", ":src_traceconv_lib", ":src_traceconv_pprofbuilder", ":src_traceconv_utils", @@ -1004,6 +1014,7 @@ PERFETTO_CONFIG.deps.sqlite + PERFETTO_CONFIG.deps.sqlite_ext_percentile + PERFETTO_CONFIG.deps.zlib + + PERFETTO_CONFIG.deps.zstd + PERFETTO_CONFIG.deps.demangle_wrapper, linkstatic = True, ) @@ -1176,8 +1187,10 @@ ":src_tracing_ipc_common", ":src_tracing_ipc_default_socket", ":src_tracing_ipc_service_service", + ":src_tracing_service_packet_compressor_common", ":src_tracing_service_service", ":src_tracing_service_zlib_compressor", + ":src_tracing_service_zstd_compressor", ] + select({ "@platforms//os:windows": [], "//conditions:default": [ @@ -1233,79 +1246,80 @@ ":include_perfetto_tracing_tracing", ], deps = [ - ":perfetto_ipc", - ":protos_perfetto_common_cpp", - ":protos_perfetto_common_semantic_type_cpp", - ":protos_perfetto_common_semantic_type_zero", - ":protos_perfetto_common_zero", - ":protos_perfetto_config_android_cpp", - ":protos_perfetto_config_android_zero", - ":protos_perfetto_config_cpp", - ":protos_perfetto_config_ftrace_cpp", - ":protos_perfetto_config_ftrace_zero", - ":protos_perfetto_config_gpu_cpp", - ":protos_perfetto_config_gpu_zero", - ":protos_perfetto_config_inode_file_cpp", - ":protos_perfetto_config_inode_file_zero", - ":protos_perfetto_config_interceptors_cpp", - ":protos_perfetto_config_interceptors_zero", - ":protos_perfetto_config_linux_cpp", - ":protos_perfetto_config_linux_zero", - ":protos_perfetto_config_power_cpp", - ":protos_perfetto_config_power_zero", - ":protos_perfetto_config_priority_boost_cpp", - ":protos_perfetto_config_priority_boost_zero", - ":protos_perfetto_config_process_stats_cpp", - ":protos_perfetto_config_process_stats_zero", - ":protos_perfetto_config_profiling_cpp", - ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_protovm_cpp", - ":protos_perfetto_config_protovm_zero", - ":protos_perfetto_config_qnx_cpp", - ":protos_perfetto_config_qnx_zero", - ":protos_perfetto_config_statsd_cpp", - ":protos_perfetto_config_statsd_zero", - ":protos_perfetto_config_sys_stats_cpp", - ":protos_perfetto_config_sys_stats_zero", - ":protos_perfetto_config_system_info_cpp", - ":protos_perfetto_config_system_info_zero", - ":protos_perfetto_config_track_event_cpp", - ":protos_perfetto_config_track_event_zero", - ":protos_perfetto_config_zero", - ":protos_perfetto_ipc_cpp", - ":protos_perfetto_ipc_ipc", - ":protos_perfetto_protovm_cpp", - ":protos_perfetto_protovm_zero", - ":protos_perfetto_trace_android_zero", - ":protos_perfetto_trace_chrome_zero", - ":protos_perfetto_trace_etw_zero", - ":protos_perfetto_trace_filesystem_zero", - ":protos_perfetto_trace_ftrace_zero", - ":protos_perfetto_trace_generic_kernel_zero", - ":protos_perfetto_trace_gpu_zero", - ":protos_perfetto_trace_interned_data_zero", - ":protos_perfetto_trace_linux_zero", - ":protos_perfetto_trace_minimal_zero", - ":protos_perfetto_trace_non_minimal_zero", - ":protos_perfetto_trace_perfetto_zero", - ":protos_perfetto_trace_power_zero", - ":protos_perfetto_trace_profiling_zero", - ":protos_perfetto_trace_ps_zero", - ":protos_perfetto_trace_statsd_cpp", - ":protos_perfetto_trace_statsd_zero", - ":protos_perfetto_trace_sys_stats_zero", - ":protos_perfetto_trace_system_info_zero", - ":protos_perfetto_trace_track_event_cpp", - ":protos_perfetto_trace_track_event_zero", - ":protos_perfetto_trace_translation_zero", - ":protos_third_party_statsd_config_zero", - ":protozero", - ":src_base_base", - ":src_base_clock_snapshots", - ":src_base_regex_regex", - ":src_base_version", - ":src_protovm_protovm", - ] + PERFETTO_CONFIG.deps.zlib, + ":perfetto_ipc", + ":protos_perfetto_common_cpp", + ":protos_perfetto_common_semantic_type_cpp", + ":protos_perfetto_common_semantic_type_zero", + ":protos_perfetto_common_zero", + ":protos_perfetto_config_android_cpp", + ":protos_perfetto_config_android_zero", + ":protos_perfetto_config_cpp", + ":protos_perfetto_config_ftrace_cpp", + ":protos_perfetto_config_ftrace_zero", + ":protos_perfetto_config_gpu_cpp", + ":protos_perfetto_config_gpu_zero", + ":protos_perfetto_config_inode_file_cpp", + ":protos_perfetto_config_inode_file_zero", + ":protos_perfetto_config_interceptors_cpp", + ":protos_perfetto_config_interceptors_zero", + ":protos_perfetto_config_linux_cpp", + ":protos_perfetto_config_linux_zero", + ":protos_perfetto_config_power_cpp", + ":protos_perfetto_config_power_zero", + ":protos_perfetto_config_priority_boost_cpp", + ":protos_perfetto_config_priority_boost_zero", + ":protos_perfetto_config_process_stats_cpp", + ":protos_perfetto_config_process_stats_zero", + ":protos_perfetto_config_profiling_cpp", + ":protos_perfetto_config_profiling_zero", + ":protos_perfetto_config_protovm_cpp", + ":protos_perfetto_config_protovm_zero", + ":protos_perfetto_config_qnx_cpp", + ":protos_perfetto_config_qnx_zero", + ":protos_perfetto_config_statsd_cpp", + ":protos_perfetto_config_statsd_zero", + ":protos_perfetto_config_sys_stats_cpp", + ":protos_perfetto_config_sys_stats_zero", + ":protos_perfetto_config_system_info_cpp", + ":protos_perfetto_config_system_info_zero", + ":protos_perfetto_config_track_event_cpp", + ":protos_perfetto_config_track_event_zero", + ":protos_perfetto_config_zero", + ":protos_perfetto_ipc_cpp", + ":protos_perfetto_ipc_ipc", + ":protos_perfetto_protovm_cpp", + ":protos_perfetto_protovm_zero", + ":protos_perfetto_trace_android_zero", + ":protos_perfetto_trace_chrome_zero", + ":protos_perfetto_trace_etw_zero", + ":protos_perfetto_trace_filesystem_zero", + ":protos_perfetto_trace_ftrace_zero", + ":protos_perfetto_trace_generic_kernel_zero", + ":protos_perfetto_trace_gpu_zero", + ":protos_perfetto_trace_interned_data_zero", + ":protos_perfetto_trace_linux_zero", + ":protos_perfetto_trace_minimal_zero", + ":protos_perfetto_trace_non_minimal_zero", + ":protos_perfetto_trace_perfetto_zero", + ":protos_perfetto_trace_power_zero", + ":protos_perfetto_trace_profiling_zero", + ":protos_perfetto_trace_ps_zero", + ":protos_perfetto_trace_statsd_cpp", + ":protos_perfetto_trace_statsd_zero", + ":protos_perfetto_trace_sys_stats_zero", + ":protos_perfetto_trace_system_info_zero", + ":protos_perfetto_trace_track_event_cpp", + ":protos_perfetto_trace_track_event_zero", + ":protos_perfetto_trace_translation_zero", + ":protos_third_party_statsd_config_zero", + ":protozero", + ":src_base_base", + ":src_base_clock_snapshots", + ":src_base_regex_regex", + ":src_base_version", + ":src_protovm_protovm", + ] + PERFETTO_CONFIG.deps.zlib + + PERFETTO_CONFIG.deps.zstd, linkstatic = True, ) @@ -2526,8 +2540,8 @@ srcs = [ "src/trace_processor/importers/archive/archive_entry.cc", "src/trace_processor/importers/archive/archive_entry.h", - "src/trace_processor/importers/archive/gzip_trace_parser.cc", - "src/trace_processor/importers/archive/gzip_trace_parser.h", + "src/trace_processor/importers/archive/decompressing_trace_reader.cc", + "src/trace_processor/importers/archive/decompressing_trace_reader.h", "src/trace_processor/importers/archive/tar_trace_reader.cc", "src/trace_processor/importers/archive/tar_trace_reader.h", "src/trace_processor/importers/archive/zip_trace_reader.cc", @@ -5797,6 +5811,15 @@ ], ) +# GN target: //src/trace_processor/util:decompress +perfetto_filegroup( + name = "src_trace_processor_util_decompress", + srcs = [ + "src/trace_processor/util/decompress.cc", + "src/trace_processor/util/decompress.h", + ], +) + # GN target: //src/trace_processor/util:descriptors perfetto_filegroup( name = "src_trace_processor_util_descriptors", @@ -5827,8 +5850,8 @@ perfetto_filegroup( name = "src_trace_processor_util_gzip", srcs = [ - "src/trace_processor/util/gzip_utils.cc", - "src/trace_processor/util/gzip_utils.h", + "src/trace_processor/util/gzip_decompressor.cc", + "src/trace_processor/util/gzip_decompressor.h", ], ) @@ -5990,6 +6013,15 @@ ], ) +# GN target: //src/trace_processor/util:stream_decompressor +perfetto_filegroup( + name = "src_trace_processor_util_stream_decompressor", + srcs = [ + "src/trace_processor/util/stream_decompressor.cc", + "src/trace_processor/util/stream_decompressor.h", + ], +) + # GN target: //src/trace_processor/util:tar_writer perfetto_filegroup( name = "src_trace_processor_util_tar_writer", @@ -6028,6 +6060,15 @@ ], ) +# GN target: //src/trace_processor/util:zstd +perfetto_filegroup( + name = "src_trace_processor_util_zstd", + srcs = [ + "src/trace_processor/util/zstd_decompressor.cc", + "src/trace_processor/util/zstd_decompressor.h", + ], +) + # GN target: //src/trace_processor:demangle perfetto_cc_library( name = "src_trace_processor_demangle", @@ -6560,6 +6601,14 @@ ], ) +# GN target: //src/tracing/service:packet_compressor_common +perfetto_filegroup( + name = "src_tracing_service_packet_compressor_common", + srcs = [ + "src/tracing/service/packet_compressor_common.h", + ], +) + # GN target: //src/tracing/service:service perfetto_filegroup( name = "src_tracing_service_service", @@ -6600,6 +6649,15 @@ ], ) +# GN target: //src/tracing/service:zstd_compressor +perfetto_filegroup( + name = "src_tracing_service_zstd_compressor", + srcs = [ + "src/tracing/service/zstd_compressor.cc", + "src/tracing/service/zstd_compressor.h", + ], +) + # GN target: //src/tracing:client_api_without_backends perfetto_filegroup( name = "src_tracing_client_api_without_backends", @@ -10949,8 +11007,10 @@ ":src_tracing_ipc_default_socket", ":src_tracing_ipc_producer_producer", ":src_tracing_ipc_service_service", + ":src_tracing_service_packet_compressor_common", ":src_tracing_service_service", ":src_tracing_service_zlib_compressor", + ":src_tracing_service_zstd_compressor", ":src_tracing_system_backend", ], hdrs = [ @@ -10972,77 +11032,78 @@ "//visibility:public", ], deps = [ - ":perfetto_ipc", - ":protos_perfetto_common_cpp", - ":protos_perfetto_common_semantic_type_cpp", - ":protos_perfetto_common_semantic_type_zero", - ":protos_perfetto_common_zero", - ":protos_perfetto_config_android_cpp", - ":protos_perfetto_config_android_zero", - ":protos_perfetto_config_cpp", - ":protos_perfetto_config_ftrace_cpp", - ":protos_perfetto_config_ftrace_zero", - ":protos_perfetto_config_gpu_cpp", - ":protos_perfetto_config_gpu_zero", - ":protos_perfetto_config_inode_file_cpp", - ":protos_perfetto_config_inode_file_zero", - ":protos_perfetto_config_interceptors_cpp", - ":protos_perfetto_config_interceptors_zero", - ":protos_perfetto_config_linux_cpp", - ":protos_perfetto_config_linux_zero", - ":protos_perfetto_config_power_cpp", - ":protos_perfetto_config_power_zero", - ":protos_perfetto_config_priority_boost_cpp", - ":protos_perfetto_config_priority_boost_zero", - ":protos_perfetto_config_process_stats_cpp", - ":protos_perfetto_config_process_stats_zero", - ":protos_perfetto_config_profiling_cpp", - ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_protovm_cpp", - ":protos_perfetto_config_protovm_zero", - ":protos_perfetto_config_qnx_cpp", - ":protos_perfetto_config_qnx_zero", - ":protos_perfetto_config_statsd_cpp", - ":protos_perfetto_config_statsd_zero", - ":protos_perfetto_config_sys_stats_cpp", - ":protos_perfetto_config_sys_stats_zero", - ":protos_perfetto_config_system_info_cpp", - ":protos_perfetto_config_system_info_zero", - ":protos_perfetto_config_track_event_cpp", - ":protos_perfetto_config_track_event_zero", - ":protos_perfetto_config_zero", - ":protos_perfetto_ipc_cpp", - ":protos_perfetto_ipc_ipc", - ":protos_perfetto_protovm_cpp", - ":protos_perfetto_protovm_zero", - ":protos_perfetto_trace_android_zero", - ":protos_perfetto_trace_chrome_zero", - ":protos_perfetto_trace_etw_zero", - ":protos_perfetto_trace_filesystem_zero", - ":protos_perfetto_trace_ftrace_zero", - ":protos_perfetto_trace_generic_kernel_zero", - ":protos_perfetto_trace_gpu_zero", - ":protos_perfetto_trace_interned_data_zero", - ":protos_perfetto_trace_linux_zero", - ":protos_perfetto_trace_minimal_zero", - ":protos_perfetto_trace_non_minimal_zero", - ":protos_perfetto_trace_perfetto_zero", - ":protos_perfetto_trace_power_zero", - ":protos_perfetto_trace_profiling_zero", - ":protos_perfetto_trace_ps_zero", - ":protos_perfetto_trace_statsd_zero", - ":protos_perfetto_trace_sys_stats_zero", - ":protos_perfetto_trace_system_info_zero", - ":protos_perfetto_trace_track_event_cpp", - ":protos_perfetto_trace_track_event_zero", - ":protos_perfetto_trace_translation_zero", - ":protozero", - ":src_base_base", - ":src_base_clock_snapshots", - ":src_base_regex_regex", - ":src_base_version", - ":src_protovm_protovm", - ] + PERFETTO_CONFIG.deps.zlib, + ":perfetto_ipc", + ":protos_perfetto_common_cpp", + ":protos_perfetto_common_semantic_type_cpp", + ":protos_perfetto_common_semantic_type_zero", + ":protos_perfetto_common_zero", + ":protos_perfetto_config_android_cpp", + ":protos_perfetto_config_android_zero", + ":protos_perfetto_config_cpp", + ":protos_perfetto_config_ftrace_cpp", + ":protos_perfetto_config_ftrace_zero", + ":protos_perfetto_config_gpu_cpp", + ":protos_perfetto_config_gpu_zero", + ":protos_perfetto_config_inode_file_cpp", + ":protos_perfetto_config_inode_file_zero", + ":protos_perfetto_config_interceptors_cpp", + ":protos_perfetto_config_interceptors_zero", + ":protos_perfetto_config_linux_cpp", + ":protos_perfetto_config_linux_zero", + ":protos_perfetto_config_power_cpp", + ":protos_perfetto_config_power_zero", + ":protos_perfetto_config_priority_boost_cpp", + ":protos_perfetto_config_priority_boost_zero", + ":protos_perfetto_config_process_stats_cpp", + ":protos_perfetto_config_process_stats_zero", + ":protos_perfetto_config_profiling_cpp", + ":protos_perfetto_config_profiling_zero", + ":protos_perfetto_config_protovm_cpp", + ":protos_perfetto_config_protovm_zero", + ":protos_perfetto_config_qnx_cpp", + ":protos_perfetto_config_qnx_zero", + ":protos_perfetto_config_statsd_cpp", + ":protos_perfetto_config_statsd_zero", + ":protos_perfetto_config_sys_stats_cpp", + ":protos_perfetto_config_sys_stats_zero", + ":protos_perfetto_config_system_info_cpp", + ":protos_perfetto_config_system_info_zero", + ":protos_perfetto_config_track_event_cpp", + ":protos_perfetto_config_track_event_zero", + ":protos_perfetto_config_zero", + ":protos_perfetto_ipc_cpp", + ":protos_perfetto_ipc_ipc", + ":protos_perfetto_protovm_cpp", + ":protos_perfetto_protovm_zero", + ":protos_perfetto_trace_android_zero", + ":protos_perfetto_trace_chrome_zero", + ":protos_perfetto_trace_etw_zero", + ":protos_perfetto_trace_filesystem_zero", + ":protos_perfetto_trace_ftrace_zero", + ":protos_perfetto_trace_generic_kernel_zero", + ":protos_perfetto_trace_gpu_zero", + ":protos_perfetto_trace_interned_data_zero", + ":protos_perfetto_trace_linux_zero", + ":protos_perfetto_trace_minimal_zero", + ":protos_perfetto_trace_non_minimal_zero", + ":protos_perfetto_trace_perfetto_zero", + ":protos_perfetto_trace_power_zero", + ":protos_perfetto_trace_profiling_zero", + ":protos_perfetto_trace_ps_zero", + ":protos_perfetto_trace_statsd_zero", + ":protos_perfetto_trace_sys_stats_zero", + ":protos_perfetto_trace_system_info_zero", + ":protos_perfetto_trace_track_event_cpp", + ":protos_perfetto_trace_track_event_zero", + ":protos_perfetto_trace_translation_zero", + ":protozero", + ":src_base_base", + ":src_base_clock_snapshots", + ":src_base_regex_regex", + ":src_base_version", + ":src_protovm_protovm", + ] + PERFETTO_CONFIG.deps.zlib + + PERFETTO_CONFIG.deps.zstd, linkstatic = True, ) @@ -11310,6 +11371,7 @@ ":src_trace_processor_util_blob", ":src_trace_processor_util_bump_allocator", ":src_trace_processor_util_clock", + ":src_trace_processor_util_decompress", ":src_trace_processor_util_descriptors", ":src_trace_processor_util_elf_elf", ":src_trace_processor_util_galloping_search", @@ -11333,9 +11395,11 @@ ":src_trace_processor_util_sql_bundle", ":src_trace_processor_util_sql_module_doc_parser", ":src_trace_processor_util_stdlib", + ":src_trace_processor_util_stream_decompressor", ":src_trace_processor_util_trace_blob_view_reader", ":src_trace_processor_util_trace_type", ":src_trace_processor_util_zip_reader", + ":src_trace_processor_util_zstd", ], hdrs = [ ":include_perfetto_base_base", @@ -11448,6 +11512,7 @@ ] + PERFETTO_CONFIG.deps.sqlite + PERFETTO_CONFIG.deps.sqlite_ext_percentile + PERFETTO_CONFIG.deps.zlib + + PERFETTO_CONFIG.deps.zstd + PERFETTO_CONFIG.deps.demangle_wrapper, linkstatic = True, ) @@ -11639,6 +11704,7 @@ ":src_trace_processor_util_blob", ":src_trace_processor_util_bump_allocator", ":src_trace_processor_util_clock", + ":src_trace_processor_util_decompress", ":src_trace_processor_util_descriptors", ":src_trace_processor_util_elf_elf", ":src_trace_processor_util_galloping_search", @@ -11662,11 +11728,13 @@ ":src_trace_processor_util_sql_bundle", ":src_trace_processor_util_sql_module_doc_parser", ":src_trace_processor_util_stdlib", + ":src_trace_processor_util_stream_decompressor", ":src_trace_processor_util_tar_writer", ":src_trace_processor_util_trace_blob_view_reader", ":src_trace_processor_util_trace_enrichment_trace_enrichment", ":src_trace_processor_util_trace_type", ":src_trace_processor_util_zip_reader", + ":src_trace_processor_util_zstd", ":src_traceconv_lib", ":src_traceconv_main", ":src_traceconv_pprofbuilder", @@ -11774,6 +11842,7 @@ ] + PERFETTO_CONFIG.deps.sqlite + PERFETTO_CONFIG.deps.sqlite_ext_percentile + PERFETTO_CONFIG.deps.zlib + + PERFETTO_CONFIG.deps.zstd + PERFETTO_CONFIG.deps.demangle_wrapper, )
diff --git a/CHANGELOG b/CHANGELOG index be66cf2..f3d6d58 100644 --- a/CHANGELOG +++ b/CHANGELOG
@@ -1,5 +1,7 @@ Unreleased: Tracing service and probes: + * Added Zstd as a trace compression option alongside deflate, selected via + the new TraceConfig.compression field (which supersedes compression_type). * The procfs scraper (process_stats data source) now writes an explicit Thread message for a process' main thread, instead of leaving it implied by the process entry. This fixes missing main thread names when a trace
diff --git a/MODULE.bazel b/MODULE.bazel index 9e639dd..10941b7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -26,13 +26,14 @@ remote = "https://github.com/abseil/abseil-cpp.git", ) -bazel_dep(name = "re2", version = "2024-07-02.bcr.1") bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "platforms", version = "0.0.10") bazel_dep(name = "protobuf", version = "31.1", repo_name = "com_google_protobuf") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1") bazel_dep(name = "rules_python", version = "1.0.0") bazel_dep(name = "rules_android", version = "0.6.6") bazel_dep(name = "rules_cc", version = "0.2.19") +bazel_dep(name = "zstd", version = "1.5.7.bcr.1") remote_android_extensions = use_extension( "@rules_android//bzlmod_extensions:android_extensions.bzl",
diff --git a/bazel/deps.bzl b/bazel/deps.bzl index 17029e5..e99f126 100644 --- a/bazel/deps.bzl +++ b/bazel/deps.bzl
@@ -97,6 +97,16 @@ build_file = "//bazel:zlib.BUILD", ) + # Zstd, for the legacy WORKSPACE path (bzlmod uses the BCR module, see + # MODULE.bazel). Upstream ships no Bazel BUILD (facebook/zstd#3123). + _add_repo_if_not_existing( + new_git_repository, + name = "zstd", + remote = "https://chromium.googlesource.com/external/github.com/facebook/zstd.git", + commit = "ac66b19e6bd6b83238bf008eecc1298105298532", # refs/tags/upstream/v1.5.7 + build_file = "//bazel:zstd.BUILD", + ) + _add_repo_if_not_existing( http_archive, name = "perfetto_dep_llvm_demangle",
diff --git a/bazel/standalone/perfetto_cfg.bzl b/bazel/standalone/perfetto_cfg.bzl index f055955..d9c9689 100644 --- a/bazel/standalone/perfetto_cfg.bzl +++ b/bazel/standalone/perfetto_cfg.bzl
@@ -45,6 +45,7 @@ base_platform = ["//:perfetto_base_default_platform"], zlib = ["@perfetto_dep_zlib//:zlib"], + zstd = ["@zstd//:zstd"], re2 = ["@re2//:re2"], expat = ["@perfetto_dep_expat//:expat"], linenoise = ["@perfetto_dep_linenoise//:linenoise"], @@ -97,6 +98,7 @@ # initialized with the Perfetto build files (i.e. via perfetto_deps()). deps_copts = struct( zlib = [], + zstd = [], expat = [], linenoise = [], sqlite = [],
diff --git a/bazel/zstd.BUILD b/bazel/zstd.BUILD new file mode 100644 index 0000000..55ed704 --- /dev/null +++ b/bazel/zstd.BUILD
@@ -0,0 +1,52 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Overlay for the legacy WORKSPACE path only; bzlmod uses the BCR module (see +# MODULE.bazel). Needed because upstream zstd ships no Bazel BUILD +# (facebook/zstd#3123), so new_git_repository has no //:zstd target. + +load("@perfetto_cfg//:perfetto_cfg.bzl", "PERFETTO_CONFIG") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +cc_library( + name = "zstd", + srcs = glob([ + "lib/common/*.c", + "lib/common/*.h", + "lib/compress/*.c", + "lib/compress/*.h", + "lib/decompress/*.c", + "lib/decompress/*.h", + "lib/decompress/*.S", + "lib/dictBuilder/*.c", + "lib/dictBuilder/*.h", + ]), + hdrs = [ + "lib/zdict.h", + "lib/zstd.h", + "lib/zstd_errors.h", + ], + copts = [ + "-Wno-unused-function", + ] + PERFETTO_CONFIG.deps_copts.zstd, + # Disable the inline assembly to match the standalone GN build (which sets + # ZSTD_DISABLE_ASM on x64) and keep the build portable across toolchains. + defines = [ + "ZSTD_DISABLE_ASM", + ], + includes = [ + "lib", + ], + visibility = ["//visibility:public"], +)
diff --git a/buildtools/BUILD.gn b/buildtools/BUILD.gn index 74631e2..222112b 100644 --- a/buildtools/BUILD.gn +++ b/buildtools/BUILD.gn
@@ -1915,7 +1915,7 @@ visibility = _buildtools_visibility cflags = [ perfetto_isystem_cflag, - rebase_path("zstd", root_build_dir), + rebase_path("zstd/lib", root_build_dir), ] if (current_cpu == "x64") { defines = [ "ZSTD_DISABLE_ASM" ]
diff --git a/gn/BUILD.gn b/gn/BUILD.gn index 6f2ad9f..60ce0cb 100644 --- a/gn/BUILD.gn +++ b/gn/BUILD.gn
@@ -101,6 +101,7 @@ "PERFETTO_TP_INSTRUMENTS=$enable_perfetto_trace_processor_mac_instruments", "PERFETTO_LOCAL_SYMBOLIZER=$perfetto_local_symbolizer", "PERFETTO_ZLIB=$enable_perfetto_zlib", + "PERFETTO_ZSTD=$enable_perfetto_zstd", "PERFETTO_TRACED_PERF=$enable_perfetto_traced_perf", "PERFETTO_HEAPPROFD=$enable_perfetto_heapprofd", "PERFETTO_STDERR_CRASH_DUMP=$enable_perfetto_stderr_crash_dump", @@ -460,6 +461,19 @@ } } +# Zstd is used both by trace_processor and by the tracing service. +if (enable_perfetto_zstd) { + group("zstd") { + if (perfetto_root_path == "//") { + public_configs = [ "//buildtools:zstd_config" ] + public_deps = [ "//buildtools:zstd" ] + } else { + public_configs = [ "//third_party/zstd:zstd_config" ] + public_deps = [ "//third_party/zstd" ] + } + } +} + if (enable_perfetto_llvm_demangle) { group("llvm_demangle") { public_deps = [ "//buildtools:llvm_demangle" ]
diff --git a/gn/perfetto.gni b/gn/perfetto.gni index 7def73a..11f4dd6 100644 --- a/gn/perfetto.gni +++ b/gn/perfetto.gni
@@ -389,6 +389,12 @@ enable_perfetto_zlib = enable_perfetto_trace_processor || enable_perfetto_platform_services + # Enables Zstd support. This is used to compress traces (by the tracing + # service) and to decompress traces (by trace_processor), as an alternative + # to Zlib/deflate. + enable_perfetto_zstd = + enable_perfetto_trace_processor || enable_perfetto_platform_services + # Enables PCRE2 support. Individual modules that should not pull libpcre2 # in (e.g. libperfetto_client_experimental) can opt out by setting # -DPERFETTO_REGEX_FORCE_STD on their compile, which makes base::Regex
diff --git a/include/perfetto/base/build_config.h b/include/perfetto/base/build_config.h index 1ed224a..0c519a3 100644 --- a/include/perfetto/base/build_config.h +++ b/include/perfetto/base/build_config.h
@@ -252,16 +252,29 @@ #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB() 0 #endif +#undef PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD +#if defined(PERFETTO_SDK_ENABLE_ZSTD) && (PERFETTO_SDK_ENABLE_ZSTD == 1) +#define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD() 1 +#else +#define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD() 0 +#endif + #endif // PERFETTO_BUILDFLAG(PERFETTO_AMALGAMATED_SDK) -// Unlike PERFETTO_SDK_ENABLE_ZLIB above (amalgamated SDK opt-in), this is a -// file-scope opt-out that applies to every build. Define it to compile out the -// optional zlib compressor. Static libs like libperfetto_client_experimental -// get linked into many binaries, so keeping the compressor out means those -// binaries don't have to link libz. See tools/gen_android_bp. +// Unlike the PERFETTO_SDK_ENABLE_* opt-ins above (amalgamated SDK), these are +// file-scope opt-outs that apply to every build. Define them to compile out the +// optional zlib/zstd compressors. Static libs like +// libperfetto_client_experimental get linked into many binaries, so keeping a +// compressor out means those binaries don't have to link libz/libzstd. See +// tools/gen_android_bp. #if defined(PERFETTO_FORCE_DISABLE_ZLIB) #undef PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB() 0 #endif +#if defined(PERFETTO_FORCE_DISABLE_ZSTD) +#undef PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD +#define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD() 0 +#endif + #endif // INCLUDE_PERFETTO_BASE_BUILD_CONFIG_H_
diff --git a/include/perfetto/base/build_configs/android_tree/perfetto_build_flags.h b/include/perfetto/base/build_configs/android_tree/perfetto_build_flags.h index 8e556ee..d2f6745 100644 --- a/include/perfetto/base/build_configs/android_tree/perfetto_build_flags.h +++ b/include/perfetto/base/build_configs/android_tree/perfetto_build_flags.h
@@ -44,6 +44,7 @@ #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_INSTRUMENTS() (0) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_LOCAL_SYMBOLIZER() (0) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB() (1) +#define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD() (1) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TRACED_PERF() (1) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_HEAPPROFD() (1) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_STDERR_CRASH_DUMP() (0) @@ -84,6 +85,7 @@ {"PERFETTO_TP_HTTPD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_HTTPD()}, {"PERFETTO_TP_INSTRUMENTS", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_INSTRUMENTS()}, {"PERFETTO_ZLIB", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB()}, + {"PERFETTO_ZSTD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD()}, {"PERFETTO_TRACED_PERF", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TRACED_PERF()}, {"PERFETTO_HEAPPROFD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_HEAPPROFD()}, {"PERFETTO_STDERR_CRASH_DUMP", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_STDERR_CRASH_DUMP()},
diff --git a/include/perfetto/base/build_configs/bazel/perfetto_build_flags.h b/include/perfetto/base/build_configs/bazel/perfetto_build_flags.h index 4dfa944..a5519e4 100644 --- a/include/perfetto/base/build_configs/bazel/perfetto_build_flags.h +++ b/include/perfetto/base/build_configs/bazel/perfetto_build_flags.h
@@ -44,6 +44,7 @@ #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_INSTRUMENTS() (0) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_LOCAL_SYMBOLIZER() (PERFETTO_BUILDFLAG_DEFINE_PERFETTO_OS_LINUX() || PERFETTO_BUILDFLAG_DEFINE_PERFETTO_OS_FREEBSD() || PERFETTO_BUILDFLAG_DEFINE_PERFETTO_OS_MAC() ||PERFETTO_BUILDFLAG_DEFINE_PERFETTO_OS_WIN()) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB() (1) +#define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD() (1) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TRACED_PERF() (0) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_HEAPPROFD() (0) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_STDERR_CRASH_DUMP() (0) @@ -84,6 +85,7 @@ {"PERFETTO_TP_HTTPD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_HTTPD()}, {"PERFETTO_TP_INSTRUMENTS", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_INSTRUMENTS()}, {"PERFETTO_ZLIB", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB()}, + {"PERFETTO_ZSTD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD()}, {"PERFETTO_TRACED_PERF", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TRACED_PERF()}, {"PERFETTO_HEAPPROFD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_HEAPPROFD()}, {"PERFETTO_STDERR_CRASH_DUMP", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_STDERR_CRASH_DUMP()},
diff --git a/include/perfetto/ext/tracing/core/tracing_service.h b/include/perfetto/ext/tracing/core/tracing_service.h index aa10c57..c0eb376 100644 --- a/include/perfetto/ext/tracing/core/tracing_service.h +++ b/include/perfetto/ext/tracing/core/tracing_service.h
@@ -307,12 +307,6 @@ }; // class ConsumerEndpoint. struct PERFETTO_EXPORT_COMPONENT TracingServiceInitOpts { - // Function used by tracing service to compress packets. Takes a pointer to - // a vector of TracePackets and replaces the packets in the vector with - // compressed ones. - using CompressorFn = void (*)(std::vector<TracePacket>*); - CompressorFn compressor_fn = nullptr; - // An (optional) list of proto extension descriptors to dump into each trace // recorded. This is to support injecting protos that are known by the // embedder (e.g. the Android vendor image) but not by the upstream perfetto. @@ -385,8 +379,7 @@ kDisabled }; - // Implemented in src/core/tracing_service_impl.cc . CompressorFn can be - // nullptr, in which case TracingService will not support compression. + // Implemented in src/core/tracing_service_impl.cc . static std::unique_ptr<TracingService> CreateInstance( std::unique_ptr<SharedMemory::Factory>, base::TaskRunner*,
diff --git a/protos/perfetto/config/perfetto_config.proto b/protos/perfetto/config/perfetto_config.proto index c1dce2b..e154933 100644 --- a/protos/perfetto/config/perfetto_config.proto +++ b/protos/perfetto/config/perfetto_config.proto
@@ -4968,7 +4968,7 @@ // It contains the general config for the logging buffer(s) and the configs for // all the data source being enabled. // -// Next id: 47. +// Next id: 48. message TraceConfig { message BufferConfig { optional uint32 size_kb = 1; @@ -5428,13 +5428,40 @@ // with this key. optional string unique_session_name = 22; - // Compress trace with the given method. Best effort. + // Deprecated: use |compression| (CompressionConfig) instead. Legacy selector + // that only ever selects deflate; kept so services predating |compression| + // keep compressing. Best effort. enum CompressionType { COMPRESSION_TYPE_UNSPECIFIED = 0; COMPRESSION_TYPE_DEFLATE = 1; } optional CompressionType compression_type = 24; + // Selects and configures the compression codec, superseding |compression_type|. + // The codec is chosen by which sub-message is set; when several are set the + // service picks the highest-numbered one it supports and ignores codecs it + // doesn't know, so a single config can target old and new services. Falls back + // to |compression_type| (deflate-only), if given. + // + // Introduced in: perfetto v58. + // Supported on: Android 26Q3+. + message CompressionConfig { + // Deflate (zlib), equivalent to the legacy compression_type = DEFLATE. + message Deflate {} + message Zstd { + // Compression level, 1 (fast) to 22 (smallest); 0/unset uses zstd's + // default (3). Levels above the max are clamped; negatives are valid + // faster, lower-ratio modes. + optional int32 level = 1; + } + // If a config sets more than one of these, the service picks the one with + // the highest field number (see the rule above): e.g. with deflate (1) and + // zstd (2) both set, zstd wins. + optional Deflate deflate = 1; + optional Zstd zstd = 2; + } + optional CompressionConfig compression = 47; + // DEPRECATED, was compress_from_cli. reserved 37;
diff --git a/protos/perfetto/config/trace_config.proto b/protos/perfetto/config/trace_config.proto index 87a5f70..bf221fb 100644 --- a/protos/perfetto/config/trace_config.proto +++ b/protos/perfetto/config/trace_config.proto
@@ -28,7 +28,7 @@ // It contains the general config for the logging buffer(s) and the configs for // all the data source being enabled. // -// Next id: 47. +// Next id: 48. message TraceConfig { message BufferConfig { optional uint32 size_kb = 1; @@ -488,13 +488,40 @@ // with this key. optional string unique_session_name = 22; - // Compress trace with the given method. Best effort. + // Deprecated: use |compression| (CompressionConfig) instead. Legacy selector + // that only ever selects deflate; kept so services predating |compression| + // keep compressing. Best effort. enum CompressionType { COMPRESSION_TYPE_UNSPECIFIED = 0; COMPRESSION_TYPE_DEFLATE = 1; } optional CompressionType compression_type = 24; + // Selects and configures the compression codec, superseding + // |compression_type|. The codec is chosen by which sub-message is set; when + // several are set the service picks the highest-numbered one it supports and + // ignores codecs it doesn't know, so a single config can target old and new + // services. Falls back to |compression_type| (deflate-only), if given. + // + // Introduced in: perfetto v58. + // Supported on: Android 26Q3+. + message CompressionConfig { + // Deflate (zlib), equivalent to the legacy `COMPRESSION_TYPE_DEFLATE`. + message Deflate {} + message Zstd { + // Compression level, 1 (fast) to 22 (smallest); 0/unset uses zstd's + // default (3). Levels above the max are clamped; negatives are valid + // faster, lower-ratio modes. + optional int32 level = 1; + } + // If a config sets more than one of these, the service picks the one with + // the highest field number (see the rule above): e.g. with deflate (1) and + // zstd (2) both set, zstd wins. + optional Deflate deflate = 1; + optional Zstd zstd = 2; + } + optional CompressionConfig compression = 47; + // DEPRECATED, was compress_from_cli. reserved 37;
diff --git a/protos/perfetto/trace/perfetto_trace.proto b/protos/perfetto/trace/perfetto_trace.proto index 4e16aa5..606851f 100644 --- a/protos/perfetto/trace/perfetto_trace.proto +++ b/protos/perfetto/trace/perfetto_trace.proto
@@ -4968,7 +4968,7 @@ // It contains the general config for the logging buffer(s) and the configs for // all the data source being enabled. // -// Next id: 47. +// Next id: 48. message TraceConfig { message BufferConfig { optional uint32 size_kb = 1; @@ -5428,13 +5428,40 @@ // with this key. optional string unique_session_name = 22; - // Compress trace with the given method. Best effort. + // Deprecated: use |compression| (CompressionConfig) instead. Legacy selector + // that only ever selects deflate; kept so services predating |compression| + // keep compressing. Best effort. enum CompressionType { COMPRESSION_TYPE_UNSPECIFIED = 0; COMPRESSION_TYPE_DEFLATE = 1; } optional CompressionType compression_type = 24; + // Selects and configures the compression codec, superseding |compression_type|. + // The codec is chosen by which sub-message is set; when several are set the + // service picks the highest-numbered one it supports and ignores codecs it + // doesn't know, so a single config can target old and new services. Falls back + // to |compression_type| (deflate-only), if given. + // + // Introduced in: perfetto v58. + // Supported on: Android 26Q3+. + message CompressionConfig { + // Deflate (zlib), equivalent to the legacy compression_type = DEFLATE. + message Deflate {} + message Zstd { + // Compression level, 1 (fast) to 22 (smallest); 0/unset uses zstd's + // default (3). Levels above the max are clamped; negatives are valid + // faster, lower-ratio modes. + optional int32 level = 1; + } + // If a config sets more than one of these, the service picks the one with + // the highest field number (see the rule above): e.g. with deflate (1) and + // zstd (2) both set, zstd wins. + optional Deflate deflate = 1; + optional Zstd zstd = 2; + } + optional CompressionConfig compression = 47; + // DEPRECATED, was compress_from_cli. reserved 37; @@ -17821,7 +17848,9 @@ // efficiently partition long traces without having to fully parse them. bytes synchronization_marker = 36; - // Zero or more proto encoded trace packets compressed using deflate. + // Zero or more proto-encoded trace packets, compressed with deflate or + // zstd. The bytes carry no codec tag, and only zstd has a distinctive + // magic, so readers detect zstd by it and treat everything else as deflate. // Each compressed_packets TracePacket (including the two field ids and // sizes) should be less than 512KB. bytes compressed_packets = 50;
diff --git a/protos/perfetto/trace/trace_packet.proto b/protos/perfetto/trace/trace_packet.proto index 09d1b53..5dbce1f 100644 --- a/protos/perfetto/trace/trace_packet.proto +++ b/protos/perfetto/trace/trace_packet.proto
@@ -243,7 +243,9 @@ // efficiently partition long traces without having to fully parse them. bytes synchronization_marker = 36; - // Zero or more proto encoded trace packets compressed using deflate. + // Zero or more proto-encoded trace packets, compressed with deflate or + // zstd. The bytes carry no codec tag, and only zstd has a distinctive + // magic, so readers detect zstd by it and treat everything else as deflate. // Each compressed_packets TracePacket (including the two field ids and // sizes) should be less than 512KB. bytes compressed_packets = 50;
diff --git a/src/tools/protoprofile/BUILD.gn b/src/tools/protoprofile/BUILD.gn index 3446fad..f65711e 100644 --- a/src/tools/protoprofile/BUILD.gn +++ b/src/tools/protoprofile/BUILD.gn
@@ -30,8 +30,8 @@ "../../base", "../../protozero", "../../trace_processor:lib", + "../../trace_processor/util:decompress", "../../trace_processor/util:descriptors", - "../../trace_processor/util:gzip", "../../trace_processor/util:proto_profiler", ] sources = [ "main.cc" ]
diff --git a/src/tools/protoprofile/main.cc b/src/tools/protoprofile/main.cc index 4ae23ac..77f89cb 100644 --- a/src/tools/protoprofile/main.cc +++ b/src/tools/protoprofile/main.cc
@@ -19,7 +19,6 @@ #include <cstddef> #include <cstdint> #include <cstdio> -#include <cstring> #include <string> #include <unordered_map> #include <vector> @@ -34,8 +33,8 @@ #include "perfetto/protozero/scattered_heap_buffer.h" #include "src/trace_processor/importers/proto/android_extension.descriptor.h" #include "src/trace_processor/importers/proto/trace.descriptor.h" +#include "src/trace_processor/util/decompress.h" #include "src/trace_processor/util/descriptors.h" -#include "src/trace_processor/util/gzip_utils.h" #include "src/trace_processor/util/proto_profiler.h" #include "protos/third_party/pprof/profile.pbzero.h" @@ -205,23 +204,22 @@ std::string s; base::ReadFileDescriptor(proto_fd.get(), &s); - // Transparently handle gzip-compressed traces. Detect them by the gzip - // magic bytes (0x1f 0x8b) and decompress using the existing zlib-backed - // GzipDecompressor utility. - const uint8_t kGzipMagic[] = {0x1f, 0x8b}; - if (s.size() >= sizeof(kGzipMagic) && - memcmp(s.data(), kGzipMagic, sizeof(kGzipMagic)) == 0) { - if (!trace_processor::util::IsGzipSupported()) { + // Transparently handle compressed traces (gzip or zstd): detect the codec + // from the file's magic bytes and decompress it whole. + namespace util = trace_processor::util; + const auto* bytes = reinterpret_cast<const uint8_t*>(s.data()); + util::CompressionType codec = util::DetectCompression(bytes, s.size()); + if (codec != util::CompressionType::kNone) { + if (!util::IsCompressionSupported(codec)) { PERFETTO_ELOG( - "Input (%s) is gzip-compressed but this build lacks zlib support", + "Input (%s) is compressed but this build lacks the required codec", input_path); return 1; } std::vector<uint8_t> decompressed = - trace_processor::util::GzipDecompressor::DecompressFully( - reinterpret_cast<const uint8_t*>(s.data()), s.size()); + util::DecompressFully(codec, bytes, s.size()); if (decompressed.empty()) { - PERFETTO_ELOG("Could not decompress gzip input path (%s)", input_path); + PERFETTO_ELOG("Could not decompress input path (%s)", input_path); return 1; } s.assign(reinterpret_cast<const char*>(decompressed.data()),
diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn index 9ac91de..a2fab10 100644 --- a/src/trace_processor/BUILD.gn +++ b/src/trace_processor/BUILD.gn
@@ -148,6 +148,7 @@ "util:gzip", "util:proto_to_args_parser", "util:trace_type", + "util:zstd", ] public_deps = [ "../../include/perfetto/trace_processor:storage" ] } @@ -265,6 +266,7 @@ "trace_summary:gen_cc_trace_summary_descriptor", "types", "util:clock", + "util:decompress", "util:gzip", "util:json_parser", "util:protozero_to_json", @@ -273,6 +275,7 @@ "util:sql_bundle", "util:stdlib", "util:trace_type", + "util:zstd", ] if (enable_perfetto_etm_importer) { @@ -469,6 +472,7 @@ ":lib", "../../gn:default_deps", "../../gn:gtest_and_gmock", + "../../include/perfetto/protozero", "../../protos/perfetto/common:zero", "../../protos/perfetto/trace:zero", "../../protos/perfetto/trace_processor:zero", @@ -477,6 +481,9 @@ "sqlite", "trace_summary:integrationtests", ] + if (enable_perfetto_zstd) { + deps += [ "../../gn:zstd" ] + } } } }
diff --git a/src/trace_processor/forwarding_trace_parser.cc b/src/trace_processor/forwarding_trace_parser.cc index 985df24..d8d3267 100644 --- a/src/trace_processor/forwarding_trace_parser.cc +++ b/src/trace_processor/forwarding_trace_parser.cc
@@ -62,6 +62,7 @@ const TraceProcessorContext& context) { switch (trace_type) { case kGzipTraceType: + case kZstdTraceType: return std::nullopt; case kAndroidDumpstateTraceType:
diff --git a/src/trace_processor/importers/archive/BUILD.gn b/src/trace_processor/importers/archive/BUILD.gn index 67a84b5..fd37e5c 100644 --- a/src/trace_processor/importers/archive/BUILD.gn +++ b/src/trace_processor/importers/archive/BUILD.gn
@@ -16,8 +16,8 @@ sources = [ "archive_entry.cc", "archive_entry.h", - "gzip_trace_parser.cc", - "gzip_trace_parser.h", + "decompressing_trace_reader.cc", + "decompressing_trace_reader.h", "tar_trace_reader.cc", "tar_trace_reader.h", "zip_trace_reader.cc", @@ -33,7 +33,7 @@ "../../storage", "../../tables:tables_python", "../../types", - "../../util:gzip", + "../../util:decompress", "../../util:trace_blob_view_reader", "../../util:trace_type", "../../util:zip_reader",
diff --git a/src/trace_processor/importers/archive/decompressing_trace_reader.cc b/src/trace_processor/importers/archive/decompressing_trace_reader.cc new file mode 100644 index 0000000..a51d1c8 --- /dev/null +++ b/src/trace_processor/importers/archive/decompressing_trace_reader.cc
@@ -0,0 +1,157 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/archive/decompressing_trace_reader.h" + +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <memory> +#include <string> +#include <utility> + +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/ext/base/status_macros.h" +#include "perfetto/ext/base/string_utils.h" +#include "perfetto/ext/base/string_view.h" +#include "perfetto/trace_processor/trace_blob.h" +#include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/forwarding_trace_parser.h" +#include "src/trace_processor/importers/common/chunked_trace_reader.h" +#include "src/trace_processor/importers/common/trace_file_tracker.h" +#include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/decompress.h" + +namespace perfetto::trace_processor { + +DecompressingTraceReader::DecompressingTraceReader( + TraceProcessorContext* context, + util::CompressionType type) + : context_(context), + type_(type), + decompressor_(util::CreateDecompressor(type)) {} + +DecompressingTraceReader::DecompressingTraceReader( + std::unique_ptr<ChunkedTraceReader> reader, + util::CompressionType type) + : context_(nullptr), + type_(type), + decompressor_(util::CreateDecompressor(type)), + inner_(std::move(reader)) {} + +DecompressingTraceReader::~DecompressingTraceReader() = default; + +base::Status DecompressingTraceReader::Parse(TraceBlobView blob) { + return ParseUnowned(blob.data(), blob.size()); +} + +base::Status DecompressingTraceReader::ParseUnowned(const uint8_t* data, + size_t size) { + if (!decompressor_) { + return base::ErrStatus( + "Cannot decompress trace: the codec is not enabled in the build " + "config"); + } + + const uint8_t* start = data; + size_t len = size; + + if (!inner_) { + PERFETTO_CHECK(context_); + inner_.reset(new ForwardingTraceParser( + context_, context_->trace_file_tracker->AddFile(""))); + } + + if (!first_chunk_parsed_) { + first_chunk_parsed_ = true; + // .ctrace files (gzip) begin with "TRACE:\n" or "done. TRACE:\n"; strip it + // if present. This framing is gzip-only, so don't look for it otherwise. + if (type_ == util::CompressionType::kGzip) { + base::StringView beginning(reinterpret_cast<const char*>(start), size); + static const char* kSystraceFileHeader = "TRACE:\n"; + size_t offset = base::Find(kSystraceFileHeader, beginning); + if (offset != std::string::npos) { + start += strlen(kSystraceFileHeader) + offset; + len -= strlen(kSystraceFileHeader) + offset; + } + } + } + + // Our default uncompressed buffer size is 32MB as it allows for good + // throughput. + constexpr size_t kUncompressedBufferSize = 32ul * 1024 * 1024; + decompressor_->Feed(start, len); + + using ResultCode = util::StreamDecompressor::ResultCode; + for (;;) { + if (!buffer_) { + buffer_.reset(new uint8_t[kUncompressedBufferSize]); + bytes_written_ = 0; + } + + auto result = + decompressor_->ExtractOutput(buffer_.get() + bytes_written_, + kUncompressedBufferSize - bytes_written_); + if (result.ret == ResultCode::kError) + return base::ErrStatus("Failed to decompress trace chunk"); + + if (result.ret == ResultCode::kNeedsMoreInput) { + PERFETTO_DCHECK(result.bytes_written == 0); + return base::OkStatus(); + } + bytes_written_ += result.bytes_written; + output_state_ = kMidStream; + + if (bytes_written_ == kUncompressedBufferSize || + result.ret == ResultCode::kEof) { + TraceBlob blob = + TraceBlob::TakeOwnership(std::move(buffer_), bytes_written_); + RETURN_IF_ERROR(inner_->Parse(TraceBlobView(std::move(blob)))); + } + + // A compressed file may contain multiple concatenated streams/frames (valid + // for gzip per RFC1952 §2.2, and for zstd). When one is fully decoded, + // reset the decompressor to begin the next: all other state can be + // preserved. + if (result.ret == ResultCode::kEof) { + decompressor_->Reset(); + output_state_ = kStreamBoundary; + + if (decompressor_->AvailIn() == 0) { + return base::OkStatus(); + } + } + } +} + +base::Status DecompressingTraceReader::OnPushDataToSorter() { + if (output_state_ != kStreamBoundary || + (decompressor_ && decompressor_->AvailIn() > 0)) { + return base::ErrStatus( + "Compressed stream incomplete, trace is likely corrupt"); + } + PERFETTO_CHECK(!buffer_); + return inner_ ? inner_->OnPushDataToSorter() : base::OkStatus(); +} + +void DecompressingTraceReader::OnEventsFullyExtracted() { + if (inner_) { + inner_->OnEventsFullyExtracted(); + } +} + +} // namespace perfetto::trace_processor
diff --git a/src/trace_processor/importers/archive/decompressing_trace_reader.h b/src/trace_processor/importers/archive/decompressing_trace_reader.h new file mode 100644 index 0000000..e54851a --- /dev/null +++ b/src/trace_processor/importers/archive/decompressing_trace_reader.h
@@ -0,0 +1,65 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_ARCHIVE_DECOMPRESSING_TRACE_READER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_ARCHIVE_DECOMPRESSING_TRACE_READER_H_ + +#include <cstddef> +#include <cstdint> +#include <memory> + +#include "perfetto/base/status.h" +#include "src/trace_processor/importers/common/chunked_trace_reader.h" +#include "src/trace_processor/util/stream_decompressor.h" + +namespace perfetto::trace_processor { + +class TraceProcessorContext; + +// Parses a whole-file compressed trace (gzip or zstd), streaming the +// decompressed bytes into an inner ForwardingTraceParser. The codec is fixed at +// construction; the trace type detection picks it (kGzipTraceType, +// kCtraceTraceType -> gzip; kZstdTraceType -> zstd). +class DecompressingTraceReader : public ChunkedTraceReader { + public: + DecompressingTraceReader(TraceProcessorContext*, util::CompressionType); + DecompressingTraceReader(std::unique_ptr<ChunkedTraceReader>, + util::CompressionType); + ~DecompressingTraceReader() override; + + // ChunkedTraceReader implementation + base::Status Parse(TraceBlobView) override; + base::Status OnPushDataToSorter() override; + void OnEventsFullyExtracted() override; + + base::Status ParseUnowned(const uint8_t*, size_t); + + private: + TraceProcessorContext* const context_; + const util::CompressionType type_; + std::unique_ptr<util::StreamDecompressor> decompressor_; + std::unique_ptr<ChunkedTraceReader> inner_; + + std::unique_ptr<uint8_t[]> buffer_; + size_t bytes_written_ = 0; + + bool first_chunk_parsed_ = false; + enum { kStreamBoundary, kMidStream } output_state_ = kStreamBoundary; +}; + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_ARCHIVE_DECOMPRESSING_TRACE_READER_H_
diff --git a/src/trace_processor/importers/archive/gzip_trace_parser.cc b/src/trace_processor/importers/archive/gzip_trace_parser.cc deleted file mode 100644 index 01e9f50..0000000 --- a/src/trace_processor/importers/archive/gzip_trace_parser.cc +++ /dev/null
@@ -1,142 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/archive/gzip_trace_parser.h" - -#include <cstdint> -#include <cstring> -#include <memory> -#include <string> -#include <utility> - -#include "perfetto/base/logging.h" -#include "perfetto/base/status.h" -#include "perfetto/ext/base/status_macros.h" -#include "perfetto/ext/base/string_utils.h" -#include "perfetto/ext/base/string_view.h" -#include "perfetto/trace_processor/trace_blob.h" -#include "perfetto/trace_processor/trace_blob_view.h" -#include "src/trace_processor/forwarding_trace_parser.h" -#include "src/trace_processor/importers/common/chunked_trace_reader.h" -#include "src/trace_processor/importers/common/trace_file_tracker.h" -#include "src/trace_processor/types/trace_processor_context.h" -#include "src/trace_processor/util/gzip_utils.h" - -namespace perfetto::trace_processor { - -namespace { - -using ResultCode = util::GzipDecompressor::ResultCode; - -} // namespace - -GzipTraceParser::GzipTraceParser(TraceProcessorContext* context) - : context_(context) {} - -GzipTraceParser::GzipTraceParser(std::unique_ptr<ChunkedTraceReader> reader) - : context_(nullptr), inner_(std::move(reader)) {} - -GzipTraceParser::~GzipTraceParser() = default; - -base::Status GzipTraceParser::Parse(TraceBlobView blob) { - return ParseUnowned(blob.data(), blob.size()); -} - -base::Status GzipTraceParser::ParseUnowned(const uint8_t* data, size_t size) { - const uint8_t* start = data; - size_t len = size; - - if (!inner_) { - PERFETTO_CHECK(context_); - inner_.reset(new ForwardingTraceParser( - context_, context_->trace_file_tracker->AddFile(""))); - } - - if (!first_chunk_parsed_) { - // .ctrace files begin with: "TRACE:\n" or "done. TRACE:\n" strip this if - // present. - base::StringView beginning(reinterpret_cast<const char*>(start), size); - - static const char* kSystraceFileHeader = "TRACE:\n"; - size_t offset = Find(kSystraceFileHeader, beginning); - if (offset != std::string::npos) { - start += strlen(kSystraceFileHeader) + offset; - len -= strlen(kSystraceFileHeader) + offset; - } - first_chunk_parsed_ = true; - } - - // Our default uncompressed buffer size is 32MB as it allows for good - // throughput. - constexpr size_t kUncompressedBufferSize = 32ul * 1024 * 1024; - decompressor_.Feed(start, len); - - for (;;) { - if (!buffer_) { - buffer_.reset(new uint8_t[kUncompressedBufferSize]); - bytes_written_ = 0; - } - - auto result = - decompressor_.ExtractOutput(buffer_.get() + bytes_written_, - kUncompressedBufferSize - bytes_written_); - util::GzipDecompressor::ResultCode ret = result.ret; - if (ret == ResultCode::kError) - return base::ErrStatus("Failed to decompress trace chunk"); - - if (ret == ResultCode::kNeedsMoreInput) { - PERFETTO_DCHECK(result.bytes_written == 0); - return base::OkStatus(); - } - bytes_written_ += result.bytes_written; - output_state_ = kMidStream; - - if (bytes_written_ == kUncompressedBufferSize || ret == ResultCode::kEof) { - TraceBlob blob = - TraceBlob::TakeOwnership(std::move(buffer_), bytes_written_); - RETURN_IF_ERROR(inner_->Parse(TraceBlobView(std::move(blob)))); - } - - // We support multiple gzip streams in a single gzip file (which is valid - // according to RFC1952 section 2.2): in that case, we just need to reset - // the decompressor to begin processing the next stream: all other variables - // can be preserved. - if (ret == ResultCode::kEof) { - decompressor_.Reset(); - output_state_ = kStreamBoundary; - - if (decompressor_.AvailIn() == 0) { - return base::OkStatus(); - } - } - } -} - -base::Status GzipTraceParser::OnPushDataToSorter() { - if (output_state_ != kStreamBoundary || decompressor_.AvailIn() > 0) { - return base::ErrStatus("GZIP stream incomplete, trace is likely corrupt"); - } - PERFETTO_CHECK(!buffer_); - return inner_ ? inner_->OnPushDataToSorter() : base::OkStatus(); -} - -void GzipTraceParser::OnEventsFullyExtracted() { - if (inner_) { - inner_->OnEventsFullyExtracted(); - } -} - -} // namespace perfetto::trace_processor
diff --git a/src/trace_processor/importers/archive/gzip_trace_parser.h b/src/trace_processor/importers/archive/gzip_trace_parser.h deleted file mode 100644 index b756d86..0000000 --- a/src/trace_processor/importers/archive/gzip_trace_parser.h +++ /dev/null
@@ -1,59 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_ARCHIVE_GZIP_TRACE_PARSER_H_ -#define SRC_TRACE_PROCESSOR_IMPORTERS_ARCHIVE_GZIP_TRACE_PARSER_H_ - -#include <cstddef> -#include <cstdint> -#include <memory> - -#include "perfetto/base/status.h" -#include "src/trace_processor/importers/common/chunked_trace_reader.h" -#include "src/trace_processor/util/gzip_utils.h" - -namespace perfetto::trace_processor { - -class TraceProcessorContext; - -class GzipTraceParser : public ChunkedTraceReader { - public: - explicit GzipTraceParser(TraceProcessorContext*); - explicit GzipTraceParser(std::unique_ptr<ChunkedTraceReader>); - ~GzipTraceParser() override; - - // ChunkedTraceReader implementation - base::Status Parse(TraceBlobView) override; - base::Status OnPushDataToSorter() override; - void OnEventsFullyExtracted() override; - - base::Status ParseUnowned(const uint8_t*, size_t); - - private: - TraceProcessorContext* const context_; - util::GzipDecompressor decompressor_; - std::unique_ptr<ChunkedTraceReader> inner_; - - std::unique_ptr<uint8_t[]> buffer_; - size_t bytes_written_ = 0; - - bool first_chunk_parsed_ = false; - enum { kStreamBoundary, kMidStream } output_state_ = kStreamBoundary; -}; - -} // namespace perfetto::trace_processor - -#endif // SRC_TRACE_PROCESSOR_IMPORTERS_ARCHIVE_GZIP_TRACE_PARSER_H_
diff --git a/src/trace_processor/importers/proto/BUILD.gn b/src/trace_processor/importers/proto/BUILD.gn index edde569..3de5b15 100644 --- a/src/trace_processor/importers/proto/BUILD.gn +++ b/src/trace_processor/importers/proto/BUILD.gn
@@ -115,12 +115,14 @@ "../../tables", "../../types", "../../util:build_id", + "../../util:decompress", "../../util:gzip", "../../util:json_args", "../../util:json_parser", "../../util:profiler_util", "../../util:simple_json_parser", "../../util:trace_blob_view_reader", + "../../util:zstd", "../common", "../common:parser_types", "../common:v8_profile_parser",
diff --git a/src/trace_processor/importers/proto/proto_trace_reader.cc b/src/trace_processor/importers/proto/proto_trace_reader.cc index b2e8808..20b3c59 100644 --- a/src/trace_processor/importers/proto/proto_trace_reader.cc +++ b/src/trace_processor/importers/proto/proto_trace_reader.cc
@@ -60,8 +60,8 @@ #include "src/trace_processor/tables/metadata_tables_py.h" #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/types/variadic.h" +#include "src/trace_processor/util/decompress.h" #include "src/trace_processor/util/descriptors.h" -#include "src/trace_processor/util/gzip_utils.h" #include "protos/perfetto/common/builtin_clock.pbzero.h" #include "protos/perfetto/common/trace_attributes.pbzero.h" @@ -229,8 +229,8 @@ size = extension.size; } else if (decoder.has_extension_set_gzip()) { auto gzipped = decoder.extension_set_gzip(); - decompressed = - util::GzipDecompressor::DecompressFully(gzipped.data, gzipped.size); + decompressed = util::DecompressFully(util::CompressionType::kGzip, + gzipped.data, gzipped.size); if (decompressed.empty()) { return base::ErrStatus( "Failed to decompress gzipped extension descriptor");
diff --git a/src/trace_processor/importers/proto/proto_trace_tokenizer.cc b/src/trace_processor/importers/proto/proto_trace_tokenizer.cc index 67414bc..2defd21 100644 --- a/src/trace_processor/importers/proto/proto_trace_tokenizer.cc +++ b/src/trace_processor/importers/proto/proto_trace_tokenizer.cc
@@ -18,6 +18,7 @@ #include "perfetto/trace_processor/trace_blob.h" #include "perfetto/ext/base/utils.h" +#include "src/trace_processor/util/decompress.h" namespace perfetto { namespace trace_processor { @@ -26,22 +27,34 @@ base::Status ProtoTraceTokenizer::Decompress(TraceBlobView input, TraceBlobView* output) { - PERFETTO_DCHECK(util::IsGzipSupported()); + util::CompressionType type = + util::DetectPacketCompression(input.data(), input.length()); + + // Reuse the cached decompressor across packets; only rebuild it if the codec + // changed (rare). Reset() clears any prior frame's state. + if (!decompressor_ || decompressor_type_ != type) { + decompressor_ = util::CreateDecompressor(type); + decompressor_type_ = type; + if (!decompressor_) { + return base::ErrStatus( + "Cannot decompress compressed_packets: %s is not enabled in the " + "build config", + type == util::CompressionType::kZstd ? "zstd" : "zlib"); + } + } else { + decompressor_->Reset(); + } std::vector<uint8_t> data; data.reserve(input.length()); - - // Ensure that the decompressor is able to cope with a new stream of data. - decompressor_.Reset(); - using ResultCode = util::GzipDecompressor::ResultCode; - ResultCode ret = decompressor_.FeedAndExtract( - input.data(), input.length(), - [&data](const uint8_t* buffer, size_t buffer_len) { - data.insert(data.end(), buffer, buffer + buffer_len); - }); - - if (ret == ResultCode::kError || ret == ResultCode::kNeedsMoreInput) { - return base::ErrStatus("Failed to decompress (error code: %d)", + auto consumer = [&data](const uint8_t* buffer, size_t buffer_len) { + data.insert(data.end(), buffer, buffer + buffer_len); + }; + using ResultCode = util::StreamDecompressor::ResultCode; + ResultCode ret = + decompressor_->FeedAndExtract(input.data(), input.length(), consumer); + if (ret != ResultCode::kEof) { + return base::ErrStatus("Failed to decompress compressed_packets (code: %d)", static_cast<int>(ret)); }
diff --git a/src/trace_processor/importers/proto/proto_trace_tokenizer.h b/src/trace_processor/importers/proto/proto_trace_tokenizer.h index e712bd2..033def7 100644 --- a/src/trace_processor/importers/proto/proto_trace_tokenizer.h +++ b/src/trace_processor/importers/proto/proto_trace_tokenizer.h
@@ -20,6 +20,7 @@ #include <algorithm> #include <cstddef> #include <cstdint> +#include <memory> #include <optional> #include <utility> @@ -30,7 +31,7 @@ #include "perfetto/public/compiler.h" #include "perfetto/trace_processor/trace_blob_view.h" #include "protos/perfetto/trace/trace_packet.pbzero.h" -#include "src/trace_processor/util/gzip_utils.h" +#include "src/trace_processor/util/stream_decompressor.h" #include "perfetto/ext/base/status_macros.h" #include "protos/perfetto/trace/trace.pbzero.h" @@ -181,11 +182,6 @@ continue; } - if (!util::IsGzipSupported()) { - return base::ErrStatus( - "Cannot decode compressed packets. Zlib not enabled"); - } - protozero::ConstBytes field = decoder.compressed_packets(); TraceBlobView compressed_packets = packet->slice(field.data, field.size); TraceBlobView packets; @@ -224,8 +220,10 @@ // Parse() boundaries. util::TraceBlobViewReader reader_; - // Allows support for compressed trace packets. - util::GzipDecompressor decompressor_; + // Decompressor for `compressed_packets`, reused across packets; + // `decompressor_type_` is the codec it was built for. + std::unique_ptr<util::StreamDecompressor> decompressor_; + util::CompressionType decompressor_type_ = util::CompressionType::kNone; }; } // namespace perfetto::trace_processor
diff --git a/src/trace_processor/plugins/wattson/BUILD.gn b/src/trace_processor/plugins/wattson/BUILD.gn index 6f45b1d..fdc3704 100644 --- a/src/trace_processor/plugins/wattson/BUILD.gn +++ b/src/trace_processor/plugins/wattson/BUILD.gn
@@ -116,6 +116,6 @@ "../../core/plugin", "../../storage", "../../types", - "../../util:gzip", + "../../util:decompress", ] }
diff --git a/src/trace_processor/plugins/wattson/table_function.cc b/src/trace_processor/plugins/wattson/table_function.cc index 1e05c5f..bd93e7d 100644 --- a/src/trace_processor/plugins/wattson/table_function.cc +++ b/src/trace_processor/plugins/wattson/table_function.cc
@@ -35,7 +35,7 @@ #include "src/trace_processor/core/dataframe/adhoc_dataframe_builder.h" #include "src/trace_processor/core/dataframe/dataframe.h" #include "src/trace_processor/core/dataframe/specs.h" -#include "src/trace_processor/util/gzip_utils.h" +#include "src/trace_processor/util/decompress.h" namespace perfetto::trace_processor::wattson { namespace { @@ -167,8 +167,8 @@ } base::StatusOr<Dataframe> WattsonCurvesTableFunction::BuildDataframe() const { - std::vector<uint8_t> bytes = util::GzipDecompressor::DecompressFully( - compressed_blob_, compressed_blob_size_); + std::vector<uint8_t> bytes = util::DecompressFully( + util::CompressionType::kGzip, compressed_blob_, compressed_blob_size_); WattsonCurvesBlobReader reader(bytes); uint32_t row_count = reader.Read<uint32_t>(); uint32_t string_count = reader.Read<uint32_t>();
diff --git a/src/trace_processor/read_trace.cc b/src/trace_processor/read_trace.cc index a580b20..6c3d2a2 100644 --- a/src/trace_processor/read_trace.cc +++ b/src/trace_processor/read_trace.cc
@@ -27,11 +27,11 @@ #include "perfetto/protozero/proto_utils.h" #include "perfetto/trace_processor/trace_blob_view.h" #include "perfetto/trace_processor/trace_processor.h" -#include "src/trace_processor/importers/archive/gzip_trace_parser.h" +#include "src/trace_processor/importers/archive/decompressing_trace_reader.h" #include "src/trace_processor/importers/common/chunked_trace_reader.h" #include "src/trace_processor/importers/proto/proto_trace_tokenizer.h" #include "src/trace_processor/read_trace_internal.h" -#include "src/trace_processor/util/gzip_utils.h" +#include "src/trace_processor/util/decompress.h" #include "src/trace_processor/util/trace_type.h" #include "protos/perfetto/trace/trace.pbzero.h" @@ -91,15 +91,20 @@ size_t size, std::vector<uint8_t>* output) { TraceType type = GuessTraceType(data, size); - if (type != TraceType::kGzipTraceType && type != TraceType::kProtoTraceType) { + if (type != TraceType::kGzipTraceType && type != TraceType::kZstdTraceType && + type != TraceType::kProtoTraceType) { return base::ErrStatus( - "Only GZIP and proto trace types are supported by DecompressTrace"); + "Only GZIP, ZSTD and proto trace types are supported by " + "DecompressTrace"); } - if (type == TraceType::kGzipTraceType) { + if (type == TraceType::kGzipTraceType || type == TraceType::kZstdTraceType) { + auto codec = type == TraceType::kZstdTraceType + ? util::CompressionType::kZstd + : util::CompressionType::kGzip; std::unique_ptr<ChunkedTraceReader> reader( new SerializingProtoTraceReader(output)); - GzipTraceParser parser(std::move(reader)); + DecompressingTraceReader parser(std::move(reader), codec); RETURN_IF_ERROR(parser.ParseUnowned(data, size)); RETURN_IF_ERROR(parser.OnPushDataToSorter()); parser.OnEventsFullyExtracted(); @@ -109,10 +114,13 @@ PERFETTO_CHECK(type == TraceType::kProtoTraceType); protos::pbzero::Trace::Decoder decoder(data, size); - util::GzipDecompressor decompressor; if (size > 0 && !decoder.packet()) { return base::ErrStatus("Trace does not contain valid packets"); } + // Reused across compressed_packets entries; rebuilt only if the codec changes + // (it doesn't within a trace). + std::unique_ptr<util::StreamDecompressor> decompressor; + util::CompressionType decompressor_type = util::CompressionType::kNone; for (auto it = decoder.packet(); it; ++it) { protos::pbzero::TracePacket::Decoder packet(*it); if (!packet.has_compressed_packets()) { @@ -120,16 +128,27 @@ continue; } - // Make sure that to reset the stream between the gzip streams. auto bytes = packet.compressed_packets(); - decompressor.Reset(); - using ResultCode = util::GzipDecompressor::ResultCode; - ResultCode ret = decompressor.FeedAndExtract( - bytes.data, bytes.size, [&output](const uint8_t* buf, size_t buf_len) { - output->insert(output->end(), buf, buf + buf_len); - }); - if (ret == ResultCode::kError || ret == ResultCode::kNeedsMoreInput) { - return base::ErrStatus("Failed while decompressing stream"); + auto consumer = [&output](const uint8_t* buf, size_t buf_len) { + output->insert(output->end(), buf, buf + buf_len); + }; + util::CompressionType codec = + util::DetectPacketCompression(bytes.data, bytes.size); + if (!decompressor || decompressor_type != codec) { + decompressor = util::CreateDecompressor(codec); + decompressor_type = codec; + if (!decompressor) { + return base::ErrStatus( + "Cannot decompress compressed_packets: %s is not enabled in the " + "build config", + codec == util::CompressionType::kZstd ? "zstd" : "zlib"); + } + } else { + decompressor->Reset(); + } + if (decompressor->FeedAndExtract(bytes.data, bytes.size, consumer) != + util::StreamDecompressor::ResultCode::kEof) { + return base::ErrStatus("Failed while decompressing compressed_packets"); } } return base::OkStatus();
diff --git a/src/trace_processor/read_trace_integrationtest.cc b/src/trace_processor/read_trace_integrationtest.cc index 7eb883c..d1f1bbf 100644 --- a/src/trace_processor/read_trace_integrationtest.cc +++ b/src/trace_processor/read_trace_integrationtest.cc
@@ -14,9 +14,11 @@ * limitations under the License. */ +#include "perfetto/base/build_config.h" #include "perfetto/ext/base/file_utils.h" #include "perfetto/ext/base/scoped_file.h" #include "perfetto/ext/base/utils.h" +#include "perfetto/protozero/scattered_heap_buffer.h" #include "perfetto/trace_processor/read_trace.h" #include "src/base/test/utils.h" @@ -25,6 +27,10 @@ #include "protos/perfetto/trace/trace.pbzero.h" #include "protos/perfetto/trace/trace_packet.pbzero.h" +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +#include <zstd.h> +#endif + namespace perfetto { namespace trace_processor { namespace { @@ -132,6 +138,65 @@ ASSERT_EQ(packet_count, 2412u); } +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +// End-to-end check of the zstd decode path: build the exact blob the tracing +// service's ZstdCompressFn emits (a zstd frame whose plaintext is a sequence of +// field-1-framed TracePackets) inside TracePacket.compressed_packets, and +// assert DecompressTrace reconstructs the original packets via the zstd +// decompressor. +TEST_F(ReadTraceIntegrationTest, ZstdCompressedPackets) { + // The framed packet stream is itself a valid Trace proto. + constexpr uint64_t kTimestamps[] = {1000, 2000, 3000}; + protozero::HeapBuffered<protos::pbzero::Trace> inner; + for (uint64_t ts : kTimestamps) { + inner->add_packet()->set_timestamp(ts); + } + std::vector<uint8_t> inner_bytes = inner.SerializeAsArray(); + + size_t bound = ZSTD_compressBound(inner_bytes.size()); + std::vector<uint8_t> frame(bound); + size_t frame_size = ZSTD_compress(frame.data(), bound, inner_bytes.data(), + inner_bytes.size(), /*level=*/3); + ASSERT_FALSE(ZSTD_isError(frame_size)); + frame.resize(frame_size); + ASSERT_EQ(frame[0], + 0x28u); // zstd magic, so DetectCompressedPackets -> kZstd. + + protozero::HeapBuffered<protos::pbzero::Trace> outer; + outer->add_packet()->set_compressed_packets(frame.data(), frame.size()); + std::vector<uint8_t> outer_bytes = outer.SerializeAsArray(); + + std::vector<uint8_t> decompressed; + base::Status status = trace_processor::DecompressTrace( + outer_bytes.data(), outer_bytes.size(), &decompressed); + ASSERT_TRUE(status.ok()) << status.message(); + + protos::pbzero::Trace::Decoder decoder(decompressed.data(), + decompressed.size()); + std::vector<uint64_t> got; + for (auto it = decoder.packet(); it; ++it) { + protos::pbzero::TracePacket::Decoder packet(*it); + ASSERT_FALSE(packet.has_compressed_packets()); + got.push_back(packet.timestamp()); + } + EXPECT_THAT(got, testing::ElementsAre(1000u, 2000u, 3000u)); +} + +// A corrupt zstd frame (valid magic, garbage body) must surface as an error, +// not a silent partial/empty decode. +TEST_F(ReadTraceIntegrationTest, ZstdCorruptCompressedPacketsFails) { + const uint8_t kGarbage[] = {0x28, 0xb5, 0x2f, 0xfd, 0x01, 0x02, 0x03, 0x04}; + protozero::HeapBuffered<protos::pbzero::Trace> outer; + outer->add_packet()->set_compressed_packets(kGarbage, sizeof(kGarbage)); + std::vector<uint8_t> outer_bytes = outer.SerializeAsArray(); + + std::vector<uint8_t> decompressed; + base::Status status = trace_processor::DecompressTrace( + outer_bytes.data(), outer_bytes.size(), &decompressed); + ASSERT_FALSE(status.ok()); +} +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + } // namespace } // namespace trace_processor } // namespace perfetto
diff --git a/src/trace_processor/trace_processor_impl.cc b/src/trace_processor/trace_processor_impl.cc index 1c7778d..9d98589 100644 --- a/src/trace_processor/trace_processor_impl.cc +++ b/src/trace_processor/trace_processor_impl.cc
@@ -53,7 +53,7 @@ #include "src/trace_processor/importers/android_bugreport/android_dumpstate_reader.h" #include "src/trace_processor/importers/android_bugreport/android_log_event_parser.h" #include "src/trace_processor/importers/android_bugreport/android_log_reader.h" -#include "src/trace_processor/importers/archive/gzip_trace_parser.h" +#include "src/trace_processor/importers/archive/decompressing_trace_reader.h" #include "src/trace_processor/importers/archive/tar_trace_reader.h" #include "src/trace_processor/importers/archive/zip_trace_reader.h" #include "src/trace_processor/importers/art_hprof/art_hprof_parser.h" @@ -153,12 +153,13 @@ #include "src/trace_processor/trace_summary/trace_summary.descriptor.h" #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/util/descriptors.h" -#include "src/trace_processor/util/gzip_utils.h" +#include "src/trace_processor/util/gzip_decompressor.h" #include "src/trace_processor/util/protozero_to_json.h" #include "src/trace_processor/util/protozero_to_text.h" #include "src/trace_processor/util/sql_bundle.h" #include "src/trace_processor/util/sql_modules.h" #include "src/trace_processor/util/trace_type.h" +#include "src/trace_processor/util/zstd_decompressor.h" #include "protos/perfetto/trace/clock_snapshot.pbzero.h" #include "protos/perfetto/trace/perfetto/perfetto_metatrace.pbzero.h" @@ -448,12 +449,19 @@ kInstrumentsXmlTraceType); #endif if constexpr (util::IsGzipSupported()) { - context()->reader_registry->RegisterTraceReader<GzipTraceParser>( - kGzipTraceType); - context()->reader_registry->RegisterTraceReader<GzipTraceParser>( - kCtraceTraceType); + context() + ->reader_registry->RegisterTraceReaderWithArg<DecompressingTraceReader>( + kGzipTraceType, util::CompressionType::kGzip); + context() + ->reader_registry->RegisterTraceReaderWithArg<DecompressingTraceReader>( + kCtraceTraceType, util::CompressionType::kGzip); context()->reader_registry->RegisterTraceReader<ZipTraceReader>(kZipFile); } + if constexpr (util::IsZstdSupported()) { + context() + ->reader_registry->RegisterTraceReaderWithArg<DecompressingTraceReader>( + kZstdTraceType, util::CompressionType::kZstd); + } context()->reader_registry->RegisterTraceReader<JsonTraceTokenizer>( kJsonTraceType); context()
diff --git a/src/trace_processor/trace_reader_registry.cc b/src/trace_processor/trace_reader_registry.cc index 0feffe8..3996bda 100644 --- a/src/trace_processor/trace_reader_registry.cc +++ b/src/trace_processor/trace_reader_registry.cc
@@ -24,13 +24,20 @@ #include "perfetto/ext/base/status_or.h" #include "src/trace_processor/importers/common/chunked_trace_reader.h" #include "src/trace_processor/types/trace_processor_context.h" -#include "src/trace_processor/util/gzip_utils.h" +#include "src/trace_processor/util/gzip_decompressor.h" #include "src/trace_processor/util/trace_type.h" +#include "src/trace_processor/util/zstd_decompressor.h" namespace perfetto::trace_processor { namespace { const char kNoZlibErr[] = "Cannot open compressed trace. zlib not enabled in the build config"; +const char kNoZstdErr[] = + "Cannot open compressed trace. zstd not enabled in the build config"; + +bool RequiresZstdSupport(TraceType type) { + return type == kZstdTraceType; +} bool RequiresZlibSupport(TraceType type) { switch (type) { @@ -40,6 +47,7 @@ case kZipFile: return true; + case kZstdTraceType: case kCollapsedStackTraceType: case kNinjaLogTraceType: case kSystraceTraceType: @@ -95,6 +103,11 @@ TraceTypeToString(type), kNoZlibErr); } + if (RequiresZstdSupport(type) && !util::IsZstdSupported()) { + return base::ErrStatus("%s support is disabled. %s", + TraceTypeToString(type), kNoZstdErr); + } + return base::ErrStatus("%s support is disabled", TraceTypeToString(type)); }
diff --git a/src/trace_processor/trace_reader_registry.h b/src/trace_processor/trace_reader_registry.h index 2e18ac3..89eb407 100644 --- a/src/trace_processor/trace_reader_registry.h +++ b/src/trace_processor/trace_reader_registry.h
@@ -57,6 +57,15 @@ }); } + // Like RegisterTraceReader, but passes a fixed extra argument to the reader's + // constructor (e.g. the compression codec for DecompressingTraceReader). + template <typename Reader, typename Arg> + void RegisterTraceReaderWithArg(TraceType trace_type, Arg arg) { + RegisterFactory(trace_type, [arg](TraceProcessorContext* ctxt, uint32_t) { + return std::make_unique<Reader>(ctxt, arg); + }); + } + // Registers a trace reader factory that captures its own state (e.g. a // plugin's `this` pointer). The TraceProcessorContext* passed at creation // time is ignored by the wrapper.
diff --git a/src/trace_processor/util/BUILD.gn b/src/trace_processor/util/BUILD.gn index 1674cab..c0324cb 100644 --- a/src/trace_processor/util/BUILD.gn +++ b/src/trace_processor/util/BUILD.gn
@@ -60,22 +60,70 @@ ] } -source_set("gzip") { +# Codec-agnostic streaming decompressor interface + codec detection. Dep-free +# (no zlib/zstd) so low-level utils can sniff/declare without pulling a codec. +source_set("stream_decompressor") { sources = [ - "gzip_utils.cc", - "gzip_utils.h", + "stream_decompressor.cc", + "stream_decompressor.h", ] deps = [ "../../../gn:default_deps", "../../../include/perfetto/base", ] +} - # gzip_utils optionally depends on zlib. +source_set("gzip") { + sources = [ + "gzip_decompressor.cc", + "gzip_decompressor.h", + ] + public_deps = [ ":stream_decompressor" ] + deps = [ + "../../../gn:default_deps", + "../../../include/perfetto/base", + ] + + # gzip_decompressor optionally depends on zlib. if (enable_perfetto_zlib) { deps += [ "../../../gn:zlib" ] } } +source_set("zstd") { + sources = [ + "zstd_decompressor.cc", + "zstd_decompressor.h", + ] + public_deps = [ ":stream_decompressor" ] + deps = [ + "../../../gn:default_deps", + "../../../include/perfetto/base", + ] + + # zstd_decompressor optionally depends on zstd. + if (enable_perfetto_zstd) { + deps += [ "../../../gn:zstd" ] + } +} + +# The codec-agnostic decompression entry point: IsCompressionSupported(), +# CreateDecompressor() and DecompressFully(). Depend on this (rather +# than :gzip/:zstd directly) to decompress without naming a codec. +source_set("decompress") { + sources = [ + "decompress.cc", + "decompress.h", + ] + public_deps = [ ":stream_decompressor" ] + deps = [ + ":gzip", + ":zstd", + "../../../gn:default_deps", + "../../../include/perfetto/base", + ] +} + perfetto_component("build_id") { sources = [ "build_id.cc", @@ -503,9 +551,17 @@ ] } if (enable_perfetto_zlib) { - sources += [ "gzip_utils_unittest.cc" ] + sources += [ "gzip_decompressor_unittest.cc" ] deps += [ "../../../gn:zlib" ] } + if (enable_perfetto_zstd) { + sources += [ "zstd_decompressor_unittest.cc" ] + deps += [ + ":decompress", + ":zstd", + "../../../gn:zstd", + ] + } if (enable_perfetto_trace_processor_sqlite) { sources += [ "sql_module_doc_parser_unittest.cc" ] deps += [ ":sql_module_doc_parser" ]
diff --git a/src/trace_processor/util/decompress.cc b/src/trace_processor/util/decompress.cc new file mode 100644 index 0000000..b3339c7 --- /dev/null +++ b/src/trace_processor/util/decompress.cc
@@ -0,0 +1,77 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/util/decompress.h" + +#include <memory> + +#include "src/trace_processor/util/gzip_decompressor.h" +#include "src/trace_processor/util/stream_decompressor.h" +#include "src/trace_processor/util/zstd_decompressor.h" + +namespace perfetto::trace_processor::util { + +bool IsCompressionSupported(CompressionType type) { + switch (type) { + case CompressionType::kNone: + return true; + case CompressionType::kGzip: + case CompressionType::kRawDeflate: + return IsGzipSupported(); + case CompressionType::kZstd: + return IsZstdSupported(); + } + return false; +} + +std::unique_ptr<StreamDecompressor> CreateDecompressor(CompressionType type) { + if (!IsCompressionSupported(type)) { + return nullptr; + } + switch (type) { + case CompressionType::kNone: + return nullptr; + case CompressionType::kGzip: + return std::make_unique<GzipDecompressor>( + GzipDecompressor::InputMode::kGzip); + case CompressionType::kRawDeflate: + return std::make_unique<GzipDecompressor>( + GzipDecompressor::InputMode::kRawDeflate); + case CompressionType::kZstd: + return std::make_unique<ZstdDecompressor>(); + } + return nullptr; +} + +std::vector<uint8_t> DecompressFully(CompressionType type, + const uint8_t* data, + size_t len) { + std::unique_ptr<StreamDecompressor> decompressor = CreateDecompressor(type); + if (!decompressor) { + return {}; + } + // Returns whatever was extracted, without requiring the stream to end cleanly + // at kEof; callers that care detect failure via the returned size or + // downstream parsing. + std::vector<uint8_t> output; + decompressor->FeedAndExtract(data, len, + [&output](const uint8_t* ptr, size_t size) { + output.insert(output.end(), ptr, ptr + size); + }); + return output; +} + +} // namespace perfetto::trace_processor::util
diff --git a/src/trace_processor/util/decompress.h b/src/trace_processor/util/decompress.h new file mode 100644 index 0000000..feed66e --- /dev/null +++ b/src/trace_processor/util/decompress.h
@@ -0,0 +1,52 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_UTIL_DECOMPRESS_H_ +#define SRC_TRACE_PROCESSOR_UTIL_DECOMPRESS_H_ + +#include <cstddef> +#include <cstdint> +#include <memory> +#include <vector> + +#include "src/trace_processor/util/stream_decompressor.h" + +// Codec-agnostic entry point for decompression. Call sites sniff the codec with +// DetectCompression() (stream_decompressor.h), check IsCompressionSupported(), +// then either DecompressFully() for one-shot use or CreateDecompressor() for +// streaming. None of them name a concrete codec. + +namespace perfetto::trace_processor::util { + +// Whether this build was compiled with the library needed to decompress `type`. +// kNone is trivially "supported"; unsupported codecs make CreateDecompressor() +// return nullptr and DecompressFully() return an empty vector. +bool IsCompressionSupported(CompressionType type); + +// Creates a streaming decompressor for `type`, or nullptr if `type` is kNone or +// the build wasn't compiled with support for it (see IsCompressionSupported). +std::unique_ptr<StreamDecompressor> CreateDecompressor(CompressionType type); + +// Decompress an entire in-memory block at once. Returns the decompressed bytes, +// or an empty vector if `type` is unsupported or the input is corrupt. For +// large or streamed inputs use CreateDecompressor() instead. +std::vector<uint8_t> DecompressFully(CompressionType type, + const uint8_t* data, + size_t len); + +} // namespace perfetto::trace_processor::util + +#endif // SRC_TRACE_PROCESSOR_UTIL_DECOMPRESS_H_
diff --git a/src/trace_processor/util/gzip_utils.cc b/src/trace_processor/util/gzip_decompressor.cc similarity index 81% rename from src/trace_processor/util/gzip_utils.cc rename to src/trace_processor/util/gzip_decompressor.cc index 50ea9db..e9b505e 100644 --- a/src/trace_processor/util/gzip_utils.cc +++ b/src/trace_processor/util/gzip_decompressor.cc
@@ -14,12 +14,11 @@ * limitations under the License. */ -#include "src/trace_processor/util/gzip_utils.h" +#include "src/trace_processor/util/gzip_decompressor.h" #include <cstddef> #include <cstdint> #include <memory> -#include <vector> #include "perfetto/base/build_config.h" @@ -50,9 +49,7 @@ } void GzipDecompressor::Feed(const uint8_t* data, size_t size) { - // This const_cast is not harmfull as zlib will not modify the data in this - // pointer. This is only necessary because of the build flags we use to be - // compatible with other embedders. + // zlib won't modify the input, so casting away const on next_in is safe. z_stream_->next_in = const_cast<uint8_t*>(data); z_stream_->avail_in = static_cast<uInt>(size); } @@ -106,16 +103,4 @@ #endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB) -// static -std::vector<uint8_t> GzipDecompressor::DecompressFully(const uint8_t* data, - size_t len) { - std::vector<uint8_t> whole_data; - GzipDecompressor decompressor; - auto decom_output_consumer = [&](const uint8_t* buf, size_t buf_len) { - whole_data.insert(whole_data.end(), buf, buf + buf_len); - }; - decompressor.FeedAndExtract(data, len, decom_output_consumer); - return whole_data; -} - } // namespace perfetto::trace_processor::util
diff --git a/src/trace_processor/util/gzip_decompressor.h b/src/trace_processor/util/gzip_decompressor.h new file mode 100644 index 0000000..2e8fd81 --- /dev/null +++ b/src/trace_processor/util/gzip_decompressor.h
@@ -0,0 +1,74 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_UTIL_GZIP_DECOMPRESSOR_H_ +#define SRC_TRACE_PROCESSOR_UTIL_GZIP_DECOMPRESSOR_H_ + +#include <cstddef> +#include <cstdint> +#include <memory> + +#include "perfetto/base/build_config.h" +#include "src/trace_processor/util/stream_decompressor.h" + +struct z_stream_s; + +namespace perfetto::trace_processor::util { + +// Returns whether gzip related functionality is supported with the current +// build flags. +constexpr bool IsGzipSupported() { +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) + return true; +#else + return false; +#endif +} + +// gzip/deflate StreamDecompressor. Prefer obtaining one through +// CreateDecompressor(); construct directly only when you need to pin +// the InputMode (e.g. raw deflate for zip entries). See stream_decompressor.h +// for the streaming usage contract. +class GzipDecompressor : public StreamDecompressor { + public: + enum class InputMode { + // The input stream contains a gzip header. This is for the common case of + // decompressing .gz files. + kGzip = 0, + + // A raw deflate stream. This is for the case of uncompressing files from + // a .zip archive, where the compression type is specified in the zip file + // entry, rather than in the stream header. + kRawDeflate = 1, + }; + + explicit GzipDecompressor(InputMode = InputMode::kGzip); + + void Feed(const uint8_t* data, size_t size) override; + Result ExtractOutput(uint8_t* out, size_t out_capacity) override; + void Reset() override; + size_t AvailIn() const override; + + private: + struct Deleter { + void operator()(z_stream_s*) const; + }; + std::unique_ptr<z_stream_s, Deleter> z_stream_; +}; + +} // namespace perfetto::trace_processor::util + +#endif // SRC_TRACE_PROCESSOR_UTIL_GZIP_DECOMPRESSOR_H_
diff --git a/src/trace_processor/util/gzip_utils_unittest.cc b/src/trace_processor/util/gzip_decompressor_unittest.cc similarity index 98% rename from src/trace_processor/util/gzip_utils_unittest.cc rename to src/trace_processor/util/gzip_decompressor_unittest.cc index fa4e752..43911da 100644 --- a/src/trace_processor/util/gzip_utils_unittest.cc +++ b/src/trace_processor/util/gzip_decompressor_unittest.cc
@@ -16,7 +16,7 @@ #include "test/gtest_and_gmock.h" -#include "src/trace_processor/util/gzip_utils.h" +#include "src/trace_processor/util/gzip_decompressor.h" #include <zconf.h> #include <zlib.h>
diff --git a/src/trace_processor/util/gzip_utils.h b/src/trace_processor/util/gzip_utils.h deleted file mode 100644 index c13ace4..0000000 --- a/src/trace_processor/util/gzip_utils.h +++ /dev/null
@@ -1,141 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SRC_TRACE_PROCESSOR_UTIL_GZIP_UTILS_H_ -#define SRC_TRACE_PROCESSOR_UTIL_GZIP_UTILS_H_ - -#include <cstddef> -#include <cstdint> -#include <memory> -#include <vector> - -#include "perfetto/base/build_config.h" - -struct z_stream_s; - -namespace perfetto::trace_processor::util { - -// Returns whether gzip related functioanlity is supported with the current -// build flags. -constexpr bool IsGzipSupported() { -#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) - return true; -#else - return false; -#endif -} - -// Usage: To decompress in a streaming way, there are two ways of using it: -// 1. [Commonly used] - Feed the sequence of mem-blocks in 'FeedAndExtract' one -// by one. Output will be produced in given output_consumer, which is simply -// a callback. On each 'FeedAndExtract', output_consumer could get invoked -// any number of times, based on how much partial output is available. - -// 2. [Uncommon ; Discouraged] - Feed the sequence of mem-blocks one by one, by -// calling 'Feed'. For each time 'Feed' is called, client should call -// 'ExtractOutput' again and again to extract the partially available output, -// until there in no more output to extract. Also see 'ResultCode' enum. -class GzipDecompressor { - public: - enum class ResultCode { - // 'kOk' means nothing bad happened so far, but continue doing what you - // were doing. - kOk, - // While calling 'ExtractOutput' repeatedly, if we get 'kEof', it means - // we have extracted all the partially available data and we are also - // done, i.e. there is no need to feed more input. - kEof, - // Some error. Possibly invalid compressed stream or corrupted data. - kError, - // While calling 'ExtractOutput' repeatedly, if we get 'kNeedsMoreInput', - // it means we have extracted all the partially available data, but we are - // not done yet. We need to call the 'Feed' to feed the next input - // mem-block and go through the ExtractOutput loop again. - kNeedsMoreInput, - }; - struct Result { - // The return code of the decompression. - ResultCode ret; - - // The amount of bytes written to output. - // Valid in all cases except |ResultCode::kError|. - size_t bytes_written; - }; - enum class InputMode { - // The input stream contains a gzip header. This is for the common case of - // decompressing .gz files. - kGzip = 0, - - // A raw deflate stream. This is for the case of uncompressing files from - // a .zip archive, where the compression type is specified in the zip file - // entry, rather than in the stream header. - kRawDeflate = 1, - }; - - explicit GzipDecompressor(InputMode = InputMode::kGzip); - - // Feed the next mem-block. - void Feed(const uint8_t* data, size_t size); - - // Feed the next mem-block and extract output in the callback consumer. - // callback can get invoked multiple times if there are multiple - // mem-blocks to output. - // - // Note the output of this function is guaranteed *not* to be kOk. - template <typename Callback = void(const uint8_t* ptr, size_t size)> - ResultCode FeedAndExtract(const uint8_t* data, - size_t size, - const Callback& output_consumer) { - Feed(data, size); - uint8_t buffer[4096]; - Result result; - do { - result = ExtractOutput(buffer, sizeof(buffer)); - if (result.ret != ResultCode::kError && result.bytes_written > 0) { - output_consumer(buffer, result.bytes_written); - } - } while (result.ret == ResultCode::kOk); - return result.ret; - } - - // Extract the newly available partial output. On each 'Feed', this method - // should be called repeatedly until there is no more data to output - // i.e. (either 'kEof' or 'kNeedsMoreInput'). - Result ExtractOutput(uint8_t* out, size_t out_capacity); - - // Sets the state of the decompressor to reuse with other gzip streams. - // This is almost like constructing a new 'GzipDecompressor' object - // but without paying the cost of internal memory allocation. - void Reset(); - - // Decompress the entire mem-block and return decompressed mem-block. - // This is used for decompressing small strings or small files - // which doesn't require streaming decompression. - static std::vector<uint8_t> DecompressFully(const uint8_t* data, size_t len); - - // Returns the amount of input bytes left unprocessed. - size_t AvailIn() const; - - private: - struct Deleter { - void operator()(z_stream_s*) const; - }; - std::unique_ptr<z_stream_s, Deleter> z_stream_; -}; - -} // namespace perfetto::trace_processor::util - -#endif // SRC_TRACE_PROCESSOR_UTIL_GZIP_UTILS_H_
diff --git a/src/trace_processor/util/stream_decompressor.cc b/src/trace_processor/util/stream_decompressor.cc new file mode 100644 index 0000000..1471126 --- /dev/null +++ b/src/trace_processor/util/stream_decompressor.cc
@@ -0,0 +1,44 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/util/stream_decompressor.h" + +#include <cstring> + +namespace perfetto::trace_processor::util { + +// Out-of-line to anchor the vtable to a single translation unit. +StreamDecompressor::~StreamDecompressor() = default; + +CompressionType DetectCompression(const uint8_t* data, size_t size) { + if (size >= sizeof(kZstdMagic) && + memcmp(data, kZstdMagic, sizeof(kZstdMagic)) == 0) { + return CompressionType::kZstd; + } + if (size >= sizeof(kGzipMagic) && + memcmp(data, kGzipMagic, sizeof(kGzipMagic)) == 0) { + return CompressionType::kGzip; + } + return CompressionType::kNone; +} + +CompressionType DetectPacketCompression(const uint8_t* data, size_t size) { + return DetectCompression(data, size) == CompressionType::kZstd + ? CompressionType::kZstd + : CompressionType::kGzip; +} + +} // namespace perfetto::trace_processor::util
diff --git a/src/trace_processor/util/stream_decompressor.h b/src/trace_processor/util/stream_decompressor.h new file mode 100644 index 0000000..6f8f2c1 --- /dev/null +++ b/src/trace_processor/util/stream_decompressor.h
@@ -0,0 +1,129 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_UTIL_STREAM_DECOMPRESSOR_H_ +#define SRC_TRACE_PROCESSOR_UTIL_STREAM_DECOMPRESSOR_H_ + +#include <cstddef> +#include <cstdint> + +namespace perfetto::trace_processor::util { + +// The compression codecs trace_processor can detect and decompress. To add one, +// give it a magic in DetectCompression() and a case in the decompress.h +// factory. +enum class CompressionType { + // Not compressed, or a header we don't recognize. + kNone, + // gzip-framed deflate (e.g. a .gz file). Magic bytes 0x1f 0x8b. + kGzip, + // Headerless deflate, as stored in .zip entries. Has no magic of its own, so + // DetectCompression() never returns it; callers that know they are looking at + // a zip entry pass it explicitly. + kRawDeflate, + // A zstd frame. Magic bytes 0x28 0xb5 0x2f 0xfd. + kZstd, +}; + +// Leading bytes that identify a self-describing codec from a stream header. +inline constexpr uint8_t kGzipMagic[] = {0x1f, 0x8b}; +inline constexpr uint8_t kZstdMagic[] = {0x28, 0xb5, 0x2f, 0xfd}; + +// Sniffs the leading bytes of `data` and returns the codec, or kNone if the +// header matches no known codec (or `size` is too small to tell). Never returns +// kRawDeflate (raw deflate is not self-describing). +CompressionType DetectCompression(const uint8_t* data, size_t size); + +// Picks the codec for an in-trace `compressed_packets` blob. These carry no +// codec tag: a zstd blob starts with the zstd magic, a deflate blob is +// zlib-wrapped and has none. Returns kZstd on a zstd magic, else kGzip (whose +// auto-detect handles zlib- and gzip-wrapped deflate). +CompressionType DetectPacketCompression(const uint8_t* data, size_t size); + +// Codec-agnostic streaming decompressor. Concrete codecs (gzip, zstd, ...) +// subclass this; obtain one via CreateDecompressor() in decompress.h so call +// sites never branch on the codec. +// +// Streaming usage, two ways: +// 1. [Common] Feed each input block to FeedAndExtract(); output is delivered to +// the callback, possibly several times per call. +// 2. [Low-level] Call Feed() once, then ExtractOutput() repeatedly until it +// returns kEof or kNeedsMoreInput. See ResultCode. +class StreamDecompressor { + public: + enum class ResultCode { + // Made progress; keep calling ExtractOutput to drain more output. + kOk, + // The stream is fully decompressed; no more input is needed. + kEof, + // Corrupt/invalid input. + kError, + // All available input was consumed but the stream is not complete; feed the + // next mem-block and continue. + kNeedsMoreInput, + }; + struct Result { + ResultCode ret; + // Bytes written to output. Valid in all cases except |ResultCode::kError|. + size_t bytes_written; + }; + + StreamDecompressor() = default; + virtual ~StreamDecompressor(); + + // Hands out / holds internal pointers; never copy or move. + StreamDecompressor(const StreamDecompressor&) = delete; + StreamDecompressor& operator=(const StreamDecompressor&) = delete; + StreamDecompressor(StreamDecompressor&&) = delete; + StreamDecompressor& operator=(StreamDecompressor&&) = delete; + + // Feed the next input mem-block. + virtual void Feed(const uint8_t* data, size_t size) = 0; + + // Extract the newly available partial output. After each Feed(), call this + // repeatedly until it returns kEof or kNeedsMoreInput. + virtual Result ExtractOutput(uint8_t* out, size_t out_capacity) = 0; + + // Reset to decode the next stream/frame, reusing internal buffers. Any fed + // but unconsumed input is preserved (so multi-stream inputs can continue). + virtual void Reset() = 0; + + // The amount of input bytes left unprocessed. + virtual size_t AvailIn() const = 0; + + // Feed the next mem-block and push all resulting output to `output_consumer`, + // which may be invoked any number of times. The returned code is guaranteed + // not to be kOk. + template <typename Callback> + ResultCode FeedAndExtract(const uint8_t* data, + size_t size, + const Callback& output_consumer) { + Feed(data, size); + uint8_t buffer[4096]; + Result result; + do { + result = ExtractOutput(buffer, sizeof(buffer)); + if (result.ret != ResultCode::kError && result.bytes_written > 0) { + output_consumer(buffer, result.bytes_written); + } + } while (result.ret == ResultCode::kOk); + return result.ret; + } +}; + +} // namespace perfetto::trace_processor::util + +#endif // SRC_TRACE_PROCESSOR_UTIL_STREAM_DECOMPRESSOR_H_
diff --git a/src/trace_processor/util/trace_type.cc b/src/trace_processor/util/trace_type.cc index 27bde2e..ea1bb72 100644 --- a/src/trace_processor/util/trace_type.cc +++ b/src/trace_processor/util/trace_type.cc
@@ -41,6 +41,9 @@ constexpr char kPerfMagic[] = {'P', 'E', 'R', 'F', 'I', 'L', 'E', '2'}; constexpr char kZipMagic[] = {'P', 'K', '\x03', '\x04'}; constexpr char kGzipMagic[] = {'\x1f', '\x8b'}; +// zstd frame magic: 0xFD2FB528 in little-endian order. Ref: +// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md +constexpr char kZstdMagic[] = {'\x28', '\xb5', '\x2f', '\xfd'}; constexpr char kArtMethodStreamingMagic[] = {'S', 'L', 'O', 'W'}; constexpr char kArtHprofStreamingMagic[] = {'J', 'A', 'V', 'A', ' ', 'P', 'R', 'O', 'F', 'I', 'L', 'E'}; @@ -264,6 +267,8 @@ return "systrace"; case kGzipTraceType: return "gzip"; + case kZstdTraceType: + return "zstd"; case kCtraceTraceType: return "ctrace"; case kZipFile: @@ -309,6 +314,7 @@ bool IsContainerTraceType(TraceType trace_type) { switch (trace_type) { case kGzipTraceType: + case kZstdTraceType: case kCtraceTraceType: case kZipFile: case kAndroidBugreportTraceType: @@ -372,6 +378,10 @@ return kGzipTraceType; } + if (MatchesMagic(data, size, kZstdMagic)) { + return kZstdTraceType; + } + if (MatchesMagic(data, size, kArtMethodStreamingMagic)) { if (size >= 6) { uint16_t version = data[4] | static_cast<uint16_t>(data[5] << 8);
diff --git a/src/trace_processor/util/trace_type.h b/src/trace_processor/util/trace_type.h index a9e7d37..d37e935 100644 --- a/src/trace_processor/util/trace_type.h +++ b/src/trace_processor/util/trace_type.h
@@ -30,6 +30,7 @@ kCtraceTraceType, kFuchsiaTraceType, kGzipTraceType, + kZstdTraceType, kJsonTraceType, kNinjaLogTraceType, kPerfDataTraceType,
diff --git a/src/trace_processor/util/zip_reader.cc b/src/trace_processor/util/zip_reader.cc index b1aa7f6..ae1705e 100644 --- a/src/trace_processor/util/zip_reader.cc +++ b/src/trace_processor/util/zip_reader.cc
@@ -34,7 +34,7 @@ #include "perfetto/ext/base/string_view.h" #include "perfetto/ext/base/utils.h" #include "perfetto/trace_processor/trace_blob_view.h" -#include "src/trace_processor/util/gzip_utils.h" +#include "src/trace_processor/util/gzip_decompressor.h" #include "src/trace_processor/util/streaming_line_reader.h" #if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) @@ -294,16 +294,11 @@ auto end = reader_.end_offset(); auto slice = reader_.SliceOff(start, end - start); PERFETTO_CHECK(slice); - auto res_code = cur_.decompressor.FeedAndExtract(slice->data(), slice->size(), - [](const uint8_t*, size_t) { - // Intentionally do - // nothing: we are only - // looking for the bounds - // of the deflate stream, - // we are not actually - // interested in the - // output. - }); + auto res_code = cur_.decompressor->FeedAndExtract( + slice->data(), slice->size(), [](const uint8_t*, size_t) { + // Intentionally do nothing: we only want the bounds of the deflate + // stream, not its output. + }); switch (res_code) { case GzipDecompressor::ResultCode::kNeedsMoreInput: cur_.decompressor_bytes_fed += slice->size(); @@ -317,7 +312,7 @@ case GzipDecompressor::ResultCode::kEof: break; } - cur_.decompressor_bytes_fed += slice->size() - cur_.decompressor.AvailIn(); + cur_.decompressor_bytes_fed += slice->size() - cur_.decompressor->AvailIn(); auto raw_compressed = reader_.SliceOff(reader_.start_offset(), cur_.decompressor_bytes_fed); PERFETTO_CHECK(raw_compressed);
diff --git a/src/trace_processor/util/zip_reader.h b/src/trace_processor/util/zip_reader.h index f325947..8e54dee 100644 --- a/src/trace_processor/util/zip_reader.h +++ b/src/trace_processor/util/zip_reader.h
@@ -20,6 +20,7 @@ #include <cstddef> #include <cstdint> #include <functional> +#include <memory> #include <optional> #include <string> #include <utility> @@ -29,7 +30,7 @@ #include "perfetto/ext/base/status_or.h" #include "perfetto/ext/base/string_view.h" #include "perfetto/trace_processor/trace_blob_view.h" -#include "src/trace_processor/util/gzip_utils.h" +#include "src/trace_processor/util/gzip_decompressor.h" #include "src/trace_processor/util/trace_blob_view_reader.h" // ZipReader allows to read Zip files in a streaming fashion. @@ -175,7 +176,12 @@ // Used to track the number of bytes fed into the decompressor when we don't // know the compressed size upfront. size_t decompressor_bytes_fed = 0; - GzipDecompressor decompressor{GzipDecompressor::InputMode::kRawDeflate}; + // Held by pointer (not value) so FileParseState stays movable for the + // `cur_ = FileParseState()` reset; StreamDecompressor is deliberately not + // movable. Zip entries are always raw deflate, hence the fixed mode. + std::unique_ptr<GzipDecompressor> decompressor = + std::make_unique<GzipDecompressor>( + GzipDecompressor::InputMode::kRawDeflate); std::optional<TraceBlobView> compressed; ZipFile::Header hdr{}; };
diff --git a/src/trace_processor/util/zstd_decompressor.cc b/src/trace_processor/util/zstd_decompressor.cc new file mode 100644 index 0000000..8cd72a2 --- /dev/null +++ b/src/trace_processor/util/zstd_decompressor.cc
@@ -0,0 +1,113 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/util/zstd_decompressor.h" + +#include <cstddef> +#include <cstdint> +#include <memory> + +#include "perfetto/base/build_config.h" +#include "perfetto/base/logging.h" + +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +#include <zstd.h> +#else +struct ZSTD_DCtx_s {}; +#endif + +namespace perfetto::trace_processor::util { + +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) // Real Implementation + +ZstdDecompressor::ZstdDecompressor() : dstream_(ZSTD_createDStream()) { + PERFETTO_CHECK(dstream_); + PERFETTO_CHECK(!ZSTD_isError(ZSTD_initDStream(dstream_.get()))); +} + +void ZstdDecompressor::Reset() { + // Only reset the session: keep the (possibly partially consumed) input so a + // multi-frame stream can continue decoding the next frame in place. + ZSTD_DCtx_reset(dstream_.get(), ZSTD_reset_session_only); + eof_ = false; +} + +void ZstdDecompressor::Feed(const uint8_t* data, size_t size) { + in_data_ = data; + in_size_ = size; + in_pos_ = 0; + // Clear any prior EOF so a fresh stream fed after a completed one decodes. + // Reset() clears it too, for the multi-frame-in-place case. + eof_ = false; +} + +ZstdDecompressor::Result ZstdDecompressor::ExtractOutput(uint8_t* out, + size_t out_size) { + if (eof_) + return Result{ResultCode::kEof, 0}; + + ZSTD_outBuffer out_buf = {out, out_size, 0}; + ZSTD_inBuffer in_buf = {in_data_, in_size_, in_pos_}; + const size_t prev_in_pos = in_pos_; + // Note: we intentionally do not early-return when the input is exhausted. + // zstd can hold a decompressed block in an internal buffer when the caller's + // output buffer is smaller than the block, and that data must be flushed with + // further calls even with no new input. + size_t ret = ZSTD_decompressStream(dstream_.get(), &out_buf, &in_buf); + in_pos_ = in_buf.pos; + + if (ZSTD_isError(ret)) + return Result{ResultCode::kError, 0}; + if (ret == 0) { + eof_ = true; + return Result{ResultCode::kEof, out_buf.pos}; + } + // ret > 0: the frame is not complete yet. + if (out_buf.pos > 0) + return Result{ResultCode::kOk, + out_buf.pos}; // Produced output; keep going. + // No output this call: keep going only if we made input progress, otherwise + // we genuinely need more input. This keeps the invariant that + // kNeedsMoreInput always reports zero bytes written. + if (in_pos_ > prev_in_pos) + return Result{ResultCode::kOk, 0}; + return Result{ResultCode::kNeedsMoreInput, 0}; +} + +size_t ZstdDecompressor::AvailIn() const { + return in_size_ - in_pos_; +} + +void ZstdDecompressor::Deleter::operator()(ZSTD_DCtx_s* stream) const { + ZSTD_freeDStream(stream); +} + +#else // Dummy Implementation + +ZstdDecompressor::ZstdDecompressor() = default; +void ZstdDecompressor::Reset() {} +void ZstdDecompressor::Feed(const uint8_t*, size_t) {} +ZstdDecompressor::Result ZstdDecompressor::ExtractOutput(uint8_t*, size_t) { + return Result{ResultCode::kError, 0}; +} +size_t ZstdDecompressor::AvailIn() const { + return 0; +} +void ZstdDecompressor::Deleter::operator()(ZSTD_DCtx_s*) const {} + +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + +} // namespace perfetto::trace_processor::util
diff --git a/src/trace_processor/util/zstd_decompressor.h b/src/trace_processor/util/zstd_decompressor.h new file mode 100644 index 0000000..ec7ed7e --- /dev/null +++ b/src/trace_processor/util/zstd_decompressor.h
@@ -0,0 +1,66 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_UTIL_ZSTD_DECOMPRESSOR_H_ +#define SRC_TRACE_PROCESSOR_UTIL_ZSTD_DECOMPRESSOR_H_ + +#include <cstddef> +#include <cstdint> +#include <memory> + +#include "perfetto/base/build_config.h" +#include "src/trace_processor/util/stream_decompressor.h" + +struct ZSTD_DCtx_s; + +namespace perfetto::trace_processor::util { + +// Returns whether zstd related functionality is supported with the current +// build flags. +constexpr bool IsZstdSupported() { +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + return true; +#else + return false; +#endif +} + +// zstd StreamDecompressor. Prefer obtaining one through +// CreateDecompressor(); see stream_decompressor.h for the streaming +// usage contract. +class ZstdDecompressor : public StreamDecompressor { + public: + ZstdDecompressor(); + + void Feed(const uint8_t* data, size_t size) override; + Result ExtractOutput(uint8_t* out, size_t out_capacity) override; + void Reset() override; + size_t AvailIn() const override; + + private: + struct Deleter { + void operator()(ZSTD_DCtx_s*) const; + }; + std::unique_ptr<ZSTD_DCtx_s, Deleter> dstream_; + const uint8_t* in_data_ = nullptr; + size_t in_size_ = 0; + size_t in_pos_ = 0; + bool eof_ = false; +}; + +} // namespace perfetto::trace_processor::util + +#endif // SRC_TRACE_PROCESSOR_UTIL_ZSTD_DECOMPRESSOR_H_
diff --git a/src/trace_processor/util/zstd_decompressor_unittest.cc b/src/trace_processor/util/zstd_decompressor_unittest.cc new file mode 100644 index 0000000..70c9d84 --- /dev/null +++ b/src/trace_processor/util/zstd_decompressor_unittest.cc
@@ -0,0 +1,125 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/util/zstd_decompressor.h" + +#include "src/trace_processor/util/decompress.h" + +#include <cstdint> +#include <random> +#include <string> +#include <vector> + +#include <zstd.h> + +#include "perfetto/base/logging.h" +#include "test/gtest_and_gmock.h" + +namespace perfetto::trace_processor::util { +namespace { + +std::vector<uint8_t> Compress(const std::string& input) { + size_t bound = ZSTD_compressBound(input.size()); + std::vector<uint8_t> out(bound); + size_t sz = ZSTD_compress(out.data(), out.size(), input.data(), input.size(), + /*level=*/3); + PERFETTO_CHECK(!ZSTD_isError(sz)); + out.resize(sz); + return out; +} + +std::string RandomString(size_t size) { + std::default_random_engine rnd(0); + std::uniform_int_distribution<> dist(0, 255); + std::string s(size, '\0'); + for (char& c : s) + c = static_cast<char>(dist(rnd)); + return s; +} + +TEST(ZstdUtilsTest, DecompressFullyRoundTrip) { + std::string input = "the quick brown fox jumps over the lazy dog"; + std::vector<uint8_t> compressed = Compress(input); + + std::vector<uint8_t> out = DecompressFully( + CompressionType::kZstd, compressed.data(), compressed.size()); + ASSERT_EQ(std::string(out.begin(), out.end()), input); +} + +TEST(ZstdUtilsTest, DecompressFullyEmpty) { + std::vector<uint8_t> compressed = Compress(""); + std::vector<uint8_t> out = DecompressFully( + CompressionType::kZstd, compressed.data(), compressed.size()); + EXPECT_TRUE(out.empty()); +} + +TEST(ZstdUtilsTest, DecompressFullyCorrupt) { + // Valid zstd magic followed by garbage. + std::vector<uint8_t> bad = {0x28, 0xB5, 0x2F, 0xFD, 0x01, 0x02, 0x03, 0x04}; + std::vector<uint8_t> out = + DecompressFully(CompressionType::kZstd, bad.data(), bad.size()); + EXPECT_TRUE(out.empty()); +} + +// Drives the streaming path with a tiny output buffer so the internal zstd +// block buffer must be flushed across multiple ExtractOutput() calls. This is +// the case the whole-file ZstdTraceParser relies on. +TEST(ZstdUtilsTest, StreamingSmallOutputBuffer) { + std::string input = RandomString(200 * 1024); + std::vector<uint8_t> compressed = Compress(input); + + ZstdDecompressor decompressor; + std::vector<uint8_t> out; + decompressor.Feed(compressed.data(), compressed.size()); + ZstdDecompressor::Result result; + do { + uint8_t buf[512]; + result = decompressor.ExtractOutput(buf, sizeof(buf)); + ASSERT_NE(result.ret, ZstdDecompressor::ResultCode::kError); + out.insert(out.end(), buf, buf + result.bytes_written); + } while (result.ret == ZstdDecompressor::ResultCode::kOk); + + EXPECT_EQ(result.ret, ZstdDecompressor::ResultCode::kEof); + ASSERT_EQ(out.size(), input.size()); + EXPECT_EQ(std::string(out.begin(), out.end()), input); +} + +TEST(ZstdUtilsTest, MultiFrame) { + std::vector<uint8_t> compressed = Compress("frame-one"); + std::vector<uint8_t> second = Compress("frame-two"); + compressed.insert(compressed.end(), second.begin(), second.end()); + + ZstdDecompressor decompressor; + decompressor.Feed(compressed.data(), compressed.size()); + std::string out; + for (;;) { + uint8_t buf[1024]; + auto result = decompressor.ExtractOutput(buf, sizeof(buf)); + ASSERT_NE(result.ret, ZstdDecompressor::ResultCode::kError); + out.append(reinterpret_cast<char*>(buf), result.bytes_written); + if (result.ret == ZstdDecompressor::ResultCode::kEof) { + decompressor.Reset(); + if (decompressor.AvailIn() == 0) + break; + } else if (result.ret == ZstdDecompressor::ResultCode::kNeedsMoreInput) { + break; + } + } + EXPECT_EQ(out, "frame-oneframe-two"); +} + +} // namespace +} // namespace perfetto::trace_processor::util
diff --git a/src/traceconv/BUILD.gn b/src/traceconv/BUILD.gn index 8dd9229..49e00ca 100644 --- a/src/traceconv/BUILD.gn +++ b/src/traceconv/BUILD.gn
@@ -86,8 +86,8 @@ "../protozero:proto_ring_buffer", "../trace_processor:lib", "../trace_processor:storage_minimal", + "../trace_processor/util:decompress", "../trace_processor/util:descriptors", - "../trace_processor/util:gzip", "../trace_processor/util:protozero_to_text", "../trace_processor/util:tar_writer", "../trace_processor/util:trace_type",
diff --git a/src/traceconv/trace_to_text.cc b/src/traceconv/trace_to_text.cc index 40ec2f2..cf098dc 100644 --- a/src/traceconv/trace_to_text.cc +++ b/src/traceconv/trace_to_text.cc
@@ -27,8 +27,8 @@ #include "protos/perfetto/trace/trace.pbzero.h" #include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "src/trace_processor/util/decompress.h" #include "src/trace_processor/util/descriptors.h" -#include "src/trace_processor/util/gzip_utils.h" #include "src/trace_processor/util/protozero_to_text.h" #include "src/trace_processor/util/trace_type.h" @@ -38,7 +38,7 @@ using perfetto::trace_processor::DescriptorPool; using trace_processor::TraceType; -using trace_processor::util::GzipDecompressor; +namespace util = trace_processor::util; template <size_t N> static void WriteToOutput(std::ostream* output, const char (&str)[N]) { @@ -87,9 +87,11 @@ void OnlineTraceToText::PrintCompressedPackets(protozero::ConstBytes packets) { WriteToOutput(output_, "compressed_packets {\n"); - if (trace_processor::util::IsGzipSupported()) { + util::CompressionType type = + util::DetectPacketCompression(packets.data, packets.size); + if (util::IsCompressionSupported(type)) { std::vector<uint8_t> whole_data = - GzipDecompressor::DecompressFully(packets.data, packets.size); + util::DecompressFully(type, packets.data, packets.size); protos::pbzero::Trace::Decoder decoder(whole_data.data(), whole_data.size()); for (auto it = decoder.packet(); it; ++it) { @@ -100,8 +102,8 @@ } } else { static const char kErrMsg[] = - "Cannot decode compressed packets. zlib not enabled in the build " - "config"; + "Cannot decode compressed packets: the codec is not enabled in the " + "build config"; WriteToOutput(output_, kErrMsg); static bool log_once = [] { PERFETTO_ELOG("%s", kErrMsg); @@ -191,15 +193,21 @@ input_reader.Read(buffer.get(), &buffer_len, kMaxMsgSize); TraceType type = trace_processor::GuessTraceType(buffer.get(), buffer_len); - if (type == TraceType::kGzipTraceType) { - GzipDecompressor decompressor; + if (type == TraceType::kGzipTraceType || type == TraceType::kZstdTraceType) { + auto codec = type == TraceType::kZstdTraceType + ? util::CompressionType::kZstd + : util::CompressionType::kGzip; + std::unique_ptr<util::StreamDecompressor> decompressor = + util::CreateDecompressor(codec); + if (!decompressor) + return false; // The codec isn't enabled in this build. auto consumer = [&](const uint8_t* data, size_t len) { online_trace_to_text.Feed(data, len); }; - using ResultCode = GzipDecompressor::ResultCode; + using ResultCode = util::StreamDecompressor::ResultCode; do { ResultCode code = - decompressor.FeedAndExtract(buffer.get(), buffer_len, consumer); + decompressor->FeedAndExtract(buffer.get(), buffer_len, consumer); if (code == ResultCode::kError || !online_trace_to_text.ok()) return false; } while (input_reader.Read(buffer.get(), &buffer_len, kMaxMsgSize));
diff --git a/src/traced/service/BUILD.gn b/src/traced/service/BUILD.gn index cbfca41..2b88ecd 100644 --- a/src/traced/service/BUILD.gn +++ b/src/traced/service/BUILD.gn
@@ -55,9 +55,6 @@ "../../tracing/ipc/service", "../../tracing/service:service", ] - if (enable_perfetto_zlib) { - deps += [ "../../tracing/service:zlib_compressor" ] - } sources = [ "service.cc" ]
diff --git a/src/traced/service/service.cc b/src/traced/service/service.cc index 25776c6..2e1cdaf 100644 --- a/src/traced/service/service.cc +++ b/src/traced/service/service.cc
@@ -39,10 +39,6 @@ #include <sys/system_properties.h> #endif -#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) -#include "src/tracing/service/zlib_compressor.h" -#endif - namespace perfetto { namespace { void PrintUsage(const char* prog_name) { @@ -155,9 +151,6 @@ base::MaybeLockFreeTaskRunner task_runner; std::unique_ptr<ServiceIPCHost> svc; TracingService::InitOpts init_opts = {}; -#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) - init_opts.compressor_fn = &ZlibCompressFn; -#endif #if PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID) // See /rfcs/0017-out-of-tree-protos.md . std::vector<base::ScopedMmap> extension_descriptor_mmaps;
diff --git a/src/tracing/BUILD.gn b/src/tracing/BUILD.gn index 730ccc9..acd6bbf 100644 --- a/src/tracing/BUILD.gn +++ b/src/tracing/BUILD.gn
@@ -205,9 +205,6 @@ "core", "service", ] - if (enable_perfetto_zlib) { - deps += [ "service:zlib_compressor" ] - } sources = [ "internal/in_process_tracing_backend.cc" ] }
diff --git a/src/tracing/internal/in_process_tracing_backend.cc b/src/tracing/internal/in_process_tracing_backend.cc index a88be1e..7405833 100644 --- a/src/tracing/internal/in_process_tracing_backend.cc +++ b/src/tracing/internal/in_process_tracing_backend.cc
@@ -25,10 +25,6 @@ #include "src/tracing/core/in_process_shared_memory.h" -#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) -#include "src/tracing/service/zlib_compressor.h" -#endif - // TODO(primiano): When the in-process backend is used, we should never end up // in a situation where the thread where the TracingService and Producer live // writes a packet and hence can get into the GetNewChunk() stall. @@ -74,13 +70,6 @@ std::unique_ptr<InProcessSharedMemory::Factory> shm( new InProcessSharedMemory::Factory()); TracingService::InitOpts init_opts = {}; -#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) - // Wire the zlib compressor so TraceConfig.compression_type = - // COMPRESSION_TYPE_DEFLATE takes effect on the in-process backend's - // service, mirroring what src/traced/service/service.cc does for the - // system backend. - init_opts.compressor_fn = &ZlibCompressFn; -#endif service_ = TracingService::CreateInstance(std::move(shm), task_runner, init_opts); service_->SetSMBScrapingEnabled(true);
diff --git a/src/tracing/service/BUILD.gn b/src/tracing/service/BUILD.gn index bb8e2b3..3dc096e 100644 --- a/src/tracing/service/BUILD.gn +++ b/src/tracing/service/BUILD.gn
@@ -74,11 +74,34 @@ "../../android_internal:lazy_library_loader", ] } + + # The service compresses ReadBuffers output using the codecs below, which + # pull in the optional compression libraries. Depend on them only when + # enabled, so libz/libzstd stay out of builds that don't compress. + if (enable_perfetto_zlib) { + deps += [ ":zlib_compressor" ] + } + if (enable_perfetto_zstd) { + deps += [ ":zstd_compressor" ] + } +} + +# Shared by the zlib and zstd packet compressors below; header-only. +if (enable_perfetto_zlib || enable_perfetto_zstd) { + source_set("packet_compressor_common") { + public_deps = [ + "../../../gn:default_deps", + "../../../include/perfetto/ext/tracing/core", + "../../protozero", + ] + sources = [ "packet_compressor_common.h" ] + } } if (enable_perfetto_zlib) { source_set("zlib_compressor") { deps = [ + ":packet_compressor_common", "../../../gn:default_deps", "../../../gn:zlib", "../../../include/perfetto/tracing", @@ -91,6 +114,22 @@ } } +if (enable_perfetto_zstd) { + source_set("zstd_compressor") { + deps = [ + ":packet_compressor_common", + "../../../gn:default_deps", + "../../../gn:zstd", + "../../../include/perfetto/tracing", + "../core", + ] + sources = [ + "zstd_compressor.cc", + "zstd_compressor.h", + ] + } +} + perfetto_unittest_source_set("unittests") { testonly = true deps = [ @@ -118,6 +157,13 @@ ] } + if (enable_perfetto_zstd) { + deps += [ + ":zstd_compressor", + "../../../gn:zstd", + ] + } + sources = [ "histogram_unittest.cc", "packet_stream_validator_unittest.cc", @@ -125,8 +171,9 @@ "trace_buffer_v2_unittest.cc", ] - if (enable_perfetto_zlib) { - sources += [ "zlib_compressor_unittest.cc" ] + if (enable_perfetto_zlib || enable_perfetto_zstd) { + deps += [ ":packet_compressor_common" ] + sources += [ "compressor_unittest.cc" ] } # These tests rely on test_task_runner.h which @@ -145,14 +192,29 @@ "../../../gn:default_deps", "../../../protos/perfetto/trace:zero", "../../../protos/perfetto/trace/ftrace:zero", + "../../../protos/perfetto/trace/track_event:zero", + "../../../protos/third_party/android/art:art_heap_graph_zero", "../../protozero", "../core", "../test:test_support", ] sources = [ + "compression_benchmark.cc", "packet_stream_validator_benchmark.cc", "trace_buffer_benchmark.cc", ] + if (enable_perfetto_zlib) { + deps += [ + ":zlib_compressor", + "../../../gn:zlib", + ] + } + if (enable_perfetto_zstd) { + deps += [ + ":zstd_compressor", + "../../../gn:zstd", + ] + } } }
diff --git a/src/tracing/service/compression_benchmark.cc b/src/tracing/service/compression_benchmark.cc new file mode 100644 index 0000000..7f95418 --- /dev/null +++ b/src/tracing/service/compression_benchmark.cc
@@ -0,0 +1,266 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <benchmark/benchmark.h> + +#include <cstdint> +#include <cstring> +#include <random> +#include <string> +#include <vector> + +#include "perfetto/base/build_config.h" +#include "perfetto/ext/tracing/core/slice.h" +#include "perfetto/ext/tracing/core/trace_packet.h" +#include "perfetto/protozero/packed_repeated_fields.h" +#include "perfetto/protozero/scattered_heap_buffer.h" +#include "perfetto/tracing/core/trace_config.h" + +#include "protos/perfetto/trace/ftrace/ftrace_event.pbzero.h" +#include "protos/perfetto/trace/ftrace/ftrace_event_bundle.pbzero.h" +#include "protos/perfetto/trace/ftrace/sched.pbzero.h" +#include "protos/perfetto/trace/test_event.pbzero.h" +#include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "protos/perfetto/trace/track_event/track_event.pbzero.h" +#include "protos/third_party/android/art/heap_graph.pbzero.h" + +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +#include "src/tracing/service/zlib_compressor.h" +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +#include "src/tracing/service/zstd_compressor.h" +#endif + +namespace perfetto { +namespace { + +// Workload policies: Make(i) returns one serialized TracePacket, varied by `i` +// so a batch isn't trivially deduped away (zstd dedupes across its window) and +// the measured ratios stay representative. + +// Structured, highly repetitive: a sched_switch ftrace bundle (~64 events), the +// bulk of a system trace. +struct FtraceSched { + static std::vector<uint8_t> Make(uint32_t i) { + protozero::HeapBuffered<protos::pbzero::TracePacket> packet; + auto* bundle = packet->set_ftrace_events(); + bundle->set_cpu(i % 8); + uint64_t ts = static_cast<uint64_t>(i) * 1000000; + for (uint32_t e = 0; e < 64; e++) { + auto* evt = bundle->add_event(); + evt->set_pid(1000 + ((i + e) % 256)); + evt->set_timestamp(ts + e * 1000); + auto* ss = evt->set_sched_switch(); + ss->set_prev_comm("thread-" + std::to_string((i + e) % 50)); + ss->set_prev_pid(1000 + ((i + e) % 256)); + ss->set_prev_state(e % 4); + ss->set_next_comm("thread-" + std::to_string((i + e + 1) % 50)); + ss->set_next_pid(1000 + ((i + e + 1) % 256)); + } + return packet.SerializeAsArray(); + } +}; + +// Java heap dump: a HeapGraph with types and cross-referencing objects (mostly +// packed varint ids). heap_graph is an out-of-tree field (number 56), not +// declared on TracePacket, so append it by field number. +struct JavaHeapDump { + static std::vector<uint8_t> Make(uint32_t i) { + namespace art = com::android::art::tracing::pbzero; + protozero::HeapBuffered<art::HeapGraph> hg; + hg->set_pid(static_cast<int32_t>(1000 + i)); + for (uint32_t t = 0; t < 8; t++) { + auto* type = hg->add_types(); + type->set_id(t + 1); + type->set_class_name("com.example.Class" + std::to_string((i + t) % 64)); + type->set_object_size(16 * (t + 1)); + } + for (uint32_t o = 0; o < 64; o++) { + uint64_t id = (static_cast<uint64_t>(i) << 16) + o + 1; + auto* obj = hg->add_objects(); + obj->set_id(id); + obj->set_type_id((o % 8) + 1); + obj->set_self_size(16 + (o % 4) * 8); + protozero::PackedVarInt refs; + refs.Append(id + 1); + refs.Append(id + 7); + obj->set_reference_object_id(refs); + } + std::vector<uint8_t> heap_graph = hg.SerializeAsArray(); + + protozero::HeapBuffered<protos::pbzero::TracePacket> packet; + // 56 = HeapGraph heap_graph (out-of-tree field, see heap_graph.proto). + packet->AppendBytes(56, heap_graph.data(), heap_graph.size()); + return packet.SerializeAsArray(); + } +}; + +// String-heavy: a TrackEvent slice with a name and categories. +struct TrackEvent { + static std::vector<uint8_t> Make(uint32_t i) { + protozero::HeapBuffered<protos::pbzero::TracePacket> packet; + packet->set_timestamp(static_cast<uint64_t>(i) * 1000); + auto* te = packet->set_track_event(); + te->set_track_uuid(i % 32); + te->add_categories("category-" + std::to_string(i % 16)); + te->set_name("SomeEventName::DoWork_" + std::to_string(i % 128)); + return packet.SerializeAsArray(); + } +}; + +// Worst case: incompressible random bytes (e.g. an already-compressed payload). +struct Incompressible { + static std::vector<uint8_t> Make(uint32_t i) { + std::minstd_rand rng(i + 1); + std::string s(1024, '\0'); + for (char& c : s) + c = static_cast<char>(rng()); + protozero::HeapBuffered<protos::pbzero::TracePacket> packet; + packet->set_for_testing()->set_str(s); + return packet.SerializeAsArray(); + } +}; + +// Backend policies: each runs one compile-time-selected codec at the given +// level. +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +struct Zlib { + static void Compress(std::vector<TracePacket>* packets, int /*level*/) { + ZlibCompressFn(packets); + } +}; +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +struct Zstd { + static void Compress(std::vector<TracePacket>* packets, int level) { + TraceConfig::CompressionConfig::Zstd zstd; + zstd.set_level(level); + ZstdCompressFn(packets, zstd); + } +}; +#endif + +TracePacket MakePacket(const std::vector<uint8_t>& buf) { + Slice slice = Slice::Allocate(buf.size()); + memcpy(slice.own_data(), buf.data(), buf.size()); + TracePacket packet; + packet.AddSlice(std::move(slice)); + return packet; +} + +// The compressor consumes (clears) its input, so each iteration needs a fresh +// copy of the packets. +std::vector<TracePacket> CopyPackets(const std::vector<TracePacket>& src) { + std::vector<TracePacket> out; + out.reserve(src.size()); + for (const TracePacket& packet : src) { + TracePacket copy; + for (const Slice& slice : packet.slices()) { + Slice new_slice = Slice::Allocate(slice.size); + memcpy(new_slice.own_data(), slice.start, slice.size); + copy.AddSlice(std::move(new_slice)); + } + out.push_back(std::move(copy)); + } + return out; +} + +bool IsBenchmarkFunctionalOnly() { + return getenv("BENCHMARK_FUNCTIONAL_TEST_ONLY") != nullptr; +} + +// Sweep the block size (packets per compress call) at the codec's default +// level. +static void BlockSizeArgs(benchmark::internal::Benchmark* b) { + b->Unit(benchmark::kMicrosecond); + b->ArgNames({"packets", "level"}); + if (IsBenchmarkFunctionalOnly()) { + b->Args({1, 0})->Iterations(1); + return; + } + b->RangeMultiplier(8)->Ranges({{1, 4096}, {0, 0}}); +} + +// Sweep the zstd level (fastest..max) at a fixed, representative block size. +static void ZstdLevelArgs(benchmark::internal::Benchmark* b) { + b->Unit(benchmark::kMicrosecond); + b->ArgNames({"packets", "level"}); + if (IsBenchmarkFunctionalOnly()) { + b->Args({1, 0})->Iterations(1); + return; + } + for (int level : {1, 3, 9, 19, 22}) + b->Args({512, level}); +} + +// Compresses a batch of `state.range(0)` packets per iteration. The reported +// time is the per-batch compression latency; SetBytesProcessed gives throughput +// and the counters report the compression ratio. +template <typename Backend, typename Workload> +static void BM_Compress(benchmark::State& state) { + const size_t num_packets = static_cast<size_t>(state.range(0)); + const int level = static_cast<int>(state.range(1)); + std::vector<TracePacket> packets; + size_t total_in = 0; + for (size_t i = 0; i < num_packets; i++) { + packets.push_back(MakePacket(Workload::Make(static_cast<uint32_t>(i)))); + total_in += packets.back().size(); + } + + size_t total_out = 0; + for (auto _ : state) { + state.PauseTiming(); + std::vector<TracePacket> input = CopyPackets(packets); + state.ResumeTiming(); + + Backend::Compress(&input, level); + + total_out = 0; + for (const TracePacket& packet : input) + total_out += packet.size(); + benchmark::DoNotOptimize(total_out); + } + + state.counters["packets"] = static_cast<double>(num_packets); + state.counters["in_bytes"] = static_cast<double>(total_in); + state.counters["out_bytes"] = static_cast<double>(total_out); + state.counters["ratio"] = + total_out ? static_cast<double>(total_in) / static_cast<double>(total_out) + : 0; + state.SetBytesProcessed(static_cast<int64_t>(state.iterations()) * + static_cast<int64_t>(total_in)); +} + +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +BENCHMARK_TEMPLATE(BM_Compress, Zlib, FtraceSched)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zlib, JavaHeapDump)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zlib, TrackEvent)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zlib, Incompressible)->Apply(BlockSizeArgs); +#endif + +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +BENCHMARK_TEMPLATE(BM_Compress, Zstd, FtraceSched)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, JavaHeapDump)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, TrackEvent)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, Incompressible)->Apply(BlockSizeArgs); + +// Level sweep (zstd only): how ratio and latency trade off across levels. +BENCHMARK_TEMPLATE(BM_Compress, Zstd, FtraceSched)->Apply(ZstdLevelArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, JavaHeapDump)->Apply(ZstdLevelArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, TrackEvent)->Apply(ZstdLevelArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, Incompressible)->Apply(ZstdLevelArgs); +#endif + +} // namespace +} // namespace perfetto
diff --git a/src/tracing/service/compressor_unittest.cc b/src/tracing/service/compressor_unittest.cc new file mode 100644 index 0000000..659b6c6 --- /dev/null +++ b/src/tracing/service/compressor_unittest.cc
@@ -0,0 +1,370 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "perfetto/base/build_config.h" + +#include <algorithm> +#include <cstring> +#include <random> +#include <string> +#include <vector> + +#include "perfetto/tracing/core/trace_config.h" +#include "protos/perfetto/trace/test_event.gen.h" +#include "protos/perfetto/trace/trace.gen.h" +#include "protos/perfetto/trace/trace_packet.gen.h" +#include "src/tracing/service/packet_compressor_common.h" +#include "src/tracing/service/tracing_service_impl.h" +#include "test/gtest_and_gmock.h" + +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +#include <zlib.h> +#include "src/tracing/service/zlib_compressor.h" +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +#include <zstd.h> +#include "src/tracing/service/zstd_compressor.h" +#endif + +namespace perfetto { +namespace { + +using ::testing::Each; +using ::testing::ElementsAre; +using ::testing::Field; +using ::testing::IsEmpty; +using ::testing::Le; +using ::testing::Not; +using ::testing::Property; +using ::testing::SizeIs; +using tracing_service::TracingServiceImpl; + +// The compressors cap their output slices at the service's max slice size. +static_assert(packet_compressor::kCompressSliceSize == + TracingServiceImpl::kMaxTracePacketSliceSize); + +// One backend per compile-time-selected compressor. Each provides the compress +// entrypoint and a matching decompressor for the test to verify the output. +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +struct ZlibBackend { + static constexpr const char* kName = "Zlib"; + static void Compress(std::vector<TracePacket>* packets, int /*level*/ = 0) { + ZlibCompressFn(packets); + } + static std::string Decompress(const std::string& data) { + uint8_t out[1024]; + z_stream stream{}; + stream.next_in = reinterpret_cast<uint8_t*>(const_cast<char*>(data.data())); + stream.avail_in = static_cast<unsigned int>(data.size()); + EXPECT_EQ(inflateInit(&stream), Z_OK); + std::string s; + int ret; + do { + stream.next_out = out; + stream.avail_out = sizeof(out); + ret = inflate(&stream, Z_NO_FLUSH); + EXPECT_NE(ret, Z_STREAM_ERROR); + EXPECT_NE(ret, Z_NEED_DICT); + EXPECT_NE(ret, Z_DATA_ERROR); + EXPECT_NE(ret, Z_MEM_ERROR); + s.append(reinterpret_cast<char*>(out), sizeof(out) - stream.avail_out); + } while (ret != Z_STREAM_END); + inflateEnd(&stream); + return s; + } +}; +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB) + +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +struct ZstdBackend { + static constexpr const char* kName = "Zstd"; + static void Compress(std::vector<TracePacket>* packets, int level = 0) { + TraceConfig::CompressionConfig::Zstd zstd; + zstd.set_level(level); + ZstdCompressFn(packets, zstd); + } + static std::string Decompress(const std::string& data) { + ZSTD_DStream* stream = ZSTD_createDStream(); + EXPECT_NE(stream, nullptr); + ZSTD_initDStream(stream); + uint8_t out[1024]; + ZSTD_inBuffer in = {data.data(), data.size(), 0}; + std::string s; + size_t ret = 0; + do { + ZSTD_outBuffer out_buf = {out, sizeof(out), 0}; + ret = ZSTD_decompressStream(stream, &out_buf, &in); + EXPECT_FALSE(ZSTD_isError(ret)); + s.append(reinterpret_cast<char*>(out), out_buf.pos); + } while (ret != 0 && in.pos < in.size); + ZSTD_freeDStream(stream); + return s; + } +}; +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + +using Backends = ::testing::Types< +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) + ZlibBackend +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) && PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + , +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + ZstdBackend +#endif + >; + +class BackendNames { + public: + template <typename T> + static std::string GetName(int) { + return T::kName; + } +}; + +template <typename Backend> +class CompressorTest : public ::testing::Test {}; +TYPED_TEST_SUITE(CompressorTest, Backends, BackendNames); + +template <typename F> +TracePacket CreateTracePacket(F fill_function) { + protos::gen::TracePacket msg; + fill_function(&msg); + std::vector<uint8_t> buf = msg.SerializeAsArray(); + Slice slice = Slice::Allocate(buf.size()); + memcpy(slice.own_data(), buf.data(), buf.size()); + TracePacket packet; + packet.AddSlice(std::move(slice)); + return packet; +} + +// Returns a copy of `old` that owns its slices' data. +TracePacket CopyTracePacket(const TracePacket& old) { + TracePacket ret; + for (const Slice& slice : old.slices()) { + Slice new_slice = Slice::Allocate(slice.size); + memcpy(new_slice.own_data(), slice.start, slice.size); + ret.AddSlice(std::move(new_slice)); + } + return ret; +} + +std::vector<TracePacket> CopyTracePackets(const std::vector<TracePacket>& old) { + std::vector<TracePacket> ret; + ret.reserve(old.size()); + for (const TracePacket& packet : old) + ret.push_back(CopyTracePacket(packet)); + return ret; +} + +// Distinct random bytes on every call: incompressible, and crucially never +// repeating across packets, so zstd (which dedupes across its window) can't +// collapse the MaxSliceSize input to nothing. +std::string RandomString(size_t size) { + static uint32_t seed = 0; + std::default_random_engine rnd(seed++); + std::uniform_int_distribution<> dist(0, 255); + std::string s(size, '\0'); + for (char& c : s) + c = static_cast<char>(dist(rnd)); + return s; +} + +TYPED_TEST(CompressorTest, Empty) { + std::vector<TracePacket> packets; + + TypeParam::Compress(&packets); + + EXPECT_THAT(packets, IsEmpty()); +} + +TYPED_TEST(CompressorTest, End2EndCompressAndDecompress) { + std::vector<TracePacket> packets; + packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str("abc"); + })); + packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str("def"); + })); + + TypeParam::Compress(&packets); + + ASSERT_THAT(packets, SizeIs(1)); + protos::gen::TracePacket compressed_packet_proto; + ASSERT_TRUE(compressed_packet_proto.ParseFromString( + packets[0].GetRawBytesForTesting())); + const std::string& data = compressed_packet_proto.compressed_packets(); + EXPECT_THAT(data, Not(IsEmpty())); + protos::gen::Trace subtrace; + ASSERT_TRUE(subtrace.ParseFromString(TypeParam::Decompress(data))); + EXPECT_THAT( + subtrace.packet(), + ElementsAre(Property(&protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, "abc")), + Property(&protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, "def")))); +} + +TYPED_TEST(CompressorTest, MaxSliceSize) { + constexpr size_t kStopOutputSize = + TracingServiceImpl::kMaxTracePacketSliceSize + 2000; + + std::vector<TracePacket> packets; + TracePacket compressed_packet; + while (compressed_packet.size() < kStopOutputSize) { + packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str(RandomString(65536)); + })); + std::vector<TracePacket> packets_copy = CopyTracePackets(packets); + TypeParam::Compress(&packets_copy); + ASSERT_THAT(packets_copy, SizeIs(1)); + compressed_packet = std::move(packets_copy[0]); + } + + EXPECT_GE(compressed_packet.slices().size(), 2u); + ASSERT_GT(compressed_packet.size(), + TracingServiceImpl::kMaxTracePacketSliceSize); + EXPECT_THAT(compressed_packet.slices(), + Each(Field(&Slice::size, + Le(TracingServiceImpl::kMaxTracePacketSliceSize)))); +} + +// Round-trips many distinct packets and checks they come back in order. This +// exercises the per-packet framing/tokenization beyond the two-packet case. +TYPED_TEST(CompressorTest, ManyPacketsPreserveOrder) { + constexpr size_t kNumPackets = 100; + std::vector<TracePacket> packets; + for (size_t i = 0; i < kNumPackets; i++) { + packets.push_back(CreateTracePacket([i](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str("packet-" + std::to_string(i)); + })); + } + + TypeParam::Compress(&packets); + + ASSERT_THAT(packets, SizeIs(1)); + protos::gen::TracePacket compressed; + ASSERT_TRUE(compressed.ParseFromString(packets[0].GetRawBytesForTesting())); + protos::gen::Trace subtrace; + ASSERT_TRUE(subtrace.ParseFromString( + TypeParam::Decompress(compressed.compressed_packets()))); + ASSERT_THAT(subtrace.packet(), SizeIs(kNumPackets)); + for (size_t i = 0; i < kNumPackets; i++) { + EXPECT_EQ(subtrace.packet()[i].for_testing().str(), + "packet-" + std::to_string(i)); + } +} + +// Compresses enough incompressible data that the output spans multiple slices, +// then decompresses and checks the exact content survives. End2End only covers +// single-slice output; MaxSliceSize checks slice sizes but never round-trips. +TYPED_TEST(CompressorTest, MultiSliceOutputRoundTrip) { + std::vector<std::string> expected; + std::vector<TracePacket> packets; + for (int i = 0; i < 4; i++) { + std::string payload = RandomString(65536); + expected.push_back(payload); + packets.push_back(CreateTracePacket([&](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str(payload); + })); + } + + TypeParam::Compress(&packets); + + ASSERT_THAT(packets, SizeIs(1)); + EXPECT_GE(packets[0].slices().size(), 2u); // Output spans multiple slices. + protos::gen::TracePacket compressed; + ASSERT_TRUE(compressed.ParseFromString(packets[0].GetRawBytesForTesting())); + protos::gen::Trace subtrace; + ASSERT_TRUE(subtrace.ParseFromString( + TypeParam::Decompress(compressed.compressed_packets()))); + ASSERT_THAT(subtrace.packet(), SizeIs(expected.size())); + for (size_t i = 0; i < expected.size(); i++) + EXPECT_EQ(subtrace.packet()[i].for_testing().str(), expected[i]); +} + +// A TracePacket whose payload is split across several input slices (as the +// service produces) must round-trip. Exercises the multi-slice loop in +// PushPacket, which the other tests (single-slice packets) never hit. +TYPED_TEST(CompressorTest, MultiSliceInputPacket) { + protos::gen::TracePacket msg; + msg.mutable_for_testing()->set_str("multi-slice-input-payload"); + std::vector<uint8_t> buf = msg.SerializeAsArray(); + ASSERT_GT(buf.size(), 8u); + + TracePacket packet; + constexpr size_t kInSliceSize = 4; // Tiny, to force several input slices. + for (size_t pos = 0; pos < buf.size(); pos += kInSliceSize) { + size_t n = std::min(kInSliceSize, buf.size() - pos); + Slice slice = Slice::Allocate(n); + memcpy(slice.own_data(), &buf[pos], n); + packet.AddSlice(std::move(slice)); + } + ASSERT_GT(packet.slices().size(), 1u); + + std::vector<TracePacket> packets; + packets.push_back(std::move(packet)); + + TypeParam::Compress(&packets); + + ASSERT_THAT(packets, SizeIs(1)); + protos::gen::TracePacket compressed; + ASSERT_TRUE(compressed.ParseFromString(packets[0].GetRawBytesForTesting())); + protos::gen::Trace subtrace; + ASSERT_TRUE(subtrace.ParseFromString( + TypeParam::Decompress(compressed.compressed_packets()))); + ASSERT_THAT(subtrace.packet(), SizeIs(1)); + EXPECT_EQ(subtrace.packet()[0].for_testing().str(), + "multi-slice-input-payload"); +} + +// An explicit level must reach the codec: the output still round-trips, and on +// compressible data a higher level is never worse than a lower one (catching +// the level being silently dropped). +TYPED_TEST(CompressorTest, CompressionLevelIsHonored) { + auto make_packets = [] { + std::vector<TracePacket> packets; + for (int i = 0; i < 200; i++) { + packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str(std::string(1024, 'a')); + })); + } + return packets; + }; + + auto compress_and_measure = [](std::vector<TracePacket> packets, + int level) -> size_t { + TypeParam::Compress(&packets, level); + EXPECT_THAT(packets, SizeIs(1)); + protos::gen::TracePacket compressed; + EXPECT_TRUE(compressed.ParseFromString(packets[0].GetRawBytesForTesting())); + protos::gen::Trace subtrace; + EXPECT_TRUE(subtrace.ParseFromString( + TypeParam::Decompress(compressed.compressed_packets()))); + EXPECT_THAT(subtrace.packet(), SizeIs(200)); + return packets[0].size(); + }; + + size_t low_level_size = compress_and_measure(make_packets(), 1); + size_t high_level_size = compress_and_measure(make_packets(), 9); + + EXPECT_GT(low_level_size, 0u); + EXPECT_LE(high_level_size, low_level_size); +} + +} // namespace +} // namespace perfetto
diff --git a/src/tracing/service/packet_compressor_common.h b/src/tracing/service/packet_compressor_common.h new file mode 100644 index 0000000..e1b98a8 --- /dev/null +++ b/src/tracing/service/packet_compressor_common.h
@@ -0,0 +1,70 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACING_SERVICE_PACKET_COMPRESSOR_COMMON_H_ +#define SRC_TRACING_SERVICE_PACKET_COMPRESSOR_COMMON_H_ + +#include <array> +#include <cstdint> +#include <cstring> + +#include "perfetto/base/logging.h" +#include "perfetto/ext/tracing/core/slice.h" +#include "perfetto/protozero/proto_utils.h" + +namespace perfetto { +namespace packet_compressor { + +// TODO(sashwinbalaji): Extract the output-slice buffering and packet assembly +// shared by the zlib and zstd compressors into this header, so each codec is +// just its own streaming loop. + +// Size of each compressed output slice. Mirrors the service's +// kMaxTracePacketSliceSize. +inline constexpr size_t kCompressSliceSize = 128 * 1024 - 512; + +// Builds the proto preamble (field tag + length) that the zlib and zstd packet +// compressors prefix to each TracePacket, so the compressed stream itself +// parses as a valid Trace proto. +struct Preamble { + uint32_t size; + std::array<uint8_t, 16> buf; +}; + +template <uint32_t id> +Preamble GetPreamble(size_t sz) { + Preamble preamble{}; + uint8_t* ptr = preamble.buf.data(); + constexpr uint32_t tag = protozero::proto_utils::MakeTagLengthDelimited(id); + ptr = protozero::proto_utils::WriteVarInt(tag, ptr); + ptr = protozero::proto_utils::WriteVarInt(sz, ptr); + preamble.size = + static_cast<uint32_t>(reinterpret_cast<uintptr_t>(ptr) - + reinterpret_cast<uintptr_t>(preamble.buf.data())); + PERFETTO_DCHECK(preamble.size < preamble.buf.size()); + return preamble; +} + +inline Slice PreambleToSlice(const Preamble& preamble) { + Slice slice = Slice::Allocate(preamble.size); + memcpy(slice.own_data(), preamble.buf.data(), preamble.size); + return slice; +} + +} // namespace packet_compressor +} // namespace perfetto + +#endif // SRC_TRACING_SERVICE_PACKET_COMPRESSOR_COMMON_H_
diff --git a/src/tracing/service/tracing_service_impl.cc b/src/tracing/service/tracing_service_impl.cc index 5fc2639..3e7f593 100644 --- a/src/tracing/service/tracing_service_impl.cc +++ b/src/tracing/service/tracing_service_impl.cc
@@ -117,6 +117,12 @@ #include "src/tracing/service/tracing_service_endpoints_impl.h" #include "src/tracing/service/tracing_service_session.h" #include "src/tracing/service/tracing_service_structs.h" +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +#include "src/tracing/service/zlib_compressor.h" +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +#include "src/tracing/service/zstd_compressor.h" +#endif #include "protos/perfetto/common/builtin_clock.gen.h" #include "protos/perfetto/common/builtin_clock.pbzero.h" @@ -1166,15 +1172,14 @@ cfg.fflush_post_write() == TraceConfig::FFLUSH_ENABLED; } - if (cfg.compression_type() == TraceConfig::COMPRESSION_TYPE_DEFLATE) { - if (init_opts_.compressor_fn) { - tracing_session->compress_deflate = true; - } else { - PERFETTO_LOG( - "COMPRESSION_TYPE_DEFLATE is not supported in the current build " - "configuration. Skipping compression"); - } +#if !PERFETTO_BUILDFLAG(PERFETTO_ZLIB) && !PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + if (cfg.compression_type() != TraceConfig::COMPRESSION_TYPE_UNSPECIFIED || + cfg.has_compression()) { + PERFETTO_LOG( + "Compression was requested but this build has no compressor. " + "Skipping compression"); } +#endif // Initialize the log buffers. bool did_allocate_all_buffers = true; @@ -2913,12 +2918,30 @@ void TracingServiceImpl::MaybeCompressPackets( TracingSession* tracing_session, - std::vector<TracePacket>* packets) { - if (!tracing_session->compress_deflate) { + [[maybe_unused]] std::vector<TracePacket>* packets) { + // Compress with the codec the config selects, preferring the newest (highest + // proto field number) this build supports. Leaves the packets uncompressed if + // none is available. + // + // The branches below run highest-field-number-first, so a new codec's branch + // goes at the top. + [[maybe_unused]] const auto& compression = + tracing_session->config.compression(); +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + if (compression.has_zstd()) { + ZstdCompressFn(packets, compression.zstd()); return; } - - init_opts_.compressor_fn(packets); +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) + // Deflate also serves the legacy compression_type = DEFLATE, so configs + // predating `compression` still get compressed. + if (compression.has_deflate() || tracing_session->config.compression_type() == + TraceConfig::COMPRESSION_TYPE_DEFLATE) { + ZlibCompressFn(packets); + return; + } +#endif } bool TracingServiceImpl::WriteIntoFile(TracingSession* tracing_session, @@ -4850,7 +4873,6 @@ cloned_session->flushes_requested = src->flushes_requested; cloned_session->flushes_succeeded = src->flushes_succeeded; cloned_session->flushes_failed = src->flushes_failed; - cloned_session->compress_deflate = src->compress_deflate; if (src->trace_filter && !skip_trace_filter) { // Copy the trace filter, unless it's a clone-for-bugreport (b/317065412). cloned_session->trace_filter.reset(
diff --git a/src/tracing/service/tracing_service_impl_unittest.cc b/src/tracing/service/tracing_service_impl_unittest.cc index bde0170..dc2bf01 100644 --- a/src/tracing/service/tracing_service_impl_unittest.cc +++ b/src/tracing/service/tracing_service_impl_unittest.cc
@@ -88,6 +88,11 @@ #include "src/tracing/service/zlib_compressor.h" #endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +#include <zstd.h> +#include "src/tracing/service/zstd_compressor.h" +#endif + using ::testing::_; using ::testing::AssertionFailure; using ::testing::AssertionResult; @@ -165,7 +170,7 @@ } #if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) -std::string Decompress(const std::string& data) { +std::string DecompressZlib(const std::string& data) { uint8_t out[1024]; z_stream stream{}; @@ -191,7 +196,7 @@ return s; } -std::vector<protos::gen::TracePacket> DecompressTrace( +std::vector<protos::gen::TracePacket> DecompressTraceZlib( const std::vector<protos::gen::TracePacket> compressed) { std::vector<protos::gen::TracePacket> decompressed; @@ -201,7 +206,7 @@ continue; } - std::string s = Decompress(c.compressed_packets()); + std::string s = DecompressZlib(c.compressed_packets()); protos::gen::Trace t; EXPECT_TRUE(t.ParseFromString(s)); decompressed.insert(decompressed.end(), t.packet().begin(), @@ -211,6 +216,42 @@ } #endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +std::string DecompressZstd(const std::string& data) { + ZSTD_DStream* stream = ZSTD_createDStream(); + ZSTD_initDStream(stream); + uint8_t out[1024]; + ZSTD_inBuffer in = {data.data(), data.size(), 0}; + std::string s; + size_t ret = 0; + do { + ZSTD_outBuffer out_buf = {out, sizeof(out), 0}; + ret = ZSTD_decompressStream(stream, &out_buf, &in); + EXPECT_FALSE(ZSTD_isError(ret)); + s.append(reinterpret_cast<char*>(out), out_buf.pos); + } while (ret != 0 && in.pos < in.size); + ZSTD_freeDStream(stream); + return s; +} + +std::vector<protos::gen::TracePacket> DecompressTraceZstd( + const std::vector<protos::gen::TracePacket> compressed) { + std::vector<protos::gen::TracePacket> decompressed; + for (const protos::gen::TracePacket& c : compressed) { + if (c.compressed_packets().empty()) { + decompressed.push_back(c); + continue; + } + std::string s = DecompressZstd(c.compressed_packets()); + protos::gen::Trace t; + EXPECT_TRUE(t.ParseFromString(s)); + decompressed.insert(decompressed.end(), t.packet().begin(), + t.packet().end()); + } + return decompressed; +} +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + std::vector<std::string> GetReceivedTriggers( const std::vector<protos::gen::TracePacket>& trace) { std::vector<std::string> triggers; @@ -2119,11 +2160,11 @@ producer->WaitForDataSourceStart("data_source"); } +// With no codec compiled in, a config that asks for compression must degrade to +// an uncompressed trace rather than dropping data. +#if !PERFETTO_BUILDFLAG(PERFETTO_ZLIB) && !PERFETTO_BUILDFLAG(PERFETTO_ZSTD) TEST_F(TracingServiceImplTest, CompressionConfiguredButUnsupported) { - // Initialize the service without support for compression. - TracingService::InitOpts init_opts; - init_opts.compressor_fn = nullptr; - InitializeSvcWithOpts(init_opts); + InitializeSvcWithOpts({}); std::unique_ptr<MockConsumer> consumer = CreateMockConsumer(); consumer->Connect(svc.get()); @@ -2176,11 +2217,12 @@ Property(&protos::gen::TestEvent::str, Eq("payload-2"))))); } +#endif // !PERFETTO_BUILDFLAG(PERFETTO_ZLIB) && + // !PERFETTO_BUILDFLAG(PERFETTO_ZSTD) #if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) TEST_F(TracingServiceImplTest, CompressionReadIpc) { TracingService::InitOpts init_opts; - init_opts.compressor_fn = ZlibCompressFn; InitializeSvcWithOpts(init_opts); std::unique_ptr<MockConsumer> consumer = CreateMockConsumer(); @@ -2227,7 +2269,7 @@ Each(Property(&protos::gen::TracePacket::compressed_packets, Not(IsEmpty())))); std::vector<protos::gen::TracePacket> decompressed_packets = - DecompressTrace(compressed_packets); + DecompressTraceZlib(compressed_packets); EXPECT_THAT(decompressed_packets, Contains(Property( &protos::gen::TracePacket::for_testing, @@ -2238,9 +2280,164 @@ Property(&protos::gen::TestEvent::str, Eq("payload-2"))))); } +// A config that selects zstd via `compression` but is served by a build without +// zstd must fall back to the legacy compression_type (DEFLATE), not emit an +// uncompressed trace. The output is therefore deflate-compressed. +#if !PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +TEST_F(TracingServiceImplTest, CompressionConfigZstdFallsBackToLegacyDeflate) { + InitializeSvcWithOpts({}); + + std::unique_ptr<MockConsumer> consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr<MockProducer> producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + trace_config.set_compression_type(TraceConfig::COMPRESSION_TYPE_DEFLATE); + trace_config.mutable_compression()->mutable_zstd(); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr<TraceWriter> writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + std::vector<protos::gen::TracePacket> compressed_packets = + consumer->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::compressed_packets, + Not(IsEmpty())))); + // Decodes with the zlib (deflate) decompressor, proving the fall-through. + std::vector<protos::gen::TracePacket> decompressed_packets = + DecompressTraceZlib(compressed_packets); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-1"))))); +} +#endif // !PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + +// Deflate can be selected directly via the new config (compression.deflate), +// with no legacy compression_type set. The output is deflate-compressed. +TEST_F(TracingServiceImplTest, CompressionConfigDeflate) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + std::unique_ptr<MockConsumer> consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr<MockProducer> producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + trace_config.mutable_compression()->mutable_deflate(); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr<TraceWriter> writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + std::vector<protos::gen::TracePacket> compressed_packets = + consumer->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::compressed_packets, + Not(IsEmpty())))); + std::vector<protos::gen::TracePacket> decompressed_packets = + DecompressTraceZlib(compressed_packets); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-1"))))); +} + +// A compression field naming no codec this service understands (e.g. a newer +// config with a future codec) must not crash: it degrades to no compression. +TEST_F(TracingServiceImplTest, CompressionConfigUnknownCodecDoesNotCrash) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + std::unique_ptr<MockConsumer> consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr<MockProducer> producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + // compression is set but names no codec this service understands (as + // an old service would observe a future codec's sub-message). No legacy + // compression_type is set, so this degrades to no compression, not a crash. + trace_config.mutable_compression(); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr<TraceWriter> writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + // No compression was applied; the packet is readable as-is (no crash). + std::vector<protos::gen::TracePacket> packets = consumer->ReadBuffers(); + EXPECT_THAT(packets, Contains(Property(&protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, + Eq("payload-1"))))); +} + TEST_F(TracingServiceImplTest, CompressionWriteIntoFile) { TracingService::InitOpts init_opts; - init_opts.compressor_fn = ZlibCompressFn; InitializeSvcWithOpts(init_opts); std::unique_ptr<MockConsumer> consumer = CreateMockConsumer(); @@ -2292,7 +2489,7 @@ Each(Property(&protos::gen::TracePacket::compressed_packets, Not(IsEmpty())))); std::vector<protos::gen::TracePacket> decompressed_packets = - DecompressTrace(trace.packet()); + DecompressTraceZlib(trace.packet()); EXPECT_THAT(decompressed_packets, Contains(Property( &protos::gen::TracePacket::for_testing, @@ -2303,6 +2500,245 @@ Property(&protos::gen::TestEvent::str, Eq("payload-2"))))); } +TEST_F(TracingServiceImplTest, CloneSessionWithCompression) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + // The consumer the creates the initial tracing session. + std::unique_ptr<MockConsumer> consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + // The consumer that clones it and reads back the data. + std::unique_ptr<MockConsumer> consumer2 = CreateMockConsumer(); + consumer2->Connect(svc.get()); + + std::unique_ptr<MockProducer> producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + + producer->RegisterDataSource("ds_1"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(32); + auto* ds_cfg = trace_config.add_data_sources()->mutable_config(); + ds_cfg->set_name("ds_1"); + trace_config.set_compression_type(TraceConfig::COMPRESSION_TYPE_DEFLATE); + + consumer->EnableTracing(trace_config); + producer->WaitForTracingSetup(); + + producer->WaitForDataSourceSetup("ds_1"); + + producer->WaitForDataSourceStart("ds_1"); + + std::unique_ptr<TraceWriter> writer = producer->CreateTraceWriter("ds_1"); + + // Add some data. + static constexpr size_t kNumTestPackets = 20; + for (size_t i = 0; i < kNumTestPackets; i++) { + auto tp = writer->NewTracePacket(); + std::string payload("payload" + std::to_string(i)); + tp->set_for_testing()->set_str(payload.c_str(), payload.size()); + tp->set_timestamp(static_cast<uint64_t>(i)); + } + + auto clone_done = task_runner.CreateCheckpoint("clone_done"); + EXPECT_CALL(*consumer2, OnSessionCloned(_)) + .WillOnce( + [clone_done](const Consumer::OnSessionClonedArgs&) { clone_done(); }); + consumer2->CloneSession(1); + // CloneSession() will implicitly issue a flush. Linearize with that. + FlushFlags expected_flags(FlushFlags::Initiator::kTraced, + FlushFlags::Reason::kTraceClone); + producer->ExpectFlush(writer.get(), /*reply=*/true, expected_flags); + task_runner.RunUntilCheckpoint("clone_done"); + + // Delete the initial tracing session. + consumer->DisableTracing(); + consumer->FreeBuffers(); + producer->WaitForDataSourceStop("ds_1"); + consumer->WaitForTracingDisabled(); + + // Read back the cloned trace and check that it's compressed + std::vector<protos::gen::TracePacket> compressed_packets = + consumer2->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::compressed_packets, + Not(IsEmpty())))); +} + +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB) + +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +TEST_F(TracingServiceImplTest, CompressionZstdReadIpc) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + std::unique_ptr<MockConsumer> consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr<MockProducer> producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + trace_config.mutable_compression()->mutable_zstd(); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr<TraceWriter> writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-2"); + } + + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + std::vector<protos::gen::TracePacket> compressed_packets = + consumer->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::compressed_packets, + Not(IsEmpty())))); + std::vector<protos::gen::TracePacket> decompressed_packets = + DecompressTraceZstd(compressed_packets); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-1"))))); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-2"))))); +} + +// An explicit zstd compression level from the TraceConfig must be plumbed +// through to the compressor and still produce a decodable trace. +TEST_F(TracingServiceImplTest, CompressionZstdConfiguredLevel) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + std::unique_ptr<MockConsumer> consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr<MockProducer> producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + trace_config.mutable_compression()->mutable_zstd()->set_level(19); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr<TraceWriter> writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + std::vector<protos::gen::TracePacket> compressed_packets = + consumer->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::compressed_packets, + Not(IsEmpty())))); + std::vector<protos::gen::TracePacket> decompressed_packets = + DecompressTraceZstd(compressed_packets); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-1"))))); +} +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) && PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +// When both compressors are available, the zstd compression is used in +// preference to the legacy compression_type (deflate). The output is therefore +// zstd-compressed. +TEST_F(TracingServiceImplTest, CompressionConfigZstdUsedWhenAvailable) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + std::unique_ptr<MockConsumer> consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr<MockProducer> producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + trace_config.set_compression_type(TraceConfig::COMPRESSION_TYPE_DEFLATE); + trace_config.mutable_compression()->mutable_zstd(); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr<TraceWriter> writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + std::vector<protos::gen::TracePacket> compressed_packets = + consumer->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::compressed_packets, + Not(IsEmpty())))); + // Decodes with the zstd decompressor, proving the preference won. + std::vector<protos::gen::TracePacket> decompressed_packets = + DecompressTraceZstd(compressed_packets); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-1"))))); +} +#endif // PERFETTO_ZLIB && PERFETTO_ZSTD + TEST_F(TracingServiceImplTest, FlushStrategies) { constexpr uint32_t kDefaultWriteIntoFilePeriodMs = 5000; @@ -2414,76 +2850,6 @@ } } -TEST_F(TracingServiceImplTest, CloneSessionWithCompression) { - TracingService::InitOpts init_opts; - init_opts.compressor_fn = ZlibCompressFn; - InitializeSvcWithOpts(init_opts); - - // The consumer the creates the initial tracing session. - std::unique_ptr<MockConsumer> consumer = CreateMockConsumer(); - consumer->Connect(svc.get()); - - // The consumer that clones it and reads back the data. - std::unique_ptr<MockConsumer> consumer2 = CreateMockConsumer(); - consumer2->Connect(svc.get()); - - std::unique_ptr<MockProducer> producer = CreateMockProducer(); - producer->Connect(svc.get(), "mock_producer"); - - producer->RegisterDataSource("ds_1"); - - TraceConfig trace_config; - trace_config.add_buffers()->set_size_kb(32); - auto* ds_cfg = trace_config.add_data_sources()->mutable_config(); - ds_cfg->set_name("ds_1"); - trace_config.set_compression_type(TraceConfig::COMPRESSION_TYPE_DEFLATE); - - consumer->EnableTracing(trace_config); - producer->WaitForTracingSetup(); - - producer->WaitForDataSourceSetup("ds_1"); - - producer->WaitForDataSourceStart("ds_1"); - - std::unique_ptr<TraceWriter> writer = producer->CreateTraceWriter("ds_1"); - - // Add some data. - static constexpr size_t kNumTestPackets = 20; - for (size_t i = 0; i < kNumTestPackets; i++) { - auto tp = writer->NewTracePacket(); - std::string payload("payload" + std::to_string(i)); - tp->set_for_testing()->set_str(payload.c_str(), payload.size()); - tp->set_timestamp(static_cast<uint64_t>(i)); - } - - auto clone_done = task_runner.CreateCheckpoint("clone_done"); - EXPECT_CALL(*consumer2, OnSessionCloned(_)) - .WillOnce( - [clone_done](const Consumer::OnSessionClonedArgs&) { clone_done(); }); - consumer2->CloneSession(1); - // CloneSession() will implicitly issue a flush. Linearize with that. - FlushFlags expected_flags(FlushFlags::Initiator::kTraced, - FlushFlags::Reason::kTraceClone); - producer->ExpectFlush(writer.get(), /*reply=*/true, expected_flags); - task_runner.RunUntilCheckpoint("clone_done"); - - // Delete the initial tracing session. - consumer->DisableTracing(); - consumer->FreeBuffers(); - producer->WaitForDataSourceStop("ds_1"); - consumer->WaitForTracingDisabled(); - - // Read back the cloned trace and check that it's compressed - std::vector<protos::gen::TracePacket> compressed_packets = - consumer2->ReadBuffers(); - EXPECT_THAT(compressed_packets, Not(IsEmpty())); - EXPECT_THAT(compressed_packets, - Each(Property(&protos::gen::TracePacket::compressed_packets, - Not(IsEmpty())))); -} - -#endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB) - // Note: file_write_period_ms is set to a large enough to have exactly one flush // of the tracing buffers (and therefore at most one synchronization section), // unless the test runs unrealistically slowly, or the implementation of the
diff --git a/src/tracing/service/tracing_service_session.h b/src/tracing/service/tracing_service_session.h index 9bf2e9f..c9fbb14 100644 --- a/src/tracing/service/tracing_service_session.h +++ b/src/tracing/service/tracing_service_session.h
@@ -159,9 +159,6 @@ // Whether we emitted the ProtoVM instances. bool did_emit_protovm_instances_ = false; - // Whether we should compress TracePackets after reading them. - bool compress_deflate = false; - // The number of received triggers we've emitted into the trace output. size_t num_triggers_emitted_into_trace = 0;
diff --git a/src/tracing/service/zlib_compressor.cc b/src/tracing/service/zlib_compressor.cc index a8ad617..a193415 100644 --- a/src/tracing/service/zlib_compressor.cc +++ b/src/tracing/service/zlib_compressor.cc
@@ -25,38 +25,18 @@ #include "protos/perfetto/trace/trace.pbzero.h" #include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "src/tracing/service/packet_compressor_common.h" namespace perfetto { namespace { -struct Preamble { - uint32_t size; - std::array<uint8_t, 16> buf; -}; +using packet_compressor::GetPreamble; +using packet_compressor::kCompressSliceSize; +using packet_compressor::Preamble; +using packet_compressor::PreambleToSlice; -template <uint32_t id> -Preamble GetPreamble(size_t sz) { - Preamble preamble; - uint8_t* ptr = preamble.buf.data(); - constexpr uint32_t tag = protozero::proto_utils::MakeTagLengthDelimited(id); - ptr = protozero::proto_utils::WriteVarInt(tag, ptr); - ptr = protozero::proto_utils::WriteVarInt(sz, ptr); - preamble.size = - static_cast<uint32_t>(reinterpret_cast<uintptr_t>(ptr) - - reinterpret_cast<uintptr_t>(preamble.buf.data())); - PERFETTO_DCHECK(preamble.size < preamble.buf.size()); - return preamble; -} - -Slice PreambleToSlice(const Preamble& preamble) { - Slice slice = Slice::Allocate(preamble.size); - memcpy(slice.own_data(), preamble.buf.data(), preamble.size); - return slice; -} - -// A compressor for `TracePacket`s that uses zlib. The class is exposed for -// testing. +// A compressor for `TracePacket`s that uses zlib. class ZlibPacketCompressor { public: ZlibPacketCompressor(); @@ -93,9 +73,8 @@ } void ZlibPacketCompressor::PushPacket(const TracePacket& packet) { - // We need to be able to tokenize packets in the compressed stream, so we - // prefix a proto preamble to each packet. The compressed stream looks like a - // valid Trace proto. + // Prefix each packet with its proto preamble so the compressed stream itself + // parses as a valid Trace proto, and its packets can be tokenized back out. Preamble preamble = GetPreamble<protos::pbzero::Trace::kPacketFieldNumber>(packet.size()); PushData(preamble.buf.data(), preamble.size); @@ -139,16 +118,16 @@ void ZlibPacketCompressor::NewOutputSlice() { PushCurSlice(); - cur_slice_ = std::make_unique<uint8_t[]>(kZlibCompressSliceSize); + cur_slice_ = std::make_unique<uint8_t[]>(kCompressSliceSize); stream_.next_out = reinterpret_cast<Bytef*>(cur_slice_.get()); - stream_.avail_out = kZlibCompressSliceSize; + stream_.avail_out = kCompressSliceSize; } void ZlibPacketCompressor::PushCurSlice() { if (cur_slice_) { - total_new_slices_size_ += kZlibCompressSliceSize - stream_.avail_out; + total_new_slices_size_ += kCompressSliceSize - stream_.avail_out; new_slices_.push_back(Slice::TakeOwnership( - std::move(cur_slice_), kZlibCompressSliceSize - stream_.avail_out)); + std::move(cur_slice_), kCompressSliceSize - stream_.avail_out)); } }
diff --git a/src/tracing/service/zlib_compressor.h b/src/tracing/service/zlib_compressor.h index b2b1dd9..45af806 100644 --- a/src/tracing/service/zlib_compressor.h +++ b/src/tracing/service/zlib_compressor.h
@@ -25,9 +25,6 @@ namespace perfetto { #if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) -// Matches TracingServiceImpl::kMaxTracePacketSliceSize. Exposed for testing. -static constexpr size_t kZlibCompressSliceSize = 128 * 1024 - 512; - void ZlibCompressFn(std::vector<TracePacket>*); #endif
diff --git a/src/tracing/service/zlib_compressor_unittest.cc b/src/tracing/service/zlib_compressor_unittest.cc deleted file mode 100644 index 3532f56..0000000 --- a/src/tracing/service/zlib_compressor_unittest.cc +++ /dev/null
@@ -1,179 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/tracing/service/zlib_compressor.h" - -#include <random> - -#include <zlib.h> - -#include "protos/perfetto/trace/test_event.gen.h" -#include "protos/perfetto/trace/trace.gen.h" -#include "protos/perfetto/trace/trace_packet.gen.h" -#include "src/tracing/service/tracing_service_impl.h" -#include "test/gtest_and_gmock.h" - -namespace perfetto { -namespace { - -using ::testing::Each; -using ::testing::ElementsAre; -using ::testing::Field; -using ::testing::IsEmpty; -using ::testing::Le; -using ::testing::Not; -using ::testing::Property; -using ::testing::SizeIs; -using tracing_service::TracingServiceImpl; - -template <typename F> -TracePacket CreateTracePacket(F fill_function) { - protos::gen::TracePacket msg; - fill_function(&msg); - std::vector<uint8_t> buf = msg.SerializeAsArray(); - Slice slice = Slice::Allocate(buf.size()); - memcpy(slice.own_data(), buf.data(), buf.size()); - perfetto::TracePacket packet; - packet.AddSlice(std::move(slice)); - return packet; -} - -// Return a copy of the `old` trace packets that owns its own slices data. -TracePacket CopyTracePacket(const TracePacket& old) { - TracePacket ret; - for (const Slice& slice : old.slices()) { - auto new_slice = Slice::Allocate(slice.size); - memcpy(new_slice.own_data(), slice.start, slice.size); - ret.AddSlice(std::move(new_slice)); - } - return ret; -} - -std::vector<TracePacket> CopyTracePackets(const std::vector<TracePacket>& old) { - std::vector<TracePacket> ret; - ret.reserve(old.size()); - for (const TracePacket& trace_packet : old) { - ret.push_back(CopyTracePacket(trace_packet)); - } - return ret; -} -std::string RandomString(size_t size) { - std::default_random_engine rnd(0); - std::uniform_int_distribution<> dist(0, 255); - std::string s; - s.resize(size); - for (size_t i = 0; i < s.size(); i++) - s[i] = static_cast<char>(dist(rnd)); - return s; -} - -std::string Decompress(const std::string& data) { - uint8_t out[1024]; - - z_stream stream{}; - stream.next_in = reinterpret_cast<uint8_t*>(const_cast<char*>(data.data())); - stream.avail_in = static_cast<unsigned int>(data.size()); - - EXPECT_EQ(inflateInit(&stream), Z_OK); - std::string s; - - int ret; - do { - stream.next_out = out; - stream.avail_out = sizeof(out); - ret = inflate(&stream, Z_NO_FLUSH); - EXPECT_NE(ret, Z_STREAM_ERROR); - EXPECT_NE(ret, Z_NEED_DICT); - EXPECT_NE(ret, Z_DATA_ERROR); - EXPECT_NE(ret, Z_MEM_ERROR); - s.append(reinterpret_cast<char*>(out), sizeof(out) - stream.avail_out); - } while (ret != Z_STREAM_END); - - inflateEnd(&stream); - return s; -} - -static_assert(kZlibCompressSliceSize == - TracingServiceImpl::kMaxTracePacketSliceSize); - -TEST(ZlibCompressFnTest, Empty) { - std::vector<TracePacket> packets; - - ZlibCompressFn(&packets); - - EXPECT_THAT(packets, IsEmpty()); -} - -TEST(ZlibCompressFnTest, End2EndCompressAndDecompress) { - std::vector<TracePacket> packets; - - packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { - auto* for_testing = msg->mutable_for_testing(); - for_testing->set_str("abc"); - })); - packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { - auto* for_testing = msg->mutable_for_testing(); - for_testing->set_str("def"); - })); - - ZlibCompressFn(&packets); - - ASSERT_THAT(packets, SizeIs(1)); - protos::gen::TracePacket compressed_packet_proto; - ASSERT_TRUE(compressed_packet_proto.ParseFromString( - packets[0].GetRawBytesForTesting())); - const std::string& data = compressed_packet_proto.compressed_packets(); - EXPECT_THAT(data, Not(IsEmpty())); - protos::gen::Trace subtrace; - ASSERT_TRUE(subtrace.ParseFromString(Decompress(data))); - EXPECT_THAT( - subtrace.packet(), - ElementsAre(Property(&protos::gen::TracePacket::for_testing, - Property(&protos::gen::TestEvent::str, "abc")), - Property(&protos::gen::TracePacket::for_testing, - Property(&protos::gen::TestEvent::str, "def")))); -} - -TEST(ZlibCompressFnTest, MaxSliceSize) { - std::vector<TracePacket> packets; - - constexpr size_t kStopOutputSize = - TracingServiceImpl::kMaxTracePacketSliceSize + 2000; - - TracePacket compressed_packet; - while (compressed_packet.size() < kStopOutputSize) { - packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { - auto* for_testing = msg->mutable_for_testing(); - for_testing->set_str(RandomString(65536)); - })); - { - std::vector<TracePacket> packets_copy = CopyTracePackets(packets); - ZlibCompressFn(&packets_copy); - ASSERT_THAT(packets_copy, SizeIs(1)); - compressed_packet = std::move(packets_copy[0]); - } - } - - EXPECT_GE(compressed_packet.slices().size(), 2u); - ASSERT_GT(compressed_packet.size(), - TracingServiceImpl::kMaxTracePacketSliceSize); - EXPECT_THAT(compressed_packet.slices(), - Each(Field(&Slice::size, - Le(TracingServiceImpl::kMaxTracePacketSliceSize)))); -} - -} // namespace -} // namespace perfetto
diff --git a/src/tracing/service/zstd_compressor.cc b/src/tracing/service/zstd_compressor.cc new file mode 100644 index 0000000..2e522e6 --- /dev/null +++ b/src/tracing/service/zstd_compressor.cc
@@ -0,0 +1,182 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/tracing/service/zstd_compressor.h" + +#include "perfetto/base/build_config.h" + +// File compiles to nothing when the buildflag is off (e.g. SDK opt-out). +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + +#include <memory> + +#include <zstd.h> + +#include "perfetto/base/logging.h" +#include "perfetto/tracing/core/trace_config.h" +#include "protos/perfetto/trace/trace.pbzero.h" +#include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "src/tracing/service/packet_compressor_common.h" + +namespace perfetto { + +namespace { + +using packet_compressor::GetPreamble; +using packet_compressor::kCompressSliceSize; +using packet_compressor::Preamble; +using packet_compressor::PreambleToSlice; + +// A compressor for `TracePacket`s that uses zstd's streaming API: data is fed +// in with ZSTD_compressStream2(ZSTD_e_continue) and the frame is finalized with +// ZSTD_e_end, emitting output in fixed-size slices (kCompressSliceSize). +// zstd API reference: https://facebook.github.io/zstd/zstd_manual.html +class ZstdPacketCompressor { + public: + explicit ZstdPacketCompressor(int level); + ~ZstdPacketCompressor(); + + // Owns a raw ZSTD_CStream*; copying would double-free it on destruction. + ZstdPacketCompressor(const ZstdPacketCompressor&) = delete; + ZstdPacketCompressor& operator=(const ZstdPacketCompressor&) = delete; + + // Can be called multiple times, before Finish() is called. + void PushPacket(const TracePacket& packet); + + // Returns the compressed data. Can be called at most once. After this call, + // the object is unusable (PushPacket should not be called) and must be + // destroyed. + TracePacket Finish(); + + private: + void PushData(const void* data, uint32_t size); + void NewOutputSlice(); + void PushCurSlice(); + + ZSTD_CStream* cstream_ = nullptr; + // Points into `cur_slice_`. Zero-initialized so that the first compression + // call observes a full output buffer and allocates the initial slice. + ZSTD_outBuffer out_ = {/*dst=*/nullptr, /*size=*/0, /*pos=*/0}; + size_t total_new_slices_size_ = 0; + std::vector<Slice> new_slices_; + std::unique_ptr<uint8_t[]> cur_slice_; +}; + +ZstdPacketCompressor::ZstdPacketCompressor(int level) { + cstream_ = ZSTD_createCStream(); + PERFETTO_CHECK(cstream_); + // zstd maps 0 to its default level, clamps levels above its max, and treats + // negatives as fast modes, so any int is safe to pass through here. + size_t rc = ZSTD_CCtx_setParameter(cstream_, ZSTD_c_compressionLevel, level); + PERFETTO_CHECK(!ZSTD_isError(rc)); +} + +ZstdPacketCompressor::~ZstdPacketCompressor() { + ZSTD_freeCStream(cstream_); +} + +void ZstdPacketCompressor::PushPacket(const TracePacket& packet) { + // Prefix each packet with its proto preamble so the compressed stream itself + // parses as a valid Trace proto, and its packets can be tokenized back out. + Preamble preamble = + GetPreamble<protos::pbzero::Trace::kPacketFieldNumber>(packet.size()); + PushData(preamble.buf.data(), preamble.size); + for (const Slice& slice : packet.slices()) { + PushData(slice.start, static_cast<uint32_t>(slice.size)); + } +} + +void ZstdPacketCompressor::PushData(const void* data, uint32_t size) { + // ZSTD_e_continue hands data to the encoder, which buffers and emits at its + // own discretion. It may not consume all input in one call (e.g. when the + // output slice fills), so loop until `in` is drained, giving it a fresh slice + // whenever `out_` is full. + ZSTD_inBuffer in = {/*src=*/data, /*size=*/size, /*pos=*/0}; + while (in.pos < in.size) { + if (out_.pos == out_.size) { + NewOutputSlice(); + } + size_t rc = ZSTD_compressStream2(cstream_, &out_, &in, ZSTD_e_continue); + PERFETTO_CHECK(!ZSTD_isError(rc)); + } +} + +TracePacket ZstdPacketCompressor::Finish() { + // ZSTD_e_end flushes buffered data and writes the frame epilogue. Per the + // contract, keep calling (draining `out_` into new slices) until it reports 0 + // bytes remaining. + size_t remaining; + do { + if (out_.pos == out_.size) { + NewOutputSlice(); + } + ZSTD_inBuffer in = {/*src=*/nullptr, /*size=*/0, /*pos=*/0}; + remaining = ZSTD_compressStream2(cstream_, &out_, &in, ZSTD_e_end); + PERFETTO_CHECK(!ZSTD_isError(remaining)); + } while (remaining != 0); + + PushCurSlice(); + + TracePacket packet; + packet.AddSlice(PreambleToSlice( + GetPreamble<protos::pbzero::TracePacket::kCompressedPacketsFieldNumber>( + total_new_slices_size_))); + for (auto& slice : new_slices_) { + packet.AddSlice(std::move(slice)); + } + return packet; +} + +void ZstdPacketCompressor::NewOutputSlice() { + PushCurSlice(); + cur_slice_ = std::make_unique<uint8_t[]>(kCompressSliceSize); + out_.dst = cur_slice_.get(); + out_.size = kCompressSliceSize; + out_.pos = 0; +} + +void ZstdPacketCompressor::PushCurSlice() { + if (cur_slice_) { + total_new_slices_size_ += out_.pos; + new_slices_.push_back( + Slice::TakeOwnership(std::move(cur_slice_), out_.pos)); + } +} + +} // namespace + +void ZstdCompressFn( + std::vector<TracePacket>* packets, + const protos::gen::TraceConfig_CompressionConfig_Zstd& zstd) { + if (packets->empty()) { + return; + } + + ZstdPacketCompressor stream(zstd.level()); + + for (const TracePacket& packet : *packets) { + stream.PushPacket(packet); + } + + TracePacket packet = stream.Finish(); + + packets->clear(); + packets->push_back(std::move(packet)); +} + +} // namespace perfetto + +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZSTD)
diff --git a/src/tracing/service/zstd_compressor.h b/src/tracing/service/zstd_compressor.h new file mode 100644 index 0000000..03af8fd --- /dev/null +++ b/src/tracing/service/zstd_compressor.h
@@ -0,0 +1,41 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACING_SERVICE_ZSTD_COMPRESSOR_H_ +#define SRC_TRACING_SERVICE_ZSTD_COMPRESSOR_H_ + +#include <vector> + +#include "perfetto/base/build_config.h" +#include "perfetto/ext/tracing/core/trace_packet.h" + +namespace perfetto { +namespace protos { +namespace gen { +class TraceConfig_CompressionConfig_Zstd; +} // namespace gen +} // namespace protos + +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +// Compresses `packets` with zstd, tuned by `zstd`. +void ZstdCompressFn( + std::vector<TracePacket>*, + const protos::gen::TraceConfig_CompressionConfig_Zstd& zstd); +#endif + +} // namespace perfetto + +#endif // SRC_TRACING_SERVICE_ZSTD_COMPRESSOR_H_
diff --git a/test/cts/Android.bp b/test/cts/Android.bp index e37a791..306bca8 100644 --- a/test/cts/Android.bp +++ b/test/cts/Android.bp
@@ -38,6 +38,8 @@ "libandroid", "liblog", "libpcre2", + "libz", + "libzstd", ], test_suites: [ "cts",
diff --git a/test/cts/art_module/Android.bp b/test/cts/art_module/Android.bp index edd32e6..18961d5 100644 --- a/test/cts/art_module/Android.bp +++ b/test/cts/art_module/Android.bp
@@ -32,6 +32,8 @@ "libandroid", "liblog", "libpcre2", + "libz", + "libzstd", ], test_suites: [ "cts",
diff --git a/test/cts/producer/jni/Android.bp b/test/cts/producer/jni/Android.bp index 88583bf..0eba1f7 100644 --- a/test/cts/producer/jni/Android.bp +++ b/test/cts/producer/jni/Android.bp
@@ -22,6 +22,8 @@ "libandroid", "liblog", "libpcre2", + "libz", + "libzstd", ], compile_multilib: "both", stl: "libc++_static",
diff --git a/test/cts/reporter/Android.bp b/test/cts/reporter/Android.bp index de17355..1fb8125 100644 --- a/test/cts/reporter/Android.bp +++ b/test/cts/reporter/Android.bp
@@ -34,6 +34,8 @@ "libandroid", "liblog", "libpcre2", + "libz", + "libzstd", ], test_suites: [ "cts",
diff --git a/test/cts/test_apps/jni/Android.bp b/test/cts/test_apps/jni/Android.bp index b9bcd8c..dd8e9f2 100644 --- a/test/cts/test_apps/jni/Android.bp +++ b/test/cts/test_apps/jni/Android.bp
@@ -35,6 +35,8 @@ "libandroid", "liblog", "libpcre2", + "libz", + "libzstd", ], compile_multilib: "both", stl: "libc++_static",
diff --git a/tools/gen_amalgamated b/tools/gen_amalgamated index 75eeb5b..007bd46 100755 --- a/tools/gen_amalgamated +++ b/tools/gen_amalgamated
@@ -64,9 +64,10 @@ # line). gn_args = ' '.join([ 'enable_perfetto_ipc=true', - # Compile zlib and re2 into the amalgamation; the consumer opts in at + # Compile zlib, zstd and re2 into the amalgamation; the consumer opts in at # use-site (see perfetto_amalgamated_sdk). 'enable_perfetto_zlib=true', + 'enable_perfetto_zstd=true', 'enable_perfetto_re2=true', 'enable_perfetto_pcre2=false', 'is_debug=false',
diff --git a/tools/gen_android_bp b/tools/gen_android_bp index f714670..41811a4 100755 --- a/tools/gen_android_bp +++ b/tools/gen_android_bp
@@ -359,10 +359,13 @@ # SDK is minimal, so std::regex is an acceptable tradeoff. # See b/498785154 for the binary-size regression that motivated this. # - # Same idea for zlib: the compressor would otherwise pull in libz, and - # consumers that don't link it fail with undefined deflate* symbols. - ('cflags', - {'-DPERFETTO_REGEX_FORCE_STD', '-DPERFETTO_FORCE_DISABLE_ZLIB'}), + # Same idea for the compressors: they would otherwise pull in + # libz/libzstd, and consumers that don't link them fail with undefined + # symbols. + ('cflags', { + '-DPERFETTO_REGEX_FORCE_STD', '-DPERFETTO_FORCE_DISABLE_ZLIB', + '-DPERFETTO_FORCE_DISABLE_ZSTD' + }), ], 'libperfetto_c': [ ('min_sdk_version', '30'), @@ -383,10 +386,13 @@ # SDK is minimal, so std::regex is an acceptable tradeoff. # See b/498785154 for the binary-size regression that motivated this. # - # Same idea for zlib: the compressor would otherwise pull in libz, and - # consumers that don't link it fail with undefined deflate* symbols. - ('cflags', - {'-DPERFETTO_REGEX_FORCE_STD', '-DPERFETTO_FORCE_DISABLE_ZLIB'}), + # Same idea for the compressors: they would otherwise pull in + # libz/libzstd, and consumers that don't link them fail with undefined + # symbols. + ('cflags', { + '-DPERFETTO_REGEX_FORCE_STD', '-DPERFETTO_FORCE_DISABLE_ZLIB', + '-DPERFETTO_FORCE_DISABLE_ZSTD' + }), ], 'perfetto_trace_protos': [ ('apex_available', { @@ -497,6 +503,20 @@ module.shared_libs.add('libpcre2') +def enable_zstd(module): + # Modules built with PERFETTO_FORCE_DISABLE_ZSTD compile out the compressor + # and don't reference any zstd symbols, so skip the libzstd dep for them. + if '-DPERFETTO_FORCE_DISABLE_ZSTD' in module.cflags: + return + if module.type == 'cc_binary_host': + module.static_libs.add('libzstd') + elif module.host_supported: + module.android.shared_libs.add('libzstd') + module.host.static_libs.add('libzstd') + else: + module.shared_libs.add('libzstd') + + def enable_sqlite(module): if module.type == 'cc_binary_host': module.static_libs.add('libsqlite_static_noicu') @@ -590,6 +610,8 @@ enable_sqlite, '//gn:zlib': enable_zlib, + '//gn:zstd': + enable_zstd, '//gn:expat': enable_expat, '//gn:bionic_kernel_uapi_headers':
diff --git a/tools/gen_bazel b/tools/gen_bazel index e67c8b5..235d00f 100755 --- a/tools/gen_bazel +++ b/tools/gen_bazel
@@ -201,6 +201,7 @@ 'PERFETTO_CONFIG.deps.sqlite_ext_percentile' ], '//gn:zlib': ['PERFETTO_CONFIG.deps.zlib'], + '//gn:zstd': ['PERFETTO_CONFIG.deps.zstd'], '//gn:llvm_demangle': ['PERFETTO_CONFIG.deps.llvm_demangle'], '//src/trace_processor:demangle': ['PERFETTO_CONFIG.deps.demangle_wrapper'], gn_utils.GEN_VERSION_TARGET: ['PERFETTO_CONFIG.deps.version_header'],