tp: add HprofDump data source for embedding heap dumps in traces Add HprofDumpDataSource, a system_server Perfetto data source that triggers AMS.dumpHeap() on a target process and writes the raw .hprof bytes into the trace as chunked HprofDump packets. The data source streams the hprof file in 512KB chunks via FileInputStream to avoid loading the full dump (typically 200-400MB) into memory. Each chunk is written as a TracePacket with chunk_index and last_chunk fields so the trace processor can finalize per-pid dumps independently. The target process pid from HprofDumpConfig is propagated to each packet so the TP module can demux multiple concurrent dumps by pid. Change-Id: I0000000000000000000000000000000000000005
diff --git a/src/java_datasource/android_hprof_bitmaps/HprofDumpDataSource.java b/src/java_datasource/android_hprof_bitmaps/HprofDumpDataSource.java index 192e3bc..056eca7 100644 --- a/src/java_datasource/android_hprof_bitmaps/HprofDumpDataSource.java +++ b/src/java_datasource/android_hprof_bitmaps/HprofDumpDataSource.java
@@ -32,9 +32,9 @@ import dev.perfetto.sdk.TraceContext; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.lang.ref.WeakReference; -import java.nio.file.Files; /** * Perfetto data source that triggers a Java heap dump on a target process @@ -46,7 +46,11 @@ public final class HprofDumpDataSource extends PerfettoDataSource { private static final String TAG = "HprofDumpDataSource"; private static final String DUMP_DIR = "/data/local/tmp/perfetto_hprof"; - private static final int CHUNK_SIZE = 4 * 1024 * 1024; + // Keep chunks well under the typical shmem page budget. 512KB is + // large enough to avoid excessive packet overhead but small enough + // that each TracePacket fits comfortably in the tracing service + // shared memory buffer. + private static final int CHUNK_SIZE = 512 * 1024; // TracePacket field numbers. private static final int TP_TIMESTAMP = 8; @@ -169,8 +173,9 @@ | ParcelFileDescriptor.MODE_WRITE_ONLY | ParcelFileDescriptor.MODE_TRUNCATE); + long targetPid = mConfigPid; RemoteCallback callback = new RemoteCallback( - result -> onDumpComplete(dumpDir, hprofPath)); + result -> onDumpComplete(dumpDir, hprofPath, targetPid)); ams.dumpHeap(process, UserHandle.USER_CURRENT, true, false, runGc, @@ -183,11 +188,11 @@ } } - private void onDumpComplete(File dumpDir, String hprofPath) { + private void onDumpComplete(File dumpDir, String hprofPath, long pid) { try { File hprofFile = new File(hprofPath); if (hprofFile.exists() && hprofFile.length() > 0) { - writeHprofPackets(hprofFile); + writeHprofPackets(hprofFile, (int) pid); } } catch (Exception e) { Log.e(TAG, "Failed to write dump to trace", e); @@ -196,34 +201,51 @@ } } - private void writeHprofPackets(File file) throws IOException { - byte[] data = Files.readAllBytes(file.toPath()); - int totalChunks = (data.length + CHUNK_SIZE - 1) / CHUNK_SIZE; + private void writeHprofPackets(File file, int targetPid) throws IOException { + byte[] buf = new byte[CHUNK_SIZE]; + long totalBytes = 0; + int chunkIndex = 0; - for (int i = 0; i < totalChunks; i++) { - int offset = i * CHUNK_SIZE; - int len = Math.min(CHUNK_SIZE, data.length - offset); + try (FileInputStream fis = new FileInputStream(file)) { + int bytesRead; + while ((bytesRead = readFully(fis, buf)) > 0) { + totalBytes += bytesRead; + boolean isLast = (bytesRead < buf.length); - TraceContext ctx = trace(); - if (ctx == null) return; + TraceContext ctx = trace(); + if (ctx == null) return; - ProtoWriter w = ctx.getWriter(); - w.writeVarInt(TP_TIMESTAMP, SystemClock.elapsedRealtimeNanos()); - w.writeVarInt(TP_CLOCK_ID, 6); - w.writeVarInt(TP_SEQ_ID, 1); - int dump = w.beginNested(TP_HPROF_DUMP); - w.writeVarInt(HD_PID, android.os.Process.myPid()); - w.writeBytes(HD_HPROF_DATA, data, offset, len); - w.writeVarInt(HD_CHUNK_INDEX, i); - if (i == totalChunks - 1) { - w.writeVarInt(HD_LAST_CHUNK, 1); + ProtoWriter w = ctx.getWriter(); + w.writeVarInt(TP_TIMESTAMP, SystemClock.elapsedRealtimeNanos()); + w.writeVarInt(TP_CLOCK_ID, 6); + w.writeVarInt(TP_SEQ_ID, 1); + int dump = w.beginNested(TP_HPROF_DUMP); + w.writeVarInt(HD_PID, targetPid); + w.writeBytes(HD_HPROF_DATA, buf, 0, bytesRead); + w.writeVarInt(HD_CHUNK_INDEX, chunkIndex); + if (isLast) { + w.writeVarInt(HD_LAST_CHUNK, 1); + } + w.endNested(dump); + ctx.commitPacket(); + chunkIndex++; } - w.endNested(dump); - ctx.commitPacket(); } - Log.i(TAG, "Wrote " + data.length + " bytes hprof (" - + ((data.length + CHUNK_SIZE - 1) / CHUNK_SIZE) + " chunks)"); + Log.i(TAG, "Wrote " + totalBytes + " bytes hprof (" + + chunkIndex + " chunks) for pid " + targetPid); + } + + /** Reads up to buf.length bytes, looping to handle partial reads. */ + private static int readFully(FileInputStream fis, byte[] buf) + throws IOException { + int total = 0; + while (total < buf.length) { + int n = fis.read(buf, total, buf.length - total); + if (n < 0) break; + total += n; + } + return total; } private static void deleteRecursive(File dir) {
diff --git a/src/java_datasource/android_hprof_bitmaps/README.md b/src/java_datasource/android_hprof_bitmaps/README.md index bdfa92b..61cd526 100644 --- a/src/java_datasource/android_hprof_bitmaps/README.md +++ b/src/java_datasource/android_hprof_bitmaps/README.md
@@ -4,7 +4,7 @@ A Perfetto data source that triggers a Java heap dump (`.hprof`) on a target Android process and embeds the raw hprof binary in the trace as -a packet. Optionally also extracts bitmap images as PNGs. +chunked packets. Optionally also extracts bitmap images as PNGs. This runs in **system_server** (AOSP `frameworks/base`) and uses the existing `ActivityManagerService.dumpHeap()` mechanism. @@ -13,14 +13,14 @@ 1. Receives trace config with target process pid/cmdline 2. Calls `AMS.dumpHeap(process, managed=true, dumpBitmaps, path, fd, cb)` -3. Target process runs `Debug.dumpHprofData()` → `.hprof` file -4. Optionally target process runs `Bitmap.dumpAll("png")` → PNG files -5. On completion callback: reads the files, writes trace packets +3. Target process runs `Debug.dumpHprofData()` -> `.hprof` file +4. Optionally target process runs `Bitmap.dumpAll("png")` -> PNG files +5. On completion callback: streams file in 512KB chunks as trace packets 6. Cleans up temp files ## Proto -### Config (new, in Perfetto repo) +### Config ```proto // protos/perfetto/config/profiling/hprof_dump_config.proto @@ -33,121 +33,44 @@ } ``` -### Trace output (new, in Perfetto repo) +### Trace output ```proto // protos/perfetto/trace/profiling/hprof_dump.proto message HprofDump { - // Process ID of the dumped process. - optional int32 pid = 1; - - // Raw .hprof binary data. Can be split across multiple packets - // if the dump is large (use continued flag on TracePacket). - optional bytes hprof_data = 2; + optional int32 pid = 1; // target process pid + optional bytes hprof_data = 2; // chunk of raw .hprof binary + optional uint32 chunk_index = 3; // zero-based chunk index + optional bool last_chunk = 4; // true on final chunk for this pid } ``` -Add to TracePacket: -```proto -HprofDump hprof_dump = <next_field>; -``` +The trace processor groups chunks by `pid` and finalizes each dump +when `last_chunk` is received. Multiple dumps (different pids or +sequential same-pid dumps) can coexist in one trace. -For bitmaps, reuse the `VideoFrame` proto or add: -```proto -message HprofBitmap { - optional int32 pid = 1; - optional string filename = 2; // e.g., "bitmap_0.png" - optional bytes png_image = 3; -} -``` +## Trace processor -## Trace processor changes +`HprofDumpModule` handles `TracePacket.hprof_dump`: +1. Maintains a per-pid `ArtHprofParser` instance +2. Feeds `hprof_data` chunks via `ArtHprofParser::Parse()` +3. On `last_chunk`: calls `OnPushDataToSorter()` to populate + `heap_graph_class`, `heap_graph_object`, `heap_graph_reference` +4. Any incomplete dumps are finalized at trace end -### Option A: Route raw bytes to ArtHprofParser +No new tables -- reuses the existing heap graph infrastructure. -Add a module that handles `TracePacket.hprof_dump`: -1. Extracts the `hprof_data` bytes -2. Feeds them to `ArtHprofParser::Parse()` as `TraceBlobView` chunks -3. `ArtHprofParser` populates the existing `heap_graph_*` tables +## Chunking -This reuses all existing hprof parsing -- no new tables needed. - -### Option B: Store raw bytes as BLOB - -Store the raw hprof bytes in a BLOB vector (like VideoFrame) and -expose via `hprof_dump_data(id)` SQL function. This lets the UI -download the raw `.hprof` file. - -**Recommended: both.** Parse into heap_graph tables AND store raw -bytes so the user can also download the original file. - -## System server data source (AOSP) - -### `HprofDumpDataSource.java` - -```java -public class HprofDumpDataSource extends PerfettoDataSource { - static { INSTANCE.register("android.hprof_dump"); } - - @Override - protected void onStart(int instanceIndex, byte[] config) { - // 1. Parse HprofDumpConfig from config bytes - // 2. Find target process via AMS - // 3. Create temp dir: /data/local/tmp/hprof_<sessionId>/ - // 4. Create ParcelFileDescriptor for output - // 5. Call AMS.dumpHeap(process, managed=true, - // dumpBitmaps=config.dump_bitmaps ? "png" : null, - // path, fd, finishCallback) - // 6. In finishCallback (runs when dump complete): - // a. Read .hprof file bytes - // b. Write as TracePacket { hprof_dump { pid, hprof_data } } - // Split into multiple packets if >4MB - // c. If bitmap dump enabled: - // Read each PNG file - // Write as TracePacket { hprof_bitmap { pid, filename, png } } - // d. commitPacket() - // e. Delete temp files - } -} -``` - -### Splitting large hprof files - -A heap dump can be 50-200MB. This won't fit in a single trace packet. -Use the `continued` flag on TracePacket to split across multiple -packets on the same sequence: - -```java -byte[] hprofBytes = Files.readAllBytes(hprofPath); -int chunkSize = 4 * 1024 * 1024; // 4MB chunks -for (int offset = 0; offset < hprofBytes.length; offset += chunkSize) { - int len = Math.min(chunkSize, hprofBytes.length - offset); - ProtoWriter w = ctx.getWriter(); - // Write timestamp, sequence_id, etc. - int dump = w.beginNested(HPROF_DUMP_FIELD); - w.writeVarInt(1, pid); - w.writeBytes(2, hprofBytes, offset, len); - w.endNested(dump); - ctx.commitPacket(); -} -``` - -### Shmem buffer sizing - -Hprof dumps are large. Set `shmem_size_hint_kb = 8192` (8MB). -Use `PERFETTO_DS_BUFFER_EXHAUSTED_POLICY_STALL_AND_ABORT`. - -### Threading - -The dump is async -- `AMS.dumpHeap()` returns immediately, the target -process does the work, then calls the finish callback. The data source -should defer stop until the callback fires (like LayerDataSource's -`HandleStopAsynchronously()`). +Hprof dumps are typically 200-400MB. The data source streams the file +in 512KB chunks to avoid loading the entire dump into memory. Each +chunk becomes one `TracePacket` with `HprofDump { chunk_index, ... }`. +The final chunk sets `last_chunk = true`. ## Trace config example ``` -buffers { size_kb: 262144 } # 256MB for large hprof +buffers { size_kb: 524288 } # 512MB for large hprof data_sources { config { name: "android.hprof_dump" @@ -163,14 +86,13 @@ ## Testing -On device: ```sh # Existing mechanism (works today): adb shell am dumpheap -b png com.example.app /data/local/tmp/dump.hprof # With Perfetto (after this data source is built): adb shell perfetto -c - --txt <<EOF -buffers { size_kb: 262144 } +buffers { size_kb: 524288 } data_sources { config { name: "android.hprof_dump" hprof_dump_config { process_cmdline: "com.example.app" run_gc: true } } } @@ -178,13 +100,5 @@ EOF ``` -Load the trace in Perfetto UI → heap graph tables populated from the +Load the trace in Perfetto UI -> heap graph tables populated from the embedded hprof data, alongside any other concurrent trace data. - -## Perfetto repo changes needed - -1. Config proto: `HprofDumpConfig` -2. Trace proto: `HprofDump` (raw bytes), `HprofBitmap` (PNG bytes) -3. Trace processor module: routes `hprof_dump` bytes to `ArtHprofParser` -4. Optional: BLOB storage + SQL function for raw hprof download -5. Optional: UI plugin for bitmap gallery view