| /* |
| * Copyright (C) 2020 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #include "src/profiling/perf/event_config.h" |
| |
| #include <linux/perf_event.h> |
| #include <time.h> |
| |
| #include <unwindstack/Regs.h> |
| #include <vector> |
| |
| #include "perfetto/base/flat_set.h" |
| #include "perfetto/ext/base/optional.h" |
| #include "perfetto/ext/base/utils.h" |
| #include "src/profiling/perf/regs_parsing.h" |
| |
| #include "protos/perfetto/common/perf_events.gen.h" |
| #include "protos/perfetto/config/profiling/perf_event_config.gen.h" |
| |
| namespace perfetto { |
| namespace profiling { |
| |
| namespace { |
| constexpr uint64_t kDefaultSamplingFrequencyHz = 10; |
| constexpr uint32_t kDefaultDataPagesPerRingBuffer = 256; // 1 MB: 256x 4k pages |
| constexpr uint32_t kDefaultReadTickPeriodMs = 100; |
| constexpr uint32_t kDefaultRemoteDescriptorTimeoutMs = 100; |
| |
| // Acceptable forms: "sched/sched_switch" or "sched:sched_switch". |
| std::pair<std::string, std::string> SplitTracepointString( |
| const std::string& input) { |
| auto slash_pos = input.find("/"); |
| if (slash_pos != std::string::npos) |
| return std::make_pair(input.substr(0, slash_pos), |
| input.substr(slash_pos + 1)); |
| |
| auto colon_pos = input.find(":"); |
| if (colon_pos != std::string::npos) |
| return std::make_pair(input.substr(0, colon_pos), |
| input.substr(colon_pos + 1)); |
| |
| return std::make_pair("", input); |
| } |
| |
| // If set, the returned id is guaranteed to be non-zero. |
| base::Optional<uint32_t> ParseTracepointAndResolveId( |
| const protos::gen::PerfEvents::Tracepoint& tracepoint, |
| EventConfig::tracepoint_id_fn_t tracepoint_id_lookup) { |
| std::string full_name = tracepoint.name(); |
| std::string tp_group; |
| std::string tp_name; |
| std::tie(tp_group, tp_name) = SplitTracepointString(full_name); |
| if (tp_group.empty() || tp_name.empty()) { |
| PERFETTO_ELOG( |
| "Invalid tracepoint format: %s. Should be a full path like " |
| "sched:sched_switch or sched/sched_switch.", |
| full_name.c_str()); |
| return base::nullopt; |
| } |
| |
| uint32_t tracepoint_id = tracepoint_id_lookup(tp_group, tp_name); |
| if (!tracepoint_id) { |
| PERFETTO_ELOG( |
| "Failed to resolve tracepoint %s to its id. Check that tracefs is " |
| "accessible and the event exists.", |
| full_name.c_str()); |
| return base::nullopt; |
| } |
| return base::make_optional(tracepoint_id); |
| } |
| |
| // |T| is either gen::PerfEventConfig or gen::PerfEventConfig::Scope. |
| // Note: the semantics of target_cmdline and exclude_cmdline were changed since |
| // their original introduction. They used to be put through a canonicalization |
| // function that simplified them to the binary name alone. We no longer do this, |
| // regardless of whether we're parsing an old-style config. The overall outcome |
| // shouldn't change for almost all existing uses. |
| template <typename T> |
| TargetFilter ParseTargetFilter( |
| const T& cfg, |
| base::Optional<ProcessSharding> process_sharding) { |
| TargetFilter filter; |
| for (const auto& str : cfg.target_cmdline()) { |
| filter.cmdlines.push_back(str); |
| } |
| for (const auto& str : cfg.exclude_cmdline()) { |
| filter.exclude_cmdlines.push_back(str); |
| } |
| for (const int32_t pid : cfg.target_pid()) { |
| filter.pids.insert(pid); |
| } |
| for (const int32_t pid : cfg.exclude_pid()) { |
| filter.exclude_pids.insert(pid); |
| } |
| filter.additional_cmdline_count = cfg.additional_cmdline_count(); |
| filter.process_sharding = process_sharding; |
| return filter; |
| } |
| |
| constexpr bool IsPowerOfTwo(size_t v) { |
| return (v != 0 && ((v & (v - 1)) == 0)); |
| } |
| |
| // returns |base::nullopt| if the input is invalid. |
| base::Optional<uint32_t> ChooseActualRingBufferPages(uint32_t config_value) { |
| if (!config_value) { |
| static_assert(IsPowerOfTwo(kDefaultDataPagesPerRingBuffer), ""); |
| return base::make_optional(kDefaultDataPagesPerRingBuffer); |
| } |
| |
| if (!IsPowerOfTwo(config_value)) { |
| PERFETTO_ELOG("kernel buffer size must be a power of two pages"); |
| return base::nullopt; |
| } |
| |
| return base::make_optional(config_value); |
| } |
| |
| base::Optional<PerfCounter> ToPerfCounter( |
| std::string name, |
| protos::gen::PerfEvents::Counter pb_enum) { |
| using protos::gen::PerfEvents; |
| switch (static_cast<int>(pb_enum)) { // cast to pacify -Wswitch-enum |
| case PerfEvents::SW_CPU_CLOCK: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::SW_CPU_CLOCK, |
| PERF_TYPE_SOFTWARE, |
| PERF_COUNT_SW_CPU_CLOCK); |
| case PerfEvents::SW_PAGE_FAULTS: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::SW_PAGE_FAULTS, |
| PERF_TYPE_SOFTWARE, |
| PERF_COUNT_SW_PAGE_FAULTS); |
| case PerfEvents::SW_TASK_CLOCK: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::SW_TASK_CLOCK, |
| PERF_TYPE_SOFTWARE, |
| PERF_COUNT_SW_TASK_CLOCK); |
| case PerfEvents::SW_CONTEXT_SWITCHES: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::SW_CONTEXT_SWITCHES, |
| PERF_TYPE_SOFTWARE, |
| PERF_COUNT_SW_CONTEXT_SWITCHES); |
| case PerfEvents::SW_CPU_MIGRATIONS: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::SW_CPU_MIGRATIONS, |
| PERF_TYPE_SOFTWARE, |
| PERF_COUNT_SW_CPU_MIGRATIONS); |
| case PerfEvents::SW_PAGE_FAULTS_MIN: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::SW_PAGE_FAULTS_MIN, |
| PERF_TYPE_SOFTWARE, |
| PERF_COUNT_SW_PAGE_FAULTS_MIN); |
| case PerfEvents::SW_PAGE_FAULTS_MAJ: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::SW_PAGE_FAULTS_MAJ, |
| PERF_TYPE_SOFTWARE, |
| PERF_COUNT_SW_PAGE_FAULTS_MAJ); |
| case PerfEvents::SW_ALIGNMENT_FAULTS: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::SW_ALIGNMENT_FAULTS, |
| PERF_TYPE_SOFTWARE, |
| PERF_COUNT_SW_ALIGNMENT_FAULTS); |
| case PerfEvents::SW_EMULATION_FAULTS: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::SW_EMULATION_FAULTS, |
| PERF_TYPE_SOFTWARE, |
| PERF_COUNT_SW_EMULATION_FAULTS); |
| case PerfEvents::SW_DUMMY: |
| return PerfCounter::BuiltinCounter( |
| name, PerfEvents::SW_DUMMY, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_DUMMY); |
| |
| case PerfEvents::HW_CPU_CYCLES: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::HW_CPU_CYCLES, |
| PERF_TYPE_HARDWARE, |
| PERF_COUNT_HW_CPU_CYCLES); |
| case PerfEvents::HW_INSTRUCTIONS: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::HW_INSTRUCTIONS, |
| PERF_TYPE_HARDWARE, |
| PERF_COUNT_HW_INSTRUCTIONS); |
| case PerfEvents::HW_CACHE_REFERENCES: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::HW_CACHE_REFERENCES, |
| PERF_TYPE_HARDWARE, |
| PERF_COUNT_HW_CACHE_REFERENCES); |
| case PerfEvents::HW_CACHE_MISSES: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::HW_CACHE_MISSES, |
| PERF_TYPE_HARDWARE, |
| PERF_COUNT_HW_CACHE_MISSES); |
| case PerfEvents::HW_BRANCH_INSTRUCTIONS: |
| return PerfCounter::BuiltinCounter( |
| name, PerfEvents::HW_BRANCH_INSTRUCTIONS, PERF_TYPE_HARDWARE, |
| PERF_COUNT_HW_BRANCH_INSTRUCTIONS); |
| case PerfEvents::HW_BRANCH_MISSES: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::HW_BRANCH_MISSES, |
| PERF_TYPE_HARDWARE, |
| PERF_COUNT_HW_BRANCH_MISSES); |
| case PerfEvents::HW_BUS_CYCLES: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::HW_BUS_CYCLES, |
| PERF_TYPE_HARDWARE, |
| PERF_COUNT_HW_BUS_CYCLES); |
| case PerfEvents::HW_STALLED_CYCLES_FRONTEND: |
| return PerfCounter::BuiltinCounter( |
| name, PerfEvents::HW_STALLED_CYCLES_FRONTEND, PERF_TYPE_HARDWARE, |
| PERF_COUNT_HW_STALLED_CYCLES_FRONTEND); |
| case PerfEvents::HW_STALLED_CYCLES_BACKEND: |
| return PerfCounter::BuiltinCounter( |
| name, PerfEvents::HW_STALLED_CYCLES_BACKEND, PERF_TYPE_HARDWARE, |
| PERF_COUNT_HW_STALLED_CYCLES_BACKEND); |
| case PerfEvents::HW_REF_CPU_CYCLES: |
| return PerfCounter::BuiltinCounter(name, PerfEvents::HW_REF_CPU_CYCLES, |
| PERF_TYPE_HARDWARE, |
| PERF_COUNT_HW_REF_CPU_CYCLES); |
| |
| default: |
| PERFETTO_ELOG("Unrecognised PerfEvents::Counter enum value: %zu", |
| static_cast<size_t>(pb_enum)); |
| return base::nullopt; |
| } |
| } |
| |
| int32_t ToClockId(protos::gen::PerfEvents::PerfClock pb_enum) { |
| using protos::gen::PerfEvents; |
| switch (static_cast<int>(pb_enum)) { // cast to pacify -Wswitch-enum |
| case PerfEvents::PERF_CLOCK_REALTIME: |
| return CLOCK_REALTIME; |
| case PerfEvents::PERF_CLOCK_MONOTONIC: |
| return CLOCK_MONOTONIC; |
| case PerfEvents::PERF_CLOCK_MONOTONIC_RAW: |
| return CLOCK_MONOTONIC_RAW; |
| case PerfEvents::PERF_CLOCK_BOOTTIME: |
| return CLOCK_BOOTTIME; |
| // Default to a monotonic clock since it should be compatible with all types |
| // of events. Whereas boottime cannot be used with hardware events due to |
| // potential access within non-maskable interrupts. |
| default: |
| return CLOCK_MONOTONIC_RAW; |
| } |
| } |
| |
| } // namespace |
| |
| // static |
| PerfCounter PerfCounter::BuiltinCounter( |
| std::string name, |
| protos::gen::PerfEvents::Counter counter, |
| uint32_t type, |
| uint64_t config) { |
| PerfCounter ret; |
| ret.type = PerfCounter::Type::kBuiltinCounter; |
| ret.counter = counter; |
| ret.name = std::move(name); |
| |
| ret.attr_type = type; |
| ret.attr_config = config; |
| // none of the builtin counters require config1 and config2 at the moment |
| return ret; |
| } |
| |
| // static |
| PerfCounter PerfCounter::Tracepoint(std::string name, |
| std::string tracepoint_name, |
| std::string tracepoint_filter, |
| uint64_t id) { |
| PerfCounter ret; |
| ret.type = PerfCounter::Type::kTracepoint; |
| ret.tracepoint_name = std::move(tracepoint_name); |
| ret.tracepoint_filter = std::move(tracepoint_filter); |
| ret.name = std::move(name); |
| |
| ret.attr_type = PERF_TYPE_TRACEPOINT; |
| ret.attr_config = id; |
| return ret; |
| } |
| |
| // static |
| PerfCounter PerfCounter::RawEvent(std::string name, |
| uint32_t type, |
| uint64_t config, |
| uint64_t config1, |
| uint64_t config2) { |
| PerfCounter ret; |
| ret.type = PerfCounter::Type::kRawEvent; |
| ret.name = std::move(name); |
| |
| ret.attr_type = type; |
| ret.attr_config = config; |
| ret.attr_config1 = config1; |
| ret.attr_config2 = config2; |
| return ret; |
| } |
| |
| // static |
| base::Optional<EventConfig> EventConfig::Create( |
| const protos::gen::PerfEventConfig& pb_config, |
| const DataSourceConfig& raw_ds_config, |
| base::Optional<ProcessSharding> process_sharding, |
| tracepoint_id_fn_t tracepoint_id_lookup) { |
| // Timebase: sampling interval. |
| uint64_t sampling_frequency = 0; |
| uint64_t sampling_period = 0; |
| if (pb_config.timebase().period()) { |
| sampling_period = pb_config.timebase().period(); |
| } else if (pb_config.timebase().frequency()) { |
| sampling_frequency = pb_config.timebase().frequency(); |
| } else if (pb_config.sampling_frequency()) { // backwards compatibility |
| sampling_frequency = pb_config.sampling_frequency(); |
| } else { |
| sampling_frequency = kDefaultSamplingFrequencyHz; |
| } |
| PERFETTO_DCHECK(sampling_period && !sampling_frequency || |
| !sampling_period && sampling_frequency); |
| |
| // Timebase event. Default: CPU timer. |
| PerfCounter timebase_event; |
| std::string timebase_name = pb_config.timebase().name(); |
| if (pb_config.timebase().has_counter()) { |
| auto maybe_counter = |
| ToPerfCounter(timebase_name, pb_config.timebase().counter()); |
| if (!maybe_counter) |
| return base::nullopt; |
| timebase_event = *maybe_counter; |
| |
| } else if (pb_config.timebase().has_tracepoint()) { |
| const auto& tracepoint_pb = pb_config.timebase().tracepoint(); |
| base::Optional<uint32_t> maybe_id = |
| ParseTracepointAndResolveId(tracepoint_pb, tracepoint_id_lookup); |
| if (!maybe_id) |
| return base::nullopt; |
| timebase_event = PerfCounter::Tracepoint( |
| timebase_name, tracepoint_pb.name(), tracepoint_pb.filter(), *maybe_id); |
| |
| } else if (pb_config.timebase().has_raw_event()) { |
| const auto& raw = pb_config.timebase().raw_event(); |
| timebase_event = PerfCounter::RawEvent( |
| timebase_name, raw.type(), raw.config(), raw.config1(), raw.config2()); |
| |
| } else { |
| timebase_event = PerfCounter::BuiltinCounter( |
| timebase_name, protos::gen::PerfEvents::PerfEvents::SW_CPU_CLOCK, |
| PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CPU_CLOCK); |
| } |
| |
| // Callstack sampling. |
| bool user_frames = false; |
| bool kernel_frames = false; |
| TargetFilter target_filter; |
| bool legacy_config = pb_config.all_cpus(); // all_cpus was mandatory before |
| if (pb_config.has_callstack_sampling() || legacy_config) { |
| user_frames = true; |
| |
| // Userspace callstacks. |
| using protos::gen::PerfEventConfig; |
| switch (static_cast<int>(pb_config.callstack_sampling().user_frames())) { |
| case PerfEventConfig::UNWIND_UNKNOWN: |
| // default to true, both for backwards compatibility and because it's |
| // almost always what the user wants. |
| user_frames = true; |
| break; |
| case PerfEventConfig::UNWIND_SKIP: |
| user_frames = false; |
| break; |
| case PerfEventConfig::UNWIND_DWARF: |
| user_frames = true; |
| break; |
| default: |
| // enum value from the future that we don't yet know, refuse the config |
| // TODO(rsavitski): double-check that both pbzero and ::gen propagate |
| // unknown enum values. |
| return base::nullopt; |
| } |
| |
| // Process scoping. Sharding parameter is supplied from outside as it is |
| // shared by all data sources within a tracing session. |
| target_filter = |
| pb_config.callstack_sampling().has_scope() |
| ? ParseTargetFilter(pb_config.callstack_sampling().scope(), |
| process_sharding) |
| : ParseTargetFilter(pb_config, |
| process_sharding); // backwards compatibility |
| |
| // Kernel callstacks. |
| kernel_frames = pb_config.callstack_sampling().kernel_frames() || |
| pb_config.kernel_frames(); |
| } |
| |
| // Ring buffer options. |
| base::Optional<uint32_t> ring_buffer_pages = |
| ChooseActualRingBufferPages(pb_config.ring_buffer_pages()); |
| if (!ring_buffer_pages.has_value()) |
| return base::nullopt; |
| |
| uint32_t read_tick_period_ms = pb_config.ring_buffer_read_period_ms() |
| ? pb_config.ring_buffer_read_period_ms() |
| : kDefaultReadTickPeriodMs; |
| |
| // Calculate a rough upper limit for the amount of samples the producer |
| // should read per read tick, as a safeguard against getting stuck chasing the |
| // ring buffer head indefinitely. |
| uint64_t samples_per_tick_limit = 0; |
| if (sampling_frequency) { |
| // expected = rate * period, with a conversion of period from ms to s: |
| uint64_t expected_samples_per_tick = |
| 1 + (sampling_frequency * read_tick_period_ms) / 1000; |
| // Double the limit to account of actual sample rate uncertainties, as |
| // well as any other factors: |
| samples_per_tick_limit = 2 * expected_samples_per_tick; |
| } else { // sampling_period |
| // We don't know the sample rate that a fixed period would cause, but we can |
| // still estimate how many samples will fit in one pass of the ring buffer |
| // (with the assumption that we don't want to read more than one buffer's |
| // capacity within a tick). |
| // TODO(rsavitski): for now, make an extremely conservative guess of an 8 |
| // byte sample (stack sampling samples can be up to 64KB). This is most |
| // likely as good as no limit in practice. |
| samples_per_tick_limit = *ring_buffer_pages * (base::kPageSize / 8); |
| } |
| PERFETTO_DLOG("Capping samples (not records) per tick to [%" PRIu64 "]", |
| samples_per_tick_limit); |
| if (samples_per_tick_limit == 0) |
| return base::nullopt; |
| |
| // Optional footprint controls. |
| uint64_t max_enqueued_footprint_bytes = |
| pb_config.max_enqueued_footprint_kb() * 1024; |
| |
| // Android-specific options. |
| uint32_t remote_descriptor_timeout_ms = |
| pb_config.remote_descriptor_timeout_ms() |
| ? pb_config.remote_descriptor_timeout_ms() |
| : kDefaultRemoteDescriptorTimeoutMs; |
| |
| // Build the underlying syscall config struct. |
| perf_event_attr pe = {}; |
| pe.size = sizeof(perf_event_attr); |
| pe.disabled = 1; // will be activated via ioctl |
| |
| // Sampling timebase. |
| pe.type = timebase_event.attr_type; |
| pe.config = timebase_event.attr_config; |
| pe.config1 = timebase_event.attr_config1; |
| pe.config2 = timebase_event.attr_config2; |
| if (sampling_frequency) { |
| pe.freq = true; |
| pe.sample_freq = sampling_frequency; |
| } else { |
| pe.sample_period = sampling_period; |
| } |
| |
| // What the samples will contain. |
| pe.sample_type = PERF_SAMPLE_TID | PERF_SAMPLE_TIME | PERF_SAMPLE_READ; |
| // PERF_SAMPLE_TIME: |
| pe.clockid = ToClockId(pb_config.timebase().timestamp_clock()); |
| pe.use_clockid = true; |
| |
| if (user_frames) { |
| pe.sample_type |= PERF_SAMPLE_STACK_USER | PERF_SAMPLE_REGS_USER; |
| // PERF_SAMPLE_STACK_USER: |
| // Needs to be < ((u16)(~0u)), and have bottom 8 bits clear. |
| // Note that the kernel still needs to make space for the other parts of the |
| // sample (up to the max record size of 64k), so the effective maximum |
| // can be lower than this. |
| pe.sample_stack_user = (1u << 16) - 256; |
| // PERF_SAMPLE_REGS_USER: |
| pe.sample_regs_user = |
| PerfUserRegsMaskForArch(unwindstack::Regs::CurrentArch()); |
| } |
| if (kernel_frames) { |
| pe.sample_type |= PERF_SAMPLE_CALLCHAIN; |
| pe.exclude_callchain_user = true; |
| } |
| |
| return EventConfig( |
| raw_ds_config, pe, timebase_event, user_frames, kernel_frames, |
| std::move(target_filter), ring_buffer_pages.value(), read_tick_period_ms, |
| samples_per_tick_limit, remote_descriptor_timeout_ms, |
| pb_config.unwind_state_clear_period_ms(), max_enqueued_footprint_bytes, |
| pb_config.target_installed_by()); |
| } |
| |
| EventConfig::EventConfig(const DataSourceConfig& raw_ds_config, |
| const perf_event_attr& pe, |
| const PerfCounter& timebase_event, |
| bool user_frames, |
| bool kernel_frames, |
| TargetFilter target_filter, |
| uint32_t ring_buffer_pages, |
| uint32_t read_tick_period_ms, |
| uint64_t samples_per_tick_limit, |
| uint32_t remote_descriptor_timeout_ms, |
| uint32_t unwind_state_clear_period_ms, |
| uint64_t max_enqueued_footprint_bytes, |
| std::vector<std::string> target_installed_by) |
| : perf_event_attr_(pe), |
| timebase_event_(timebase_event), |
| user_frames_(user_frames), |
| kernel_frames_(kernel_frames), |
| target_filter_(std::move(target_filter)), |
| ring_buffer_pages_(ring_buffer_pages), |
| read_tick_period_ms_(read_tick_period_ms), |
| samples_per_tick_limit_(samples_per_tick_limit), |
| remote_descriptor_timeout_ms_(remote_descriptor_timeout_ms), |
| unwind_state_clear_period_ms_(unwind_state_clear_period_ms), |
| max_enqueued_footprint_bytes_(max_enqueued_footprint_bytes), |
| target_installed_by_(std::move(target_installed_by)), |
| raw_ds_config_(raw_ds_config) /* full copy */ {} |
| |
| } // namespace profiling |
| } // namespace perfetto |