tools/gen_merged_protos

#!/usr/bin/env python
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import re
import sys

CONFIG_PROTOS = (
  'protos/perfetto/common/sys_stats_counters.proto',
  'protos/perfetto/config/chrome/chrome_config.proto',
  'protos/perfetto/config/data_source_config.proto',
  'protos/perfetto/config/ftrace/ftrace_config.proto',
  'protos/perfetto/config/inode_file/inode_file_config.proto',
  'protos/perfetto/config/process_stats/process_stats_config.proto',
  'protos/perfetto/config/sys_stats/sys_stats_config.proto',
  'protos/perfetto/config/test_config.proto',
  'protos/perfetto/config/trace_config.proto',
  'protos/perfetto/config/profiling/heapprofd_config.proto',
)

MERGED_CONFIG_PROTO = 'protos/perfetto/config/perfetto_config.proto'

TRACE_PROTOS = (
  'protos/perfetto/common/sys_stats_counters.proto',
  'protos/perfetto/trace/trace.proto',
  'protos/perfetto/trace/trace_packet.proto',
  'protos/perfetto/trace/ftrace/ftrace_event_bundle.proto',
  'protos/perfetto/trace/ftrace/ftrace_event.proto',
  'protos/perfetto/trace/filesystem/inode_file_map.proto',
  'protos/perfetto/trace/ps/process_stats.proto',
  'protos/perfetto/trace/ps/process_tree.proto',
  'protos/perfetto/trace/clock_snapshot.proto',
  'protos/perfetto/trace/sys_stats/sys_stats.proto',

  # Print proto is special: it doesn't have a enable file so is
  # not present in genfs_contexts.
  'protos/perfetto/trace/ftrace/print.proto',

  # FTrace events
  # Generated by tools/protos_from_genfs_contexts.sh.
  # MISSING cpufreq_interactive_load_change
  'protos/perfetto/trace/ftrace/binder_locked.proto',
  'protos/perfetto/trace/ftrace/binder_lock.proto',
  'protos/perfetto/trace/ftrace/binder_transaction.proto',
  'protos/perfetto/trace/ftrace/binder_transaction_received.proto',
  'protos/perfetto/trace/ftrace/binder_unlock.proto',
  'protos/perfetto/trace/ftrace/block_rq_complete.proto',
  'protos/perfetto/trace/ftrace/block_rq_issue.proto',
  'protos/perfetto/trace/ftrace/cgroup_attach_task.proto',
  'protos/perfetto/trace/ftrace/cgroup_destroy_root.proto',
  'protos/perfetto/trace/ftrace/cgroup_mkdir.proto',
  'protos/perfetto/trace/ftrace/cgroup_release.proto',
  'protos/perfetto/trace/ftrace/cgroup_remount.proto',
  'protos/perfetto/trace/ftrace/cgroup_rename.proto',
  'protos/perfetto/trace/ftrace/cgroup_rmdir.proto',
  'protos/perfetto/trace/ftrace/cgroup_setup_root.proto',
  'protos/perfetto/trace/ftrace/cgroup_transfer_tasks.proto',
  'protos/perfetto/trace/ftrace/clock_set_rate.proto',
  'protos/perfetto/trace/ftrace/cpufreq_interactive_already.proto',
  'protos/perfetto/trace/ftrace/cpufreq_interactive_boost.proto',
  'protos/perfetto/trace/ftrace/cpufreq_interactive_notyet.proto',
  'protos/perfetto/trace/ftrace/cpufreq_interactive_setspeed.proto',
  'protos/perfetto/trace/ftrace/cpufreq_interactive_target.proto',
  'protos/perfetto/trace/ftrace/cpufreq_interactive_unboost.proto',
  'protos/perfetto/trace/ftrace/cpu_frequency_limits.proto',
  'protos/perfetto/trace/ftrace/cpu_frequency.proto',
  'protos/perfetto/trace/ftrace/cpu_idle.proto',
  'protos/perfetto/trace/ftrace/ext4_da_write_begin.proto',
  'protos/perfetto/trace/ftrace/ext4_da_write_end.proto',
  'protos/perfetto/trace/ftrace/ext4_es_lookup_extent_enter.proto',
  'protos/perfetto/trace/ftrace/ext4_es_lookup_extent_exit.proto',
  'protos/perfetto/trace/ftrace/ext4_load_inode.proto',
  'protos/perfetto/trace/ftrace/ext4_sync_file_enter.proto',
  'protos/perfetto/trace/ftrace/ext4_sync_file_exit.proto',
  'protos/perfetto/trace/ftrace/f2fs_get_data_block.proto',
  'protos/perfetto/trace/ftrace/f2fs_iget.proto',
  'protos/perfetto/trace/ftrace/f2fs_sync_file_enter.proto',
  'protos/perfetto/trace/ftrace/f2fs_sync_file_exit.proto',
  'protos/perfetto/trace/ftrace/f2fs_write_begin.proto',
  'protos/perfetto/trace/ftrace/f2fs_write_end.proto',
  'protos/perfetto/trace/ftrace/lowmemory_kill.proto',
  'protos/perfetto/trace/ftrace/mm_vmscan_direct_reclaim_begin.proto',
  'protos/perfetto/trace/ftrace/mm_vmscan_direct_reclaim_end.proto',
  'protos/perfetto/trace/ftrace/mm_vmscan_kswapd_sleep.proto',
  'protos/perfetto/trace/ftrace/mm_vmscan_kswapd_wake.proto',
  'protos/perfetto/trace/ftrace/sched_blocked_reason.proto',
  'protos/perfetto/trace/ftrace/sched_cpu_hotplug.proto',
  'protos/perfetto/trace/ftrace/sched_switch.proto',
  'protos/perfetto/trace/ftrace/sched_wakeup.proto',
)

MERGED_TRACE_PROTO = 'protos/perfetto/trace/perfetto_trace.proto'

REPLACEMENT_HEADER = '''
// AUTOGENERATED - DO NOT EDIT
// ---------------------------
// This file has been generated by
// AOSP://external/perfetto/%s
// merging the perfetto config protos.
// This fused proto is intended to be copied in:
//  - Android tree, for statsd.
//  - Google internal repos.

syntax = "proto2";

package perfetto.protos;
'''

def merge_protos(proto_paths, output_path):
  root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
  merged_content = ''
  for proto in proto_paths:
    path = os.path.join(root_dir, proto)
    with open(path) as f:
      content = f.read()

    # Remove header
    header = re.match(r'\/(\*|\/)(?:.|\s)*?package.*;\n', content)
    header = header.group(0)
    content = content[len(header):]
    if merged_content == '':
      merged_content += REPLACEMENT_HEADER.lstrip() % __file__
    content = re.sub(r'^import.*?\n', '', content, flags=re.MULTILINE)
    content = re.sub(r'TODO\([^)]*\):', 'TODO:', content)
    merged_content += '\n// Begin of %s\n' % proto
    merged_content += content
    merged_content += '\n// End of %s\n' % proto

  definitions_re = r'^ *(?:message|enum) ([A-Z][A-Za-z0-9].*) {'
  definitions = re.finditer(definitions_re, merged_content, re.MULTILINE)
  types = set((match.group(1) for match in definitions))

  uses_re = r'^( +)(?:repeated)?\s?([A-Z]\w+.*)\s+[a-z]\w+\s*=\s*(\d+);'
  uses = re.finditer(uses_re, merged_content, re.MULTILINE)
  substitutions = []
  for use in uses:
    everything = use.group(0)
    indentation = use.group(1)
    used_type = use.group(2)
    field_number = use.group(3)
    if used_type not in types:
      replacement = '{}// removed field with id {}'.format(
          indentation, field_number)
      substitutions.append((everything, replacement))

  for before, after in substitutions:
    merged_content = merged_content.replace(before, after)

  out_path = os.path.join(root_dir, output_path)

  prev_content = None
  if os.path.exists(out_path):
    with open(out_path, 'rb') as fprev:
      prev_content = fprev.read()

  if prev_content == merged_content:
    return True

  if '--check-only' in sys.argv:
    return False

  print('Updating {}'.format(output_path))
  with open(out_path, 'wb') as fout:
    fout.write(merged_content)
  return True

def main():
  config_result = merge_protos(CONFIG_PROTOS, MERGED_CONFIG_PROTO)
  trace_result = merge_protos(TRACE_PROTOS, MERGED_TRACE_PROTO)
  return 0 if config_result and trace_result else 1

if __name__ == '__main__':
  sys.exit(main())