testing/benchmark/displaylist_benchmark_parser.py - mirrors/engine - Git at Google

 #!/usr/bin/env python3
 #
 # Copyright 2013 The Flutter Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 import argparse
 import csv
 import json
 import sys
 import matplotlib.pyplot as plt  # pylint: disable=import-error
 from matplotlib.backends.backend_pdf import PdfPages as pdfp  # pylint: disable=import-error


 class BenchmarkResult:  # pylint: disable=too-many-instance-attributes

   def __init__(self, name, backend, time_unit, draw_call_count):
     self.name = name
     self.series = {}
     self.series_labels = {}
     self.backend = backend
     self.large_y_values = False
     self.y_limit = 200
     self.time_unit = time_unit
     self.draw_call_count = draw_call_count
     self.optional_values = {}

   def __repr__(self):
     return 'Name: % s\nBackend: % s\nSeries: % s\nSeriesLabels: % s\n' % (
         self.name, self.backend, self.series, self.series_labels
     )

   def add_data_point(self, family, xval, yval):
     if family not in self.series:
       self.series[family] = {'x': [], 'y': []}

     self.series[family]['x'].append(xval)
     self.series[family]['y'].append(yval)

     if yval > self.y_limit:
       self.large_y_values = True

   def add_optional_value(self, name, xval, yval):
     if name not in self.optional_values:
       self.optional_values[name] = {}

     self.optional_values[name][xval] = yval

   def set_family_label(self, family, label):
     # I'm not keying the main series dict off the family label
     # just in case we get data where the two aren't a 1:1 mapping
     if family in self.series_labels:
       assert self.series_labels[family] == label
       return

     self.series_labels[family] = label

   def plot(self):
     figures = []
     figures.append(plt.figure(dpi=1200, frameon=False, figsize=(11, 8.5)))

     for family in self.series:
       plt.plot(self.series[family]['x'], self.series[family]['y'], label=self.series_labels[family])

     plt.xlabel('Benchmark Seed')
     plt.ylabel('Time (' + self.time_unit + ')')

     title = ''
     # Crop the Y axis so that we can see what's going on at the lower end
     if self.large_y_values:
       plt.ylim((0, self.y_limit))
       title = self.name + ' ' + self.backend + ' (Cropped)'
     else:
       title = self.name + ' ' + self.backend

     if self.draw_call_count != -1:
       title += '\nDraw Call Count: ' + str(int(self.draw_call_count))

     plt.title(title)

     plt.grid(which='both', axis='both')

     plt.legend(fontsize='xx-small')
     plt.plot()

     if self.large_y_values:
       # Plot again but with the full Y axis visible
       figures.append(plt.figure(dpi=1200, frameon=False, figsize=(11, 8.5)))
       for family in self.series:
         plt.plot(
             self.series[family]['x'], self.series[family]['y'], label=self.series_labels[family]
         )

       plt.xlabel('Benchmark Seed')
       plt.ylabel('Time (' + self.time_unit + ')')
       title = self.name + ' ' + self.backend + ' (Complete)'

       if self.draw_call_count != -1:
         title += '\nDraw Call Count: ' + str(int(self.draw_call_count))

       plt.title(title)

       plt.grid(which='both', axis='both')

       plt.legend(fontsize='xx-small')
       plt.plot()

     return figures

   def write_csv(self, writer):
     # For now assume that all our series have the same x values
     # this is true for now, but may differ in the future with benchmark changes
     x_values = []
     y_values = []
     for family in self.series:
       x_values = ['x'] + self.series[family]['x']
       y_values.append([self.series_labels[family]] + self.series[family]['y'])

     for name in self.optional_values:
       column = [name]
       for key in self.optional_values[name]:
         column.append(self.optional_values[name][key])
       y_values.append(column)

     writer.writerow([self.name, self.draw_call_count])
     for line, _ in enumerate(x_values):
       row = [x_values[line]]
       for series, _ in enumerate(y_values):
         row.append(y_values[series][line])
       writer.writerow(row)


 def main():
   parser = argparse.ArgumentParser()

   parser.add_argument(
       'filename', action='store', help='Path to the JSON output from Google Benchmark'
   )
   parser.add_argument(
       '-o',
       '--output-pdf',
       dest='output_pdf',
       action='store',
       default='output.pdf',
       help='Filename to output the PDF of graphs to.'
   )
   parser.add_argument(
       '-c',
       '--output-csv',
       dest='output_csv',
       action='store',
       default='output.csv',
       help='Filename to output the CSV data to.'
   )

   args = parser.parse_args()
   json_data = parse_json(args.filename)
   return process_benchmark_data(json_data, args.output_pdf, args.output_csv)


 def error(message):
   print(message)
   sys.exit(1)


 def extrac_attributes_label(benchmark_result):
   # Possible attribute keys are:
   #  AntiAliasing
   #  HairlineStroke
   #  StrokedStyle
   #  FilledStyle
   attributes = ['AntiAliasing', 'HairlineStroke', 'StrokedStyle', 'FilledStyle']
   label = ''

   for attr in attributes:
     try:
       if benchmark_result[attr] != 0:
         label += attr + ', '
     except KeyError:
       pass

   return label[:-2]


 def process_benchmark_data(benchmark_json, output_pdf, output_csv):
   benchmark_results_data = {}

   for benchmark_result in benchmark_json:
     # Skip aggregate results
     if 'aggregate_name' in benchmark_result:
       continue

     benchmark_variant = benchmark_result['name'].split('/')
     # The final split is always `real_time` and can be discarded
     benchmark_variant.remove('real_time')

     splits = len(benchmark_variant)
     # First split is always the benchmark function name
     benchmark_name = benchmark_variant[0]
     # The last split is always the seeded value into the benchmark
     benchmark_seeded_value = benchmark_variant[splits - 1]
     # The second last split is always the backend
     benchmark_backend = benchmark_variant[splits - 2]
     # Time taken (wall clock time) for benchmark to run
     benchmark_real_time = benchmark_result['real_time']
     benchmark_unit = benchmark_result['time_unit']

     benchmark_family_index = benchmark_result['family_index']

     benchmark_family_label = ''
     if splits > 3:
       for i in range(1, splits - 2):
         benchmark_family_label += benchmark_variant[i] + ', '

     benchmark_family_attributes = extrac_attributes_label(benchmark_result)

     if benchmark_family_attributes == '':
       benchmark_family_label = benchmark_family_label[:-2]
     else:
       benchmark_family_label = benchmark_family_label + benchmark_family_attributes

     if 'DrawCallCount' in benchmark_result:
       benchmark_draw_call_count = benchmark_result['DrawCallCount']
     else:
       benchmark_draw_call_count = -1

     optional_keys = ['DrawCallCount_Varies', 'VerbCount', 'PointCount', 'VertexCount', 'GlyphCount']

     if benchmark_name not in benchmark_results_data:
       benchmark_results_data[benchmark_name] = BenchmarkResult(
           benchmark_name, benchmark_backend, benchmark_unit, benchmark_draw_call_count
       )

     for key in optional_keys:
       if key in benchmark_result:
         benchmark_results_data[benchmark_name].add_optional_value(
             key, benchmark_seeded_value, benchmark_result[key]
         )

     benchmark_results_data[benchmark_name].add_data_point(
         benchmark_family_index, benchmark_seeded_value, benchmark_real_time
     )
     benchmark_results_data[benchmark_name].set_family_label(
         benchmark_family_index, benchmark_family_label
     )

   pdf = pdfp(output_pdf)

   csv_file = open(output_csv, 'w')
   csv_writer = csv.writer(csv_file)

   for benchmark in benchmark_results_data:
     figures = benchmark_results_data[benchmark].plot()
     for fig in figures:
       pdf.savefig(fig)
     benchmark_results_data[benchmark].write_csv(csv_writer)
   pdf.close()


 def parse_json(filename):
   try:
     json_file = open(filename, 'r')
   except:  # pylint: disable=bare-except
     error('Unable to load file.')

   try:
     json_data = json.load(json_file)
   except JSONDecodeError:  # pylint: disable=undefined-variable
     error('Invalid JSON. Unable to parse.')

   return json_data['benchmarks']


 if __name__ == '__main__':
   sys.exit(main())
	#!/usr/bin/env python3
	#
	# Copyright 2013 The Flutter Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	import argparse
	import csv
	import json
	import sys
	import matplotlib.pyplot as plt # pylint: disable=import-error
	from matplotlib.backends.backend_pdf import PdfPages as pdfp # pylint: disable=import-error


	class BenchmarkResult: # pylint: disable=too-many-instance-attributes

	def __init__(self, name, backend, time_unit, draw_call_count):
	self.name = name
	self.series = {}
	self.series_labels = {}
	self.backend = backend
	self.large_y_values = False
	self.y_limit = 200
	self.time_unit = time_unit
	self.draw_call_count = draw_call_count
	self.optional_values = {}

	def __repr__(self):
	return 'Name: % s\nBackend: % s\nSeries: % s\nSeriesLabels: % s\n' % (
	self.name, self.backend, self.series, self.series_labels
	)

	def add_data_point(self, family, xval, yval):
	if family not in self.series:
	self.series[family] = {'x': [], 'y': []}

	self.series[family]['x'].append(xval)
	self.series[family]['y'].append(yval)

	if yval > self.y_limit:
	self.large_y_values = True

	def add_optional_value(self, name, xval, yval):
	if name not in self.optional_values:
	self.optional_values[name] = {}

	self.optional_values[name][xval] = yval

	def set_family_label(self, family, label):
	# I'm not keying the main series dict off the family label
	# just in case we get data where the two aren't a 1:1 mapping
	if family in self.series_labels:
	assert self.series_labels[family] == label
	return

	self.series_labels[family] = label

	def plot(self):
	figures = []
	figures.append(plt.figure(dpi=1200, frameon=False, figsize=(11, 8.5)))

	for family in self.series:
	plt.plot(self.series[family]['x'], self.series[family]['y'], label=self.series_labels[family])

	plt.xlabel('Benchmark Seed')
	plt.ylabel('Time (' + self.time_unit + ')')

	title = ''
	# Crop the Y axis so that we can see what's going on at the lower end
	if self.large_y_values:
	plt.ylim((0, self.y_limit))
	title = self.name + ' ' + self.backend + ' (Cropped)'
	else:
	title = self.name + ' ' + self.backend

	if self.draw_call_count != -1:
	title += '\nDraw Call Count: ' + str(int(self.draw_call_count))

	plt.title(title)

	plt.grid(which='both', axis='both')

	plt.legend(fontsize='xx-small')
	plt.plot()

	if self.large_y_values:
	# Plot again but with the full Y axis visible
	figures.append(plt.figure(dpi=1200, frameon=False, figsize=(11, 8.5)))
	for family in self.series:
	plt.plot(
	self.series[family]['x'], self.series[family]['y'], label=self.series_labels[family]
	)

	plt.xlabel('Benchmark Seed')
	plt.ylabel('Time (' + self.time_unit + ')')
	title = self.name + ' ' + self.backend + ' (Complete)'

	if self.draw_call_count != -1:
	title += '\nDraw Call Count: ' + str(int(self.draw_call_count))

	plt.title(title)

	plt.grid(which='both', axis='both')

	plt.legend(fontsize='xx-small')
	plt.plot()

	return figures

	def write_csv(self, writer):
	# For now assume that all our series have the same x values
	# this is true for now, but may differ in the future with benchmark changes
	x_values = []
	y_values = []
	for family in self.series:
	x_values = ['x'] + self.series[family]['x']
	y_values.append([self.series_labels[family]] + self.series[family]['y'])

	for name in self.optional_values:
	column = [name]
	for key in self.optional_values[name]:
	column.append(self.optional_values[name][key])
	y_values.append(column)

	writer.writerow([self.name, self.draw_call_count])
	for line, _ in enumerate(x_values):
	row = [x_values[line]]
	for series, _ in enumerate(y_values):
	row.append(y_values[series][line])
	writer.writerow(row)


	def main():
	parser = argparse.ArgumentParser()

	parser.add_argument(
	'filename', action='store', help='Path to the JSON output from Google Benchmark'
	)
	parser.add_argument(
	'-o',
	'--output-pdf',
	dest='output_pdf',
	action='store',
	default='output.pdf',
	help='Filename to output the PDF of graphs to.'
	)
	parser.add_argument(
	'-c',
	'--output-csv',
	dest='output_csv',
	action='store',
	default='output.csv',
	help='Filename to output the CSV data to.'
	)

	args = parser.parse_args()
	json_data = parse_json(args.filename)
	return process_benchmark_data(json_data, args.output_pdf, args.output_csv)


	def error(message):
	print(message)
	sys.exit(1)


	def extrac_attributes_label(benchmark_result):
	# Possible attribute keys are:
	# AntiAliasing
	# HairlineStroke
	# StrokedStyle
	# FilledStyle
	attributes = ['AntiAliasing', 'HairlineStroke', 'StrokedStyle', 'FilledStyle']
	label = ''

	for attr in attributes:
	try:
	if benchmark_result[attr] != 0:
	label += attr + ', '
	except KeyError:
	pass

	return label[:-2]


	def process_benchmark_data(benchmark_json, output_pdf, output_csv):
	benchmark_results_data = {}

	for benchmark_result in benchmark_json:
	# Skip aggregate results
	if 'aggregate_name' in benchmark_result:
	continue

	benchmark_variant = benchmark_result['name'].split('/')
	# The final split is always `real_time` and can be discarded
	benchmark_variant.remove('real_time')

	splits = len(benchmark_variant)
	# First split is always the benchmark function name
	benchmark_name = benchmark_variant[0]
	# The last split is always the seeded value into the benchmark
	benchmark_seeded_value = benchmark_variant[splits - 1]
	# The second last split is always the backend
	benchmark_backend = benchmark_variant[splits - 2]
	# Time taken (wall clock time) for benchmark to run
	benchmark_real_time = benchmark_result['real_time']
	benchmark_unit = benchmark_result['time_unit']

	benchmark_family_index = benchmark_result['family_index']

	benchmark_family_label = ''
	if splits > 3:
	for i in range(1, splits - 2):
	benchmark_family_label += benchmark_variant[i] + ', '

	benchmark_family_attributes = extrac_attributes_label(benchmark_result)

	if benchmark_family_attributes == '':
	benchmark_family_label = benchmark_family_label[:-2]
	else:
	benchmark_family_label = benchmark_family_label + benchmark_family_attributes

	if 'DrawCallCount' in benchmark_result:
	benchmark_draw_call_count = benchmark_result['DrawCallCount']
	else:
	benchmark_draw_call_count = -1

	optional_keys = ['DrawCallCount_Varies', 'VerbCount', 'PointCount', 'VertexCount', 'GlyphCount']

	if benchmark_name not in benchmark_results_data:
	benchmark_results_data[benchmark_name] = BenchmarkResult(
	benchmark_name, benchmark_backend, benchmark_unit, benchmark_draw_call_count
	)

	for key in optional_keys:
	if key in benchmark_result:
	benchmark_results_data[benchmark_name].add_optional_value(
	key, benchmark_seeded_value, benchmark_result[key]
	)

	benchmark_results_data[benchmark_name].add_data_point(
	benchmark_family_index, benchmark_seeded_value, benchmark_real_time
	)
	benchmark_results_data[benchmark_name].set_family_label(
	benchmark_family_index, benchmark_family_label
	)

	pdf = pdfp(output_pdf)

	csv_file = open(output_csv, 'w')
	csv_writer = csv.writer(csv_file)

	for benchmark in benchmark_results_data:
	figures = benchmark_results_data[benchmark].plot()
	for fig in figures:
	pdf.savefig(fig)
	benchmark_results_data[benchmark].write_csv(csv_writer)
	pdf.close()


	def parse_json(filename):
	try:
	json_file = open(filename, 'r')
	except: # pylint: disable=bare-except
	error('Unable to load file.')

	try:
	json_data = json.load(json_file)
	except JSONDecodeError: # pylint: disable=undefined-variable
	error('Invalid JSON. Unable to parse.')

	return json_data['benchmarks']


	if __name__ == '__main__':
	sys.exit(main())