| #!/usr/bin/env python3 |
| # |
| # Copyright 2013 The Flutter Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import argparse |
| import csv |
| import json |
| import sys |
| import matplotlib.pyplot as plt |
| from matplotlib.backends.backend_pdf import PdfPages as pdfp |
| |
| |
| class BenchmarkResult: |
| |
| def __init__(self, name, backend, timeUnit, drawCallCount): |
| self.name = name |
| self.series = {} |
| self.seriesLabels = {} |
| self.backend = backend |
| self.largeYValues = False |
| self.yLimit = 200 |
| self.timeUnit = timeUnit |
| self.drawCallCount = drawCallCount |
| self.optionalValues = {} |
| |
| def __repr__(self): |
| return 'Name: % s\nBackend: % s\nSeries: % s\nSeriesLabels: % s\n' % ( |
| self.name, self.backend, self.series, self.seriesLabels |
| ) |
| |
| def addDataPoint(self, family, x, y): |
| if family not in self.series: |
| self.series[family] = {'x': [], 'y': []} |
| |
| self.series[family]['x'].append(x) |
| self.series[family]['y'].append(y) |
| |
| if y > self.yLimit: |
| self.largeYValues = True |
| |
| def addOptionalValue(self, name, x, y): |
| if name not in self.optionalValues: |
| self.optionalValues[name] = {} |
| |
| self.optionalValues[name][x] = y |
| |
| def setFamilyLabel(self, family, label): |
| # I'm not keying the main series dict off the family label |
| # just in case we get data where the two aren't a 1:1 mapping |
| if family in self.seriesLabels: |
| assert self.seriesLabels[family] == label |
| return |
| |
| self.seriesLabels[family] = label |
| |
| def plot(self): |
| figures = [] |
| figures.append(plt.figure(dpi=1200, frameon=False, figsize=(11, 8.5))) |
| |
| for family in self.series: |
| plt.plot( |
| self.series[family]['x'], |
| self.series[family]['y'], |
| label=self.seriesLabels[family] |
| ) |
| |
| plt.xlabel('Benchmark Seed') |
| plt.ylabel('Time (' + self.timeUnit + ')') |
| |
| title = '' |
| # Crop the Y axis so that we can see what's going on at the lower end |
| if self.largeYValues: |
| plt.ylim((0, self.yLimit)) |
| title = self.name + ' ' + self.backend + ' (Cropped)' |
| else: |
| title = self.name + ' ' + self.backend |
| |
| if self.drawCallCount != -1: |
| title += '\nDraw Call Count: ' + str(int(self.drawCallCount)) |
| |
| plt.title(title) |
| |
| plt.grid(which='both', axis='both') |
| |
| plt.legend(fontsize='xx-small') |
| plt.plot() |
| |
| if self.largeYValues: |
| # Plot again but with the full Y axis visible |
| figures.append(plt.figure(dpi=1200, frameon=False, figsize=(11, 8.5))) |
| for family in self.series: |
| plt.plot( |
| self.series[family]['x'], |
| self.series[family]['y'], |
| label=self.seriesLabels[family] |
| ) |
| |
| plt.xlabel('Benchmark Seed') |
| plt.ylabel('Time (' + self.timeUnit + ')') |
| title = self.name + ' ' + self.backend + ' (Complete)' |
| |
| if self.drawCallCount != -1: |
| title += '\nDraw Call Count: ' + str(int(self.drawCallCount)) |
| |
| plt.title(title) |
| |
| plt.grid(which='both', axis='both') |
| |
| plt.legend(fontsize='xx-small') |
| plt.plot() |
| |
| return figures |
| |
| def writeCSV(self, writer): |
| # For now assume that all our series have the same x values |
| # this is true for now, but may differ in the future with benchmark changes |
| x_values = [] |
| y_values = [] |
| for family in self.series: |
| x_values = ['x'] + self.series[family]['x'] |
| y_values.append([self.seriesLabels[family]] + self.series[family]['y']) |
| |
| for name in self.optionalValues: |
| column = [name] |
| for key in self.optionalValues[name]: |
| column.append(self.optionalValues[name][key]) |
| y_values.append(column) |
| |
| writer.writerow([self.name, self.drawCallCount]) |
| for line in range(len(x_values)): |
| row = [x_values[line]] |
| for series in range(len(y_values)): |
| row.append(y_values[series][line]) |
| writer.writerow(row) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser() |
| |
| parser.add_argument( |
| 'filename', |
| action='store', |
| help='Path to the JSON output from Google Benchmark' |
| ) |
| parser.add_argument( |
| '-o', |
| '--output-pdf', |
| dest='outputPDF', |
| action='store', |
| default='output.pdf', |
| help='Filename to output the PDF of graphs to.' |
| ) |
| parser.add_argument( |
| '-c', |
| '--output-csv', |
| dest='outputCSV', |
| action='store', |
| default='output.csv', |
| help='Filename to output the CSV data to.' |
| ) |
| |
| args = parser.parse_args() |
| jsonData = parseJSON(args.filename) |
| return processBenchmarkData(jsonData, args.outputPDF, args.outputCSV) |
| |
| |
| def error(message): |
| print(message) |
| exit(1) |
| |
| |
| def extractAttributesLabel(benchmarkResult): |
| # Possible attribute keys are: |
| # AntiAliasing |
| # HairlineStroke |
| # StrokedStyle |
| # FilledStyle |
| attributes = ['AntiAliasing', 'HairlineStroke', 'StrokedStyle', 'FilledStyle'] |
| label = '' |
| |
| for attr in attributes: |
| try: |
| if benchmarkResult[attr] != 0: |
| label += attr + ', ' |
| except KeyError: |
| pass |
| |
| return label[:-2] |
| |
| |
| def processBenchmarkData(benchmarkJSON, outputPDF, outputCSV): |
| benchmarkResultsData = {} |
| |
| for benchmarkResult in benchmarkJSON: |
| # Skip aggregate results |
| if 'aggregate_name' in benchmarkResult: |
| continue |
| |
| benchmarkVariant = benchmarkResult['name'].split('/') |
| # The final split is always `real_time` and can be discarded |
| benchmarkVariant.remove('real_time') |
| |
| splits = len(benchmarkVariant) |
| # First split is always the benchmark function name |
| benchmarkName = benchmarkVariant[0] |
| # The last split is always the seeded value into the benchmark |
| benchmarkSeededValue = benchmarkVariant[splits - 1] |
| # The second last split is always the backend |
| benchmarkBackend = benchmarkVariant[splits - 2] |
| # Time taken (wall clock time) for benchmark to run |
| benchmarkRealTime = benchmarkResult['real_time'] |
| benchmarkUnit = benchmarkResult['time_unit'] |
| |
| benchmarkFamilyIndex = benchmarkResult['family_index'] |
| |
| benchmarkFamilyLabel = '' |
| if splits > 3: |
| for i in range(1, splits - 2): |
| benchmarkFamilyLabel += benchmarkVariant[i] + ', ' |
| |
| benchmarkFamilyAttributes = extractAttributesLabel(benchmarkResult) |
| |
| if benchmarkFamilyAttributes == '': |
| benchmarkFamilyLabel = benchmarkFamilyLabel[:-2] |
| else: |
| benchmarkFamilyLabel = benchmarkFamilyLabel + benchmarkFamilyAttributes |
| |
| if 'DrawCallCount' in benchmarkResult: |
| benchmarkDrawCallCount = benchmarkResult['DrawCallCount'] |
| else: |
| benchmarkDrawCallCount = -1 |
| |
| optional_keys = [ |
| 'DrawCallCount_Varies', 'VerbCount', 'PointCount', 'VertexCount', |
| 'GlyphCount' |
| ] |
| |
| if benchmarkName not in benchmarkResultsData: |
| benchmarkResultsData[benchmarkName] = BenchmarkResult( |
| benchmarkName, benchmarkBackend, benchmarkUnit, benchmarkDrawCallCount |
| ) |
| |
| for key in optional_keys: |
| if key in benchmarkResult: |
| benchmarkResultsData[benchmarkName].addOptionalValue( |
| key, benchmarkSeededValue, benchmarkResult[key] |
| ) |
| |
| benchmarkResultsData[benchmarkName].addDataPoint( |
| benchmarkFamilyIndex, benchmarkSeededValue, benchmarkRealTime |
| ) |
| benchmarkResultsData[benchmarkName].setFamilyLabel( |
| benchmarkFamilyIndex, benchmarkFamilyLabel |
| ) |
| |
| pp = pdfp(outputPDF) |
| |
| csv_file = open(outputCSV, 'w') |
| csv_writer = csv.writer(csv_file) |
| |
| for benchmark in benchmarkResultsData: |
| figures = benchmarkResultsData[benchmark].plot() |
| for fig in figures: |
| pp.savefig(fig) |
| benchmarkResultsData[benchmark].writeCSV(csv_writer) |
| pp.close() |
| |
| |
| def parseJSON(filename): |
| try: |
| jsonFile = open(filename, 'r') |
| except: |
| error('Unable to load file.') |
| |
| try: |
| jsonData = json.load(jsonFile) |
| except JSONDecodeError: |
| error('Invalid JSON. Unable to parse.') |
| |
| return jsonData['benchmarks'] |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |