Added framework for generating/consuming benchmarking data sets.
This takes the code that was sitting in benchmarks/
already and makes it easier for language-specific
benchmarks to consume. Future PRs will enhance this
so that the language-specific benchmarks can report
metrics back that will be tracked over time in PerfKit.
diff --git a/benchmarks/benchmarks.proto b/benchmarks/benchmarks.proto
new file mode 100644
index 0000000..a891eb9
--- /dev/null
+++ b/benchmarks/benchmarks.proto
@@ -0,0 +1,102 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+package benchmarks;
+option java_package = "com.google.protobuf.benchmarks";
+
+message BenchmarkDataset {
+ // Name of the benchmark dataset. This should be unique across all datasets.
+ // Should only contain word characters: [a-zA-Z0-9_]
+ string name = 1;
+
+ // Fully-qualified name of the protobuf message for this dataset.
+ // It will be one of the messages defined benchmark_messages.proto.
+ // Implementations that do not support reflection can implement this with
+ // an explicit "if/else" chain that lists every possible message defined
+ // in this file.
+ string message_name = 2;
+
+ // The payload(s) for this dataset. They should be parsed or serialized
+ // in sequence, in a loop, ie.
+ //
+ // while (!benchmarkDone) { // Benchmark runner decides when to exit.
+ // for (i = 0; i < benchmark.payload.length; i++) {
+ // parse(benchmark.payload[i])
+ // }
+ // }
+ //
+ // This is intended to let datasets include a variety of data to provide
+ // potentially more realistic results than just parsing the same message
+ // over and over. A single message parsed repeatedly could yield unusually
+ // good branch prediction performance.
+ repeated bytes payload = 3;
+}
+
+// A benchmark can write out metrics that we will then upload to our metrics
+// database for tracking over time.
+message Metric {
+ // A unique ID for these results. Used for de-duping.
+ string guid = 1;
+
+ // The tags specify exactly what benchmark was run against the dataset.
+ // The specific benchmark suite can decide what these mean, but here are
+ // some common tags that have a predefined meaning:
+ //
+ // - "dataset": for tests that pertain to a specific dataset.
+ //
+ // For example:
+ //
+ // # Tests parsing from binary proto string using arenas.
+ // tags={
+ // dataset: "testalltypes",
+ // op: "parse",
+ // format: "binaryproto",
+ // input: "string"
+ // arena: "true"
+ // }
+ //
+ // # Tests serializing to JSON string.
+ // tags={
+ // dataset: "testalltypes",
+ // op: "serialize",
+ // format: "json",
+ // input: "string"
+ // }
+ map<string, string> labels = 2;
+
+ // Unit of measurement for the metric:
+ // - a speed test might be "mb_per_second" or "ops_per_second"
+ // - a size test might be "kb".
+ string unit = 3;
+
+ // Metric value.
+ double value = 4;
+}
