Add node and php to benchmark dashboard
diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index c758ee6..a9c69c7 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am
@@ -260,7 +260,7 @@ @echo 'all_data=""' >> go-benchmark @echo 'conf=()' >> go-benchmark @echo 'data_files=()' >> go-benchmark - @echo 'for arg in $$@; do if [[ $${arg:0:1} == "-" ]]; then conf+=($$arg); else data_files+=("../$$arg"); fi; done' >> go-benchmark + @echo 'for arg in $$@; do if [[ $${arg:0:1} == "-" ]]; then conf+=($$arg); else data_files+=("$$arg"); fi; done' >> go-benchmark @echo 'go test -bench=. $${conf[*]} -- $${data_files[*]}' >> go-benchmark @echo 'cd ..' >> go-benchmark @chmod +x go-benchmark @@ -533,7 +533,6 @@ @echo '#! /bin/bash' > php-c-benchmark @echo 'export PROTOBUF_PHP_SRCDIR="$$(cd $(top_srcdir) && pwd)/php/src"' >> php-c-benchmark @echo 'export PROTOBUF_PHP_EXTDIR="$$PROTOBUF_PHP_SRCDIR/../ext/google/protobuf/modules"' >> php-c-benchmark - @echo 'echo "$$PROTOBUF_PHP_EXTDIR/protobuf.so"' >> php-c-benchmark @echo 'cd tmp/php' >> php-c-benchmark @echo 'export CURRENT_DIR=$$(pwd)' >> php-c-benchmark @echo 'php -d auto_prepend_file="autoload.php" -d include_path="$$(pwd)" -d extension="$$PROTOBUF_PHP_EXTDIR/protobuf.so" Google/Protobuf/Benchmark/PhpBenchmark.php $$@' >> php-c-benchmark
diff --git a/benchmarks/js/js_benchmark.js b/benchmarks/js/js_benchmark.js index 14905e3..875be68 100644 --- a/benchmarks/js/js_benchmark.js +++ b/benchmarks/js/js_benchmark.js
@@ -18,6 +18,7 @@ } var results = []; +var json_file = ""; console.log("#####################################################"); console.log("Js Benchmark: "); @@ -25,6 +26,11 @@ if (index < 2) { return; } + if (filename.indexOf("--json_output") != -1) { + json_file = filename.replace(/^--json_output=/, ''); + return; + } + var benchmarkDataset = proto.benchmarks.BenchmarkDataset.deserializeBinary(fs.readFileSync(filename)); var messageList = []; @@ -55,8 +61,8 @@ results.push({ filename: filename, benchmarks: { - protobufjs_decoding: senarios.benches[0] * totalBytes, - protobufjs_encoding: senarios.benches[1] * totalBytes + protobufjs_decoding: senarios.benches[0] * totalBytes / 1024 / 1024, + protobufjs_encoding: senarios.benches[1] * totalBytes / 1024 / 1024 } }) @@ -68,3 +74,9 @@ }); console.log("#####################################################"); +if (json_file != "") { + fs.writeFile(json_file, JSON.stringify(results), (err) => { + if (err) throw err; + }); +} +
diff --git a/benchmarks/php/PhpBenchmark.php b/benchmarks/php/PhpBenchmark.php index 058940d..9c1132d 100644 --- a/benchmarks/php/PhpBenchmark.php +++ b/benchmarks/php/PhpBenchmark.php
@@ -93,7 +93,7 @@ } } -function runBenchmark($file) { +function runBenchmark($file, $behavior_prefix) { $datafile = fopen($file, "r") or die("Unable to open file " . $file); $bytes = fread($datafile, filesize($file)); $dataset = new BenchmarkDataset(NULL); @@ -119,8 +119,8 @@ return array( "filename" => $file, "benchmarks" => array( - "parse_php" => $parse_benchmark->runBenchmark(), - "serailize_php" => $serialize_benchmark->runBenchmark() + $behavior_prefix . "_parse" => $parse_benchmark->runBenchmark(), + $behavior_prefix . "_serailize" => $serialize_benchmark->runBenchmark() ), "message_name" => $dataset->getMessageName() ); @@ -129,15 +129,27 @@ // main $json_output = false; $results = array(); +$behavior_prefix = ""; + foreach ($argv as $index => $arg) { if ($index == 0) { continue; } if ($arg == "--json") { $json_output = true; + } else if (strpos($arg, "--behavior_prefix") == 0) { + $behavior_prefix = str_replace("--behavior_prefix=", "", $arg); + } +} + +foreach ($argv as $index => $arg) { + if ($index == 0) { + continue; + } + if (substr($arg, 0, 2) == "--") { continue; } else { - array_push($results, runBenchmark($arg)); + array_push($results, runBenchmark($arg, $behavior_prefix)); } }
diff --git a/benchmarks/python/py_benchmark.py b/benchmarks/python/py_benchmark.py index e86b61e..ebb1974 100755 --- a/benchmarks/python/py_benchmark.py +++ b/benchmarks/python/py_benchmark.py
@@ -44,9 +44,13 @@ data = open(filename).read() benchmark_dataset = benchmarks_pb2.BenchmarkDataset() benchmark_dataset.ParseFromString(data) + total_bytes = 0 + for payload in benchmark_dataset.payload: + total_bytes += len(payload) benchmark_util = Benchmark(full_iteration=len(benchmark_dataset.payload), module="py_benchmark", - setup_method="init") + setup_method="init", + total_bytes=total_bytes) result={} result["filename"] = filename result["message_name"] = benchmark_dataset.message_name @@ -61,10 +65,11 @@ def init(filename): - global benchmark_dataset, message_class, message_list, counter + global benchmark_dataset, message_class, message_list, counter, total_bytes message_list=[] counter = 0 - data = open(os.path.dirname(sys.argv[0]) + "/../" + filename).read() + total_bytes = 0 + data = open(filename).read() benchmark_dataset = benchmarks_pb2.BenchmarkDataset() benchmark_dataset.ParseFromString(data) @@ -85,6 +90,7 @@ temp = message_class() temp.ParseFromString(one_payload) message_list.append(temp) + total_bytes += len(one_payload) def parse_from_benchmark(): @@ -101,11 +107,12 @@ class Benchmark: def __init__(self, module=None, test_method=None, - setup_method=None, full_iteration = 1): + setup_method=None, total_bytes=None, full_iteration = 1): self.full_iteration = full_iteration self.module = module self.test_method = test_method self.setup_method = setup_method + self.total_bytes = total_bytes def set_test_method(self, test_method): self.test_method = test_method @@ -130,7 +137,7 @@ t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args), setup=self.full_setup_code(setup_method_args), number=reps); - return 1.0 * t / reps * (10 ** 9) + return self.total_bytes * 1.0 / 2 ** 20 / (1.0 * t / reps) if __name__ == "__main__": @@ -144,10 +151,10 @@ for result in results: print("Message %s of dataset file %s" % \ (result["message_name"], result["filename"])) - print("Average time for parse_from_benchmark: %.2f ns" % \ + print("Average throughput for parse_from_benchmark: %.2f MB/s" % \ (result["benchmarks"][ \ args.behavior_prefix + "_parse_from_benchmark"])) - print("Average time for serialize_to_benchmark: %.2f ns" % \ + print("Average throughput for serialize_to_benchmark: %.2f MB/s" % \ (result["benchmarks"][ \ args.behavior_prefix + "_serialize_to_benchmark"])) print("")
diff --git a/benchmarks/util/result_parser.py b/benchmarks/util/result_parser.py index 32f35a9..0b8fc49 100755 --- a/benchmarks/util/result_parser.py +++ b/benchmarks/util/result_parser.py
@@ -115,7 +115,6 @@ # behavior: results, # ... # }, -# "message_name": STRING # }, # ... # ], #pure-python @@ -136,8 +135,7 @@ "language": "python", "dataFilename": __extract_file_name(result["filename"]), "behavior": behavior, - "throughput": avg_size / - result["benchmarks"][behavior] * 1e9 / 2 ** 20 + "throughput": result["benchmarks"][behavior] }) @@ -220,7 +218,7 @@ continue first_slash_index = result_list[0].find('/') last_slash_index = result_list[0].rfind('/') - full_filename = result_list[0][first_slash_index+4:last_slash_index] # delete ../ prefix + full_filename = result_list[0][first_slash_index+1:last_slash_index] total_bytes, _ = __get_data_size(full_filename) behavior_with_suffix = result_list[0][last_slash_index+1:] last_dash = behavior_with_suffix.rfind("-") @@ -236,11 +234,45 @@ }) +# Node/Php results example: +# +# [ +# { +# "filename": string, +# "benchmarks": { +# behavior: results, +# ... +# }, +# }, +# ... +# ] +def __parse_js_php_result(filename, language): + if filename == "": + return + if filename[0] != '/': + filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename + with open(filename) as f: + results = json.loads(f.read()) + for result in results: + _, avg_size = __get_data_size(result["filename"]) + for behavior in result["benchmarks"]: + __results.append({ + "language": language, + "dataFilename": __extract_file_name(result["filename"]), + "behavior": behavior, + "throughput": result["benchmarks"][behavior] + }) + + + def get_result_from_file(cpp_file="", java_file="", python_file="", go_file="", - synthetic_file=""): + synthetic_file="", + node_file="", + php_c_file="", + php_file=""): results = {} if cpp_file != "": __parse_cpp_result(cpp_file) @@ -252,5 +284,11 @@ __parse_go_result(go_file) if synthetic_file != "": __parse_synthetic_result(synthetic_file) + if node_file != "": + __parse_js_php_result(node_file, "node") + if php_file != "": + __parse_js_php_result(php_file, "php") + if php_c_file != "": + __parse_js_php_result(php_c_file, "php") return __results
diff --git a/benchmarks/util/result_uploader.py b/benchmarks/util/result_uploader.py index a667da0..9167caf 100755 --- a/benchmarks/util/result_uploader.py +++ b/benchmarks/util/result_uploader.py
@@ -59,13 +59,14 @@ labels_string += ",|%s:%s|" % (key, result[key]) new_result["labels"] = labels_string[1:] new_result["timestamp"] = _INITIAL_TIME - - bq = big_query_utils.create_big_query() - row = big_query_utils.make_row(str(uuid.uuid4()), new_result) - if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET, - _TABLE + "$" + _NOW, - [row]): - print('Error when uploading result', new_result) + print(labels_string) +# +# bq = big_query_utils.create_big_query() +# row = big_query_utils.make_row(str(uuid.uuid4()), new_result) +# if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET, +# _TABLE + "$" + _NOW, +# [row]): +# print('Error when uploading result', new_result) if __name__ == "__main__": @@ -82,6 +83,15 @@ parser.add_argument("-go", "--go_input_file", help="The golang benchmark result file's name", default="") + parser.add_argument("-node", "--node_input_file", + help="The node.js benchmark result file's name", + default="") + parser.add_argument("-php", "--php_input_file", + help="The pure php benchmark result file's name", + default="") + parser.add_argument("-php_c", "--php_c_input_file", + help="The php with c ext benchmark result file's name", + default="") args = parser.parse_args() metadata = get_metadata() @@ -90,5 +100,8 @@ cpp_file=args.cpp_input_file, java_file=args.java_input_file, python_file=args.python_input_file, - go_file=args.go_input_file + go_file=args.go_input_file, + node_file=args.node_input_file, + php_file=args.php_input_file, + php_c_file=args.php_c_input_file, ), metadata) \ No newline at end of file