Fix benchmark to use docker image on kokoro (#5481)
* Fix benchmark to use docker image on kokoro
* Deleted pure php benchmark and fix long time benchmark
diff --git a/benchmarks/php/PhpBenchmark.php b/benchmarks/php/PhpBenchmark.php
index 9c1132d..7220638 100644
--- a/benchmarks/php/PhpBenchmark.php
+++ b/benchmarks/php/PhpBenchmark.php
@@ -62,7 +62,7 @@
$t = $this->runBenchmarkWithTimes(1);
$times = ceil($this->benchmark_time / $t);
return $this->total_bytes * $times /
- $this->runBenchmarkWithTimes($times) *
+ ($times == 1 ? $t : $this->runBenchmarkWithTimes($times)) *
$this->coefficient;
}
diff --git a/benchmarks/python/py_benchmark.py b/benchmarks/python/py_benchmark.py
index ebb1974..40c7af5 100755
--- a/benchmarks/python/py_benchmark.py
+++ b/benchmarks/python/py_benchmark.py
@@ -134,9 +134,10 @@
t = self.dry_run(test_method_args, setup_method_args);
if t < 3 :
reps = int(math.ceil(3 / t)) * self.full_iteration
- t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args),
- setup=self.full_setup_code(setup_method_args),
- number=reps);
+ if reps != self.full_iteration:
+ t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args),
+ setup=self.full_setup_code(setup_method_args),
+ number=reps);
return self.total_bytes * 1.0 / 2 ** 20 / (1.0 * t / reps)
diff --git a/benchmarks/util/result_uploader.py b/benchmarks/util/result_uploader.py
index cff2d9d..021cc54 100755
--- a/benchmarks/util/result_uploader.py
+++ b/benchmarks/util/result_uploader.py
@@ -61,12 +61,12 @@
new_result["timestamp"] = _INITIAL_TIME
print(labels_string)
- bq = big_query_utils.create_big_query()
- row = big_query_utils.make_row(str(uuid.uuid4()), new_result)
- if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET,
+ bq = big_query_utils.create_big_query()
+ row = big_query_utils.make_row(str(uuid.uuid4()), new_result)
+ if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET,
_TABLE + "$" + _NOW,
[row]):
- print('Error when uploading result', new_result)
+ print('Error when uploading result', new_result)
if __name__ == "__main__":
diff --git a/kokoro/linux/benchmark/build.sh b/kokoro/linux/benchmark/build.sh
old mode 100755
new mode 100644
index d832753..bec52b8
--- a/kokoro/linux/benchmark/build.sh
+++ b/kokoro/linux/benchmark/build.sh
@@ -1,108 +1,9 @@
#!/bin/bash
-#
-# Change to repo root
+
cd $(dirname $0)/../../..
+export DOCKERFILE_DIR=kokoro/linux/64-bit
+export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
export OUTPUT_DIR=testoutput
-oldpwd=`pwd`
-
-# tcmalloc
-if [ ! -f gperftools/.libs/libtcmalloc.so ]; then
- git clone https://github.com/gperftools/gperftools.git
- cd gperftools
- ./autogen.sh
- ./configure
- make -j8
- cd ..
-fi
-
-# download datasets for benchmark
-cd benchmarks
-./download_data.sh
-datasets=$(for file in $(find . -type f -name "dataset.*.pb" -not -path "./tmp/*"); do echo "$(pwd)/$file"; done | xargs)
-echo $datasets
-cd $oldpwd
-
-# build Python protobuf
-./autogen.sh
-./configure CXXFLAGS="-fPIC -O2"
-make -j8
-cd python
-python setup.py build --cpp_implementation
-pip install . --user
-
-
-# build and run Python benchmark
-cd ../benchmarks
-make python-pure-python-benchmark
-make python-cpp-reflection-benchmark
-make -j8 python-cpp-generated-code-benchmark
-echo "[" > tmp/python_result.json
-echo "benchmarking pure python..."
-./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets >> tmp/python_result.json
-echo "," >> "tmp/python_result.json"
-echo "benchmarking python cpp reflection..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json
-echo "," >> "tmp/python_result.json"
-echo "benchmarking python cpp generated code..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
-echo "]" >> "tmp/python_result.json"
-cd $oldpwd
-
-# build CPP protobuf
-./configure
-make clean && make -j8
-
-# build Java protobuf
-cd java
-mvn package
-cd ..
-
-# build CPP benchmark
-cd benchmarks
-mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp
-echo "benchmarking cpp..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./cpp-benchmark --benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="tmp/cpp_result.json" $datasets
-cd $oldpwd
-
-# build go protobuf
-export PATH="`pwd`/src:$PATH"
-export GOPATH="$HOME/gocode"
-mkdir -p "$GOPATH/src/github.com/google"
-rm -f "$GOPATH/src/github.com/protocolbuffers/protobuf"
-ln -s "`pwd`" "$GOPATH/src/github.com/protocolbuffers/protobuf"
-export PATH="$GOPATH/bin:$PATH"
-go get github.com/golang/protobuf/protoc-gen-go
-
-# build go benchmark
-cd benchmarks
-make go-benchmark
-echo "benchmarking go..."
-./go-benchmark $datasets > tmp/go_result.txt
-
-# build java benchmark
-make java-benchmark
-echo "benchmarking java..."
-./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets
-
-make js-benchmark
-echo "benchmarking js..."
-./js-benchmark $datasets --json_output=$(pwd)/tmp/node_result.json
-
-make -j8 generate_proto3_data
-proto3_datasets=$(for file in $datasets; do echo $(pwd)/tmp/proto3_data/${file#$(pwd)}; done | xargs)
-echo $proto3_datasets
-
-# build php benchmark
-make -j8 php-benchmark
-echo "benchmarking php..."
-./php-benchmark $proto3_datasets --json --behavior_prefix="php" > tmp/php_result.json
-make -j8 php-c-benchmark
-echo "benchmarking php_c..."
-./php-c-benchmark $proto3_datasets --json --behavior_prefix="php_c" > tmp/php_c_result.json
-
-# upload result to bq
-make python_add_init
-env LD_LIBRARY_PATH="$oldpwd/src/.libs" python -m util.result_uploader -php="../tmp/php_result.json" -php_c="../tmp/php_c_result.json" \
- -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" -go="../tmp/go_result.txt" -python="../tmp/python_result.json" -node="../tmp/node_result.json"
-cd $oldpwd
+export TEST_SET="benchmark"
+./kokoro/linux/build_and_run_docker.sh
diff --git a/kokoro/linux/benchmark/run.sh b/kokoro/linux/benchmark/run.sh
new file mode 100755
index 0000000..264bdaa
--- /dev/null
+++ b/kokoro/linux/benchmark/run.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+#
+# Change to repo root
+cd $(dirname $0)/../../..
+
+export OUTPUT_DIR=testoutput
+oldpwd=`pwd`
+
+# tcmalloc
+if [ ! -f gperftools/.libs/libtcmalloc.so ]; then
+ git clone https://github.com/gperftools/gperftools.git
+ cd gperftools
+ ./autogen.sh
+ ./configure
+ make -j8
+ cd ..
+fi
+
+# download datasets for benchmark
+cd benchmarks
+./download_data.sh
+datasets=$(for file in $(find . -type f -name "dataset.*.pb" -not -path "./tmp/*"); do echo "$(pwd)/$file"; done | xargs)
+echo $datasets
+cd $oldpwd
+
+# build Python protobuf
+./autogen.sh
+./configure CXXFLAGS="-fPIC -O2"
+make -j8
+cd python
+python setup.py build --cpp_implementation
+pip install . --user
+
+
+# build and run Python benchmark
+cd ../benchmarks
+make python-pure-python-benchmark
+make python-cpp-reflection-benchmark
+make -j8 python-cpp-generated-code-benchmark
+echo "[" > tmp/python_result.json
+echo "benchmarking pure python..."
+./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets >> tmp/python_result.json
+echo "," >> "tmp/python_result.json"
+echo "benchmarking python cpp reflection..."
+env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json
+echo "," >> "tmp/python_result.json"
+echo "benchmarking python cpp generated code..."
+env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
+echo "]" >> "tmp/python_result.json"
+cd $oldpwd
+
+# build CPP protobuf
+./configure
+make clean && make -j8
+
+# build Java protobuf
+cd java
+mvn package
+cd ..
+
+# build CPP benchmark
+cd benchmarks
+mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp
+echo "benchmarking cpp..."
+env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./cpp-benchmark --benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="tmp/cpp_result.json" $datasets
+cd $oldpwd
+
+# build go protobuf
+export PATH="`pwd`/src:$PATH"
+export GOPATH="$HOME/gocode"
+mkdir -p "$GOPATH/src/github.com/google"
+rm -f "$GOPATH/src/github.com/protocolbuffers/protobuf"
+ln -s "`pwd`" "$GOPATH/src/github.com/protocolbuffers/protobuf"
+export PATH="$GOPATH/bin:$PATH"
+go get github.com/golang/protobuf/protoc-gen-go
+
+# build go benchmark
+cd benchmarks
+make go-benchmark
+echo "benchmarking go..."
+./go-benchmark $datasets > tmp/go_result.txt
+
+# build java benchmark
+make java-benchmark
+echo "benchmarking java..."
+./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets
+
+make js-benchmark
+echo "benchmarking js..."
+./js-benchmark $datasets --json_output=$(pwd)/tmp/node_result.json
+
+make -j8 generate_proto3_data
+proto3_datasets=$(for file in $datasets; do echo $(pwd)/tmp/proto3_data/${file#$(pwd)}; done | xargs)
+echo $proto3_datasets
+
+# build php benchmark
+make -j8 php-c-benchmark
+echo "benchmarking php_c..."
+./php-c-benchmark $proto3_datasets --json --behavior_prefix="php_c" > tmp/php_c_result.json
+
+# upload result to bq
+make python_add_init
+env LD_LIBRARY_PATH="$oldpwd/src/.libs" python -m util.result_uploader -php_c="../tmp/php_c_result.json" \
+ -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" -go="../tmp/go_result.txt" -python="../tmp/python_result.json" -node="../tmp/node_result.json"
+cd $oldpwd
diff --git a/tests.sh b/tests.sh
index 80269d2..331e950 100755
--- a/tests.sh
+++ b/tests.sh
@@ -524,6 +524,11 @@
build_php_compatibility
}
+build_benchmark() {
+ use_php 7.1
+ cd kokoro/linux/benchmark && ./run.sh
+}
+
# -------- main --------
if [ "$#" -ne 1 ]; then
@@ -556,7 +561,8 @@
php_compatibility |
php7.1 |
php7.1_c |
- php_all)
+ php_all |
+ benchmark)
"
exit 1
fi
