diff options
| -rw-r--r-- | benchmarks/php/PhpBenchmark.php | 2 | ||||
| -rwxr-xr-x | benchmarks/python/py_benchmark.py | 7 | ||||
| -rwxr-xr-x | benchmarks/util/result_uploader.py | 8 | ||||
| -rw-r--r--[-rwxr-xr-x] | kokoro/linux/benchmark/build.sh | 109 | ||||
| -rwxr-xr-x | kokoro/linux/benchmark/run.sh | 105 | ||||
| -rwxr-xr-x | tests.sh | 8 |
6 files changed, 126 insertions, 113 deletions
diff --git a/benchmarks/php/PhpBenchmark.php b/benchmarks/php/PhpBenchmark.php index 9c1132d07..722063876 100644 --- a/benchmarks/php/PhpBenchmark.php +++ b/benchmarks/php/PhpBenchmark.php @@ -62,7 +62,7 @@ class Benchmark $t = $this->runBenchmarkWithTimes(1); $times = ceil($this->benchmark_time / $t); return $this->total_bytes * $times / - $this->runBenchmarkWithTimes($times) * + ($times == 1 ? $t : $this->runBenchmarkWithTimes($times)) * $this->coefficient; } diff --git a/benchmarks/python/py_benchmark.py b/benchmarks/python/py_benchmark.py index ebb197493..40c7af58a 100755 --- a/benchmarks/python/py_benchmark.py +++ b/benchmarks/python/py_benchmark.py @@ -134,9 +134,10 @@ class Benchmark: t = self.dry_run(test_method_args, setup_method_args); if t < 3 : reps = int(math.ceil(3 / t)) * self.full_iteration - t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args), - setup=self.full_setup_code(setup_method_args), - number=reps); + if reps != self.full_iteration: + t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args), + setup=self.full_setup_code(setup_method_args), + number=reps); return self.total_bytes * 1.0 / 2 ** 20 / (1.0 * t / reps) diff --git a/benchmarks/util/result_uploader.py b/benchmarks/util/result_uploader.py index cff2d9d2a..021cc5462 100755 --- a/benchmarks/util/result_uploader.py +++ b/benchmarks/util/result_uploader.py @@ -61,12 +61,12 @@ def upload_result(result_list, metadata): new_result["timestamp"] = _INITIAL_TIME print(labels_string) - bq = big_query_utils.create_big_query() - row = big_query_utils.make_row(str(uuid.uuid4()), new_result) - if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET, + bq = big_query_utils.create_big_query() + row = big_query_utils.make_row(str(uuid.uuid4()), new_result) + if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET, _TABLE + "$" + _NOW, [row]): - print('Error when uploading result', new_result) + print('Error when uploading result', new_result) if __name__ == "__main__": diff --git a/kokoro/linux/benchmark/build.sh b/kokoro/linux/benchmark/build.sh index d83275397..bec52b8ef 100755..100644 --- a/kokoro/linux/benchmark/build.sh +++ b/kokoro/linux/benchmark/build.sh @@ -1,108 +1,9 @@ #!/bin/bash -# -# Change to repo root + cd $(dirname $0)/../../.. +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh export OUTPUT_DIR=testoutput -oldpwd=`pwd` - -# tcmalloc -if [ ! -f gperftools/.libs/libtcmalloc.so ]; then - git clone https://github.com/gperftools/gperftools.git - cd gperftools - ./autogen.sh - ./configure - make -j8 - cd .. -fi - -# download datasets for benchmark -cd benchmarks -./download_data.sh -datasets=$(for file in $(find . -type f -name "dataset.*.pb" -not -path "./tmp/*"); do echo "$(pwd)/$file"; done | xargs) -echo $datasets -cd $oldpwd - -# build Python protobuf -./autogen.sh -./configure CXXFLAGS="-fPIC -O2" -make -j8 -cd python -python setup.py build --cpp_implementation -pip install . --user - - -# build and run Python benchmark -cd ../benchmarks -make python-pure-python-benchmark -make python-cpp-reflection-benchmark -make -j8 python-cpp-generated-code-benchmark -echo "[" > tmp/python_result.json -echo "benchmarking pure python..." -./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets >> tmp/python_result.json -echo "," >> "tmp/python_result.json" -echo "benchmarking python cpp reflection..." -env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json -echo "," >> "tmp/python_result.json" -echo "benchmarking python cpp generated code..." -env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json -echo "]" >> "tmp/python_result.json" -cd $oldpwd - -# build CPP protobuf -./configure -make clean && make -j8 - -# build Java protobuf -cd java -mvn package -cd .. - -# build CPP benchmark -cd benchmarks -mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp -echo "benchmarking cpp..." -env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./cpp-benchmark --benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="tmp/cpp_result.json" $datasets -cd $oldpwd - -# build go protobuf -export PATH="`pwd`/src:$PATH" -export GOPATH="$HOME/gocode" -mkdir -p "$GOPATH/src/github.com/google" -rm -f "$GOPATH/src/github.com/protocolbuffers/protobuf" -ln -s "`pwd`" "$GOPATH/src/github.com/protocolbuffers/protobuf" -export PATH="$GOPATH/bin:$PATH" -go get github.com/golang/protobuf/protoc-gen-go - -# build go benchmark -cd benchmarks -make go-benchmark -echo "benchmarking go..." -./go-benchmark $datasets > tmp/go_result.txt - -# build java benchmark -make java-benchmark -echo "benchmarking java..." -./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets - -make js-benchmark -echo "benchmarking js..." -./js-benchmark $datasets --json_output=$(pwd)/tmp/node_result.json - -make -j8 generate_proto3_data -proto3_datasets=$(for file in $datasets; do echo $(pwd)/tmp/proto3_data/${file#$(pwd)}; done | xargs) -echo $proto3_datasets - -# build php benchmark -make -j8 php-benchmark -echo "benchmarking php..." -./php-benchmark $proto3_datasets --json --behavior_prefix="php" > tmp/php_result.json -make -j8 php-c-benchmark -echo "benchmarking php_c..." -./php-c-benchmark $proto3_datasets --json --behavior_prefix="php_c" > tmp/php_c_result.json - -# upload result to bq -make python_add_init -env LD_LIBRARY_PATH="$oldpwd/src/.libs" python -m util.result_uploader -php="../tmp/php_result.json" -php_c="../tmp/php_c_result.json" \ - -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" -go="../tmp/go_result.txt" -python="../tmp/python_result.json" -node="../tmp/node_result.json" -cd $oldpwd +export TEST_SET="benchmark" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/benchmark/run.sh b/kokoro/linux/benchmark/run.sh new file mode 100755 index 000000000..264bdaa14 --- /dev/null +++ b/kokoro/linux/benchmark/run.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# +# Change to repo root +cd $(dirname $0)/../../.. + +export OUTPUT_DIR=testoutput +oldpwd=`pwd` + +# tcmalloc +if [ ! -f gperftools/.libs/libtcmalloc.so ]; then + git clone https://github.com/gperftools/gperftools.git + cd gperftools + ./autogen.sh + ./configure + make -j8 + cd .. +fi + +# download datasets for benchmark +cd benchmarks +./download_data.sh +datasets=$(for file in $(find . -type f -name "dataset.*.pb" -not -path "./tmp/*"); do echo "$(pwd)/$file"; done | xargs) +echo $datasets +cd $oldpwd + +# build Python protobuf +./autogen.sh +./configure CXXFLAGS="-fPIC -O2" +make -j8 +cd python +python setup.py build --cpp_implementation +pip install . --user + + +# build and run Python benchmark +cd ../benchmarks +make python-pure-python-benchmark +make python-cpp-reflection-benchmark +make -j8 python-cpp-generated-code-benchmark +echo "[" > tmp/python_result.json +echo "benchmarking pure python..." +./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets >> tmp/python_result.json +echo "," >> "tmp/python_result.json" +echo "benchmarking python cpp reflection..." +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json +echo "," >> "tmp/python_result.json" +echo "benchmarking python cpp generated code..." +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json +echo "]" >> "tmp/python_result.json" +cd $oldpwd + +# build CPP protobuf +./configure +make clean && make -j8 + +# build Java protobuf +cd java +mvn package +cd .. + +# build CPP benchmark +cd benchmarks +mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp +echo "benchmarking cpp..." +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./cpp-benchmark --benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="tmp/cpp_result.json" $datasets +cd $oldpwd + +# build go protobuf +export PATH="`pwd`/src:$PATH" +export GOPATH="$HOME/gocode" +mkdir -p "$GOPATH/src/github.com/google" +rm -f "$GOPATH/src/github.com/protocolbuffers/protobuf" +ln -s "`pwd`" "$GOPATH/src/github.com/protocolbuffers/protobuf" +export PATH="$GOPATH/bin:$PATH" +go get github.com/golang/protobuf/protoc-gen-go + +# build go benchmark +cd benchmarks +make go-benchmark +echo "benchmarking go..." +./go-benchmark $datasets > tmp/go_result.txt + +# build java benchmark +make java-benchmark +echo "benchmarking java..." +./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets + +make js-benchmark +echo "benchmarking js..." +./js-benchmark $datasets --json_output=$(pwd)/tmp/node_result.json + +make -j8 generate_proto3_data +proto3_datasets=$(for file in $datasets; do echo $(pwd)/tmp/proto3_data/${file#$(pwd)}; done | xargs) +echo $proto3_datasets + +# build php benchmark +make -j8 php-c-benchmark +echo "benchmarking php_c..." +./php-c-benchmark $proto3_datasets --json --behavior_prefix="php_c" > tmp/php_c_result.json + +# upload result to bq +make python_add_init +env LD_LIBRARY_PATH="$oldpwd/src/.libs" python -m util.result_uploader -php_c="../tmp/php_c_result.json" \ + -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" -go="../tmp/go_result.txt" -python="../tmp/python_result.json" -node="../tmp/node_result.json" +cd $oldpwd @@ -524,6 +524,11 @@ build_php_all() { build_php_compatibility } +build_benchmark() { + use_php 7.1 + cd kokoro/linux/benchmark && ./run.sh +} + # -------- main -------- if [ "$#" -ne 1 ]; then @@ -556,7 +561,8 @@ Usage: $0 { cpp | php_compatibility | php7.1 | php7.1_c | - php_all) + php_all | + benchmark) " exit 1 fi |