diff options
Diffstat (limited to 'grpc/tools/distrib')
43 files changed, 2525 insertions, 0 deletions
diff --git a/grpc/tools/distrib/bazel_style.cfg b/grpc/tools/distrib/bazel_style.cfg new file mode 100644 index 00000000..a5a1fea4 --- /dev/null +++ b/grpc/tools/distrib/bazel_style.cfg @@ -0,0 +1,4 @@ +[style] +based_on_style = google +allow_split_before_dict_value = False +spaces_around_default_or_named_assign = True diff --git a/grpc/tools/distrib/build_ruby_environment_macos.sh b/grpc/tools/distrib/build_ruby_environment_macos.sh new file mode 100644 index 00000000..fbbc1a7b --- /dev/null +++ b/grpc/tools/distrib/build_ruby_environment_macos.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +rm -rf ~/.rake-compiler + +CROSS_RUBY=$(mktemp tmpfile.XXXXXXXX) + +curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.1.0/tasks/bin/cross-ruby.rake > "$CROSS_RUBY" + +# See https://github.com/grpc/grpc/issues/12161 for verconf.h patch details +patch "$CROSS_RUBY" << EOF +--- cross-ruby.rake 2018-04-10 11:32:16.000000000 -0700 ++++ patched 2018-04-10 11:40:25.000000000 -0700 +@@ -133,8 +133,10 @@ + "--host=#{MINGW_HOST}", + "--target=#{MINGW_TARGET}", + "--build=#{RUBY_BUILD}", +- '--enable-shared', ++ '--enable-static', ++ '--disable-shared', + '--disable-install-doc', ++ '--without-gmp', + '--with-ext=' + ] + +@@ -151,6 +153,7 @@ + # make + file "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/ruby.exe" => ["#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/Makefile"] do |t| + chdir File.dirname(t.prerequisites.first) do ++ sh "test -s verconf.h || rm -f verconf.h" # if verconf.h has size 0, make sure it gets re-built by make + sh MAKE + end + end +EOF + +MAKE="make -j8" + +set +x # rvm commands are very verbose +source ~/.rvm/scripts/rvm +rvm use 2.7.0 +set -x +ruby --version | grep 'ruby 2.7.0' +for v in 2.7.0 ; do + ccache -c + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE" +done +set +x +rvm use 2.5.0 +set -x +ruby --version | grep 'ruby 2.5.0' +for v in 2.6.0 2.5.0 2.4.0 2.3.0 2.2.2 ; do + ccache -c + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE" +done + +sed 's/x86_64-darwin-11/universal-darwin/' ~/.rake-compiler/config.yml > "$CROSS_RUBY" +mv "$CROSS_RUBY" ~/.rake-compiler/config.yml diff --git a/grpc/tools/distrib/buildifier_format_code.sh b/grpc/tools/distrib/buildifier_format_code.sh new file mode 100755 index 00000000..8245cd4a --- /dev/null +++ b/grpc/tools/distrib/buildifier_format_code.sh @@ -0,0 +1,70 @@ +#! /bin/bash +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +BUILDIFIER_VERSION="0.29.0" +TEMP_BUILDIFIER_PATH="/tmp/buildifier" +EXTRA_BUILDIFIER_FLAGS=$* + +function error_handling() { + error=$1 + if [[ -x "$error" ]]; then + echo "${error}" + exit 1 + fi +} + +function download_buildifier() { + platform="$(uname -s)" + case "${platform}" in + Linux*) download_link="https://github.com/bazelbuild/buildtools/releases/download/${BUILDIFIER_VERSION}/buildifier";; + Darwin*) download_link="https://github.com/bazelbuild/buildtools/releases/download/${BUILDIFIER_VERSION}/buildifier.mac";; + *) error_handling "Unsupported platform: ${platform}";; + esac + + if [ -x "$(command -v curl)" ]; then + curl -L -o ${TEMP_BUILDIFIER_PATH} ${download_link} + elif [ -x "$(command -v wget)" ]; then + wget -O ${TEMP_BUILDIFIER_PATH} ${download_link} + else + error_handling "Download failed: curl and wget not available" + fi + + chmod +x ${TEMP_BUILDIFIER_PATH} +} + + +# Get the correct version of buildifier +if [ -x "$(command -v buildifier)" ]; then + existing_buildifier_version="$(buildifier -version 2>&1 | head -n1 | cut -d" " -f3)" + if [[ "${existing_buildifier_version}" != "${BUILDIFIER_VERSION}" ]]; then + download_buildifier + buildifier_bin="${TEMP_BUILDIFIER_PATH}" + else + buildifier_bin="buildifier" + fi +else + download_buildifier + buildifier_bin="${TEMP_BUILDIFIER_PATH}" +fi + +# cd to repo root +dir=$(dirname "${0}") +cd "${dir}/../.." + +bazel_files=$(find . \( -iname 'BUILD' -o -iname '*.bzl' -o -iname '*.bazel' -o -iname 'WORKSPACE' \) -type f -not -path "./third_party/*") +# shellcheck disable=SC2086,SC2068 +${buildifier_bin} ${EXTRA_BUILDIFIER_FLAGS[@]} -v ${bazel_files} diff --git a/grpc/tools/distrib/buildifier_format_code_strict.sh b/grpc/tools/distrib/buildifier_format_code_strict.sh new file mode 100755 index 00000000..e8a98418 --- /dev/null +++ b/grpc/tools/distrib/buildifier_format_code_strict.sh @@ -0,0 +1,20 @@ +#! /bin/bash -ex +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +dir=$(dirname "${0}") +buildifier_format_script="${dir}/buildifier_format_code.sh" + +${buildifier_format_script} -lint warn diff --git a/grpc/tools/distrib/c-ish/check_documentation.py b/grpc/tools/distrib/c-ish/check_documentation.py new file mode 100755 index 00000000..fef8f4ee --- /dev/null +++ b/grpc/tools/distrib/c-ish/check_documentation.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python2.7 + +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# check for directory level 'README.md' files +# check that all implementation and interface files have a \file doxygen comment + +import os +import sys + +# where do we run +_TARGET_DIRS = [ + 'include/grpc', 'include/grpc++', 'src/core', 'src/cpp', 'test/core', + 'test/cpp' +] + +# which file extensions do we care about +_INTERESTING_EXTENSIONS = ['.c', '.h', '.cc'] + +# find our home +_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) +os.chdir(_ROOT) + +errors = 0 + +# walk directories, find things +printed_banner = False +for target_dir in _TARGET_DIRS: + for root, dirs, filenames in os.walk(target_dir): + if 'README.md' not in filenames: + if not printed_banner: + print 'Missing README.md' + print '=================' + printed_banner = True + print root + errors += 1 +if printed_banner: print +printed_banner = False +for target_dir in _TARGET_DIRS: + for root, dirs, filenames in os.walk(target_dir): + for filename in filenames: + if os.path.splitext(filename)[1] not in _INTERESTING_EXTENSIONS: + continue + path = os.path.join(root, filename) + with open(path) as f: + contents = f.read() + if '\\file' not in contents: + if not printed_banner: + print 'Missing \\file comment' + print '======================' + printed_banner = True + print path + errors += 1 + +assert errors == 0, 'error count = %d' % errors diff --git a/grpc/tools/distrib/check_boringssl_prefix_symbol.sh b/grpc/tools/distrib/check_boringssl_prefix_symbol.sh new file mode 100755 index 00000000..52a0460c --- /dev/null +++ b/grpc/tools/distrib/check_boringssl_prefix_symbol.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Check if the current BoringSSL prefix symbols is up to date +set -e + +cd "$(dirname $0)" +cd ../../third_party/boringssl-with-bazel + +BORINGSSL_COMMIT=$(git rev-parse HEAD) +PREFIX_SYMBOLS_COMMIT=$(cat ../../src/boringssl/boringssl_prefix_symbols.h | head -n1 | awk '{print $NF}') + +[ $BORINGSSL_COMMIT == $PREFIX_SYMBOLS_COMMIT ] || { echo "The BoringSSL commit does not match the commit of the prefix symbols (src/boringssl/boringssl_prefix_symbols.h). Run tools/distrib/regenerate_boringssl_prefix_symbols.sh to update the prefix symbols." ; exit 1 ; } + +exit 0 diff --git a/grpc/tools/distrib/check_copyright.py b/grpc/tools/distrib/check_copyright.py new file mode 100755 index 00000000..831d62ef --- /dev/null +++ b/grpc/tools/distrib/check_copyright.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python2.7 + +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import datetime +import os +import re +import sys +import subprocess + +# find our home +ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +os.chdir(ROOT) + +# parse command line +argp = argparse.ArgumentParser(description='copyright checker') +argp.add_argument('-o', + '--output', + default='details', + choices=['list', 'details']) +argp.add_argument('-s', '--skips', default=0, action='store_const', const=1) +argp.add_argument('-a', '--ancient', default=0, action='store_const', const=1) +argp.add_argument('--precommit', default=False, action='store_true') +args = argp.parse_args() + +# open the license text +with open('NOTICE.txt') as f: + LICENSE_NOTICE = f.read().splitlines() + +# license format by file extension +# key is the file extension, value is a format string +# that given a line of license text, returns what should +# be in the file +LICENSE_PREFIX = { + '.bat': r'@rem\s*', + '.c': r'\s*(?://|\*)\s*', + '.cc': r'\s*(?://|\*)\s*', + '.h': r'\s*(?://|\*)\s*', + '.m': r'\s*\*\s*', + '.mm': r'\s*\*\s*', + '.php': r'\s*\*\s*', + '.js': r'\s*\*\s*', + '.py': r'#\s*', + '.pyx': r'#\s*', + '.pxd': r'#\s*', + '.pxi': r'#\s*', + '.rb': r'#\s*', + '.sh': r'#\s*', + '.proto': r'//\s*', + '.cs': r'//\s*', + '.mak': r'#\s*', + 'Makefile': r'#\s*', + 'Dockerfile': r'#\s*', + 'BUILD': r'#\s*', +} + +_EXEMPT = frozenset(( + # Generated protocol compiler output. + 'examples/python/helloworld/helloworld_pb2.py', + 'examples/python/helloworld/helloworld_pb2_grpc.py', + 'examples/python/multiplex/helloworld_pb2.py', + 'examples/python/multiplex/helloworld_pb2_grpc.py', + 'examples/python/multiplex/route_guide_pb2.py', + 'examples/python/multiplex/route_guide_pb2_grpc.py', + 'examples/python/route_guide/route_guide_pb2.py', + 'examples/python/route_guide/route_guide_pb2_grpc.py', + + # An older file originally from outside gRPC. + 'src/php/tests/bootstrap.php', + # census.proto copied from github + 'tools/grpcz/census.proto', + # status.proto copied from googleapis + 'src/proto/grpc/status/status.proto', + + # Gradle wrappers used to build for Android + 'examples/android/helloworld/gradlew.bat', + 'src/android/test/interop/gradlew.bat', + + # Designer-generated source + 'examples/csharp/HelloworldXamarin/Droid/Resources/Resource.designer.cs', + 'examples/csharp/HelloworldXamarin/iOS/ViewController.designer.cs', + + # BoringSSL generated header. It has commit version information at the head + # of the file so we cannot check the license info. + 'src/boringssl/boringssl_prefix_symbols.h', +)) + +RE_YEAR = r'Copyright (?P<first_year>[0-9]+\-)?(?P<last_year>[0-9]+) ([Tt]he )?gRPC [Aa]uthors(\.|)' +RE_LICENSE = dict( + (k, r'\n'.join(LICENSE_PREFIX[k] + + (RE_YEAR if re.search(RE_YEAR, line) else re.escape(line)) + for line in LICENSE_NOTICE)) + for k, v in LICENSE_PREFIX.iteritems()) + +if args.precommit: + FILE_LIST_COMMAND = 'git status -z | grep -Poz \'(?<=^[MARC][MARCD ] )[^\s]+\'' +else: + FILE_LIST_COMMAND = 'git ls-tree -r --name-only -r HEAD | ' \ + 'grep -v ^third_party/ |' \ + 'grep -v "\(ares_config.h\|ares_build.h\)"' + + +def load(name): + with open(name) as f: + return f.read() + + +def save(name, text): + with open(name, 'w') as f: + f.write(text) + + +assert (re.search(RE_LICENSE['Makefile'], load('Makefile'))) + + +def log(cond, why, filename): + if not cond: return + if args.output == 'details': + print '%s: %s' % (why, filename) + else: + print filename + + +# scan files, validate the text +ok = True +filename_list = [] +try: + filename_list = subprocess.check_output(FILE_LIST_COMMAND, + shell=True).splitlines() +except subprocess.CalledProcessError: + sys.exit(0) + +for filename in filename_list: + if filename in _EXEMPT: + continue + # Skip check for upb generated code. + if filename.endswith('.upb.h') or filename.endswith('.upb.c'): + continue + ext = os.path.splitext(filename)[1] + base = os.path.basename(filename) + if ext in RE_LICENSE: + re_license = RE_LICENSE[ext] + elif base in RE_LICENSE: + re_license = RE_LICENSE[base] + else: + log(args.skips, 'skip', filename) + continue + try: + text = load(filename) + except: + continue + m = re.search(re_license, text) + if m: + pass + elif 'DO NOT EDIT' not in text: + log(1, 'copyright missing', filename) + ok = False + +sys.exit(0 if ok else 1) diff --git a/grpc/tools/distrib/check_include_guards.py b/grpc/tools/distrib/check_include_guards.py new file mode 100755 index 00000000..29834411 --- /dev/null +++ b/grpc/tools/distrib/check_include_guards.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python2.7 + +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import os.path +import re +import sys +import subprocess + + +def build_valid_guard(fpath): + prefix = 'GRPC_' if not fpath.startswith('include/') else '' + return prefix + '_'.join( + fpath.replace('++', 'XX').replace('.', '_').upper().split('/')[1:]) + + +def load(fpath): + with open(fpath, 'r') as f: + return f.read() + + +def save(fpath, contents): + with open(fpath, 'w') as f: + f.write(contents) + + +class GuardValidator(object): + + def __init__(self): + self.ifndef_re = re.compile(r'#ifndef ([A-Z][A-Z_1-9]*)') + self.define_re = re.compile(r'#define ([A-Z][A-Z_1-9]*)') + self.endif_c_re = re.compile( + r'#endif /\* (?: *\\\n *)?([A-Z][A-Z_1-9]*) (?:\\\n *)?\*/$') + self.endif_cpp_re = re.compile(r'#endif // ([A-Z][A-Z_1-9]*)') + self.failed = False + + def fail(self, fpath, regexp, fcontents, match_txt, correct, fix): + cpp_header = 'grpc++' in fpath or 'grpcpp' in fpath + self.failed = True + invalid_guards_msg_template = ( + '{0}: Missing preprocessor guards (RE {1}). ' + 'Please wrap your code around the following guards:\n' + '#ifndef {2}\n' + '#define {2}\n' + '...\n' + '... epic code ...\n' + '...\n') + ('#endif // {2}' if cpp_header else '#endif /* {2} */') + if not match_txt: + print invalid_guards_msg_template.format(fpath, regexp.pattern, + build_valid_guard(fpath)) + return fcontents + + print( + '{}: Wrong preprocessor guards (RE {}):' + '\n\tFound {}, expected {}').format(fpath, regexp.pattern, + match_txt, correct) + if fix: + print 'Fixing {}...\n'.format(fpath) + fixed_fcontents = re.sub(match_txt, correct, fcontents) + if fixed_fcontents: + self.failed = False + return fixed_fcontents + else: + print + return fcontents + + def check(self, fpath, fix): + cpp_header = 'grpc++' in fpath or 'grpcpp' in fpath + valid_guard = build_valid_guard(fpath) + + fcontents = load(fpath) + + match = self.ifndef_re.search(fcontents) + if not match: + print 'something drastically wrong with: %s' % fpath + return False # failed + if match.lastindex is None: + # No ifndef. Request manual addition with hints + self.fail(fpath, match.re, match.string, '', '', False) + return False # failed + + # Does the guard end with a '_H'? + running_guard = match.group(1) + if not running_guard.endswith('_H'): + fcontents = self.fail(fpath, match.re, match.string, match.group(1), + valid_guard, fix) + if fix: save(fpath, fcontents) + + # Is it the expected one based on the file path? + if running_guard != valid_guard: + fcontents = self.fail(fpath, match.re, match.string, match.group(1), + valid_guard, fix) + if fix: save(fpath, fcontents) + + # Is there a #define? Is it the same as the #ifndef one? + match = self.define_re.search(fcontents) + if match.lastindex is None: + # No define. Request manual addition with hints + self.fail(fpath, match.re, match.string, '', '', False) + return False # failed + + # Is the #define guard the same as the #ifndef guard? + if match.group(1) != running_guard: + fcontents = self.fail(fpath, match.re, match.string, match.group(1), + valid_guard, fix) + if fix: save(fpath, fcontents) + + # Is there a properly commented #endif? + endif_re = self.endif_cpp_re if cpp_header else self.endif_c_re + flines = fcontents.rstrip().splitlines() + match = endif_re.search('\n'.join(flines[-3:])) + if not match: + # No endif. Check if we have the last line as just '#endif' and if so + # replace it with a properly commented one. + if flines[-1] == '#endif': + flines[-1] = ('#endif' + + (' // {}\n'.format(valid_guard) if cpp_header + else ' /* {} */\n'.format(valid_guard))) + if fix: + fcontents = '\n'.join(flines) + save(fpath, fcontents) + else: + # something else is wrong, bail out + self.fail(fpath, endif_re, flines[-1], '', '', False) + elif match.group(1) != running_guard: + # Is the #endif guard the same as the #ifndef and #define guards? + fcontents = self.fail(fpath, endif_re, fcontents, match.group(1), + valid_guard, fix) + if fix: save(fpath, fcontents) + + return not self.failed # Did the check succeed? (ie, not failed) + + +# find our home +ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +os.chdir(ROOT) + +# parse command line +argp = argparse.ArgumentParser(description='include guard checker') +argp.add_argument('-f', '--fix', default=False, action='store_true') +argp.add_argument('--precommit', default=False, action='store_true') +args = argp.parse_args() + +KNOWN_BAD = set([ + 'src/core/ext/filters/client_channel/health/health.pb.h', + 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', + 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h', + 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h', + 'src/core/tsi/alts/handshaker/altscontext.pb.h', + 'src/core/tsi/alts/handshaker/handshaker.pb.h', + 'src/core/tsi/alts/handshaker/transport_security_common.pb.h', + 'include/grpc++/ext/reflection.grpc.pb.h', + 'include/grpc++/ext/reflection.pb.h', +]) + +grep_filter = r"grep -E '^(include|src/core)/.*\.h$'" +if args.precommit: + git_command = 'git diff --name-only HEAD' +else: + git_command = 'git ls-tree -r --name-only -r HEAD' + +FILE_LIST_COMMAND = ' | '.join((git_command, grep_filter)) + +# scan files +ok = True +filename_list = [] +try: + filename_list = subprocess.check_output(FILE_LIST_COMMAND, + shell=True).splitlines() + # Filter out non-existent files (ie, file removed or renamed) + filename_list = (f for f in filename_list if os.path.isfile(f)) +except subprocess.CalledProcessError: + sys.exit(0) + +validator = GuardValidator() + +for filename in filename_list: + if filename in KNOWN_BAD: continue + # Skip check for upb generated code. + if filename.endswith('.upb.h') or filename.endswith('.upb.c'): + continue + ok = ok and validator.check(filename, args.fix) + +sys.exit(0 if ok else 1) diff --git a/grpc/tools/distrib/check_protobuf_pod_version.sh b/grpc/tools/distrib/check_protobuf_pod_version.sh new file mode 100755 index 00000000..174931a0 --- /dev/null +++ b/grpc/tools/distrib/check_protobuf_pod_version.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2019 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd `dirname $0`/../.. + +# get the version of protobuf in /third_party/protobuf +pushd third_party/protobuf + +version1=$(git describe --tags | cut -f 1 -d'-') +v1=${version1:1} + +popd + +# get the version of protobuf in /src/objective-c/!ProtoCompiler.podspec +v2=$(cat src/objective-c/\!ProtoCompiler.podspec | egrep "v = " | cut -f 2 -d"'") + +# get the version of protobuf in /src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +v3=$(cat src/objective-c/\!ProtoCompiler-gRPCPlugin.podspec | egrep 'dependency.*!ProtoCompiler' | cut -f 4 -d"'") + +# compare and emit error +ret=0 +if [ $v1 != $v2 ]; then + echo 'Protobuf version in src/objective-c/!ProtoCompiler.podspec does not match protobuf version in third_party/protobuf.' + ret=1 +fi + +if [ $v1 != $v3 ]; then + echo 'Protobuf version in src/objective-c/!ProtoCompiler-gRPCPlugin.podspec does not match protobuf version in third_party/protobuf.' + ret=1 +fi + +exit $ret diff --git a/grpc/tools/distrib/check_pytype.sh b/grpc/tools/distrib/check_pytype.sh new file mode 100755 index 00000000..c8a9a9df --- /dev/null +++ b/grpc/tools/distrib/check_pytype.sh @@ -0,0 +1,19 @@ +#! /bin/bash -ex +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +JOBS=$(nproc) || JOBS=4 +python3 -m pip install pytype==2019.11.27 + +python3 -m pytype --keep-going -j "$JOBS" --strict-import --config "setup.cfg" diff --git a/grpc/tools/distrib/check_trailing_newlines.sh b/grpc/tools/distrib/check_trailing_newlines.sh new file mode 100755 index 00000000..83348a5e --- /dev/null +++ b/grpc/tools/distrib/check_trailing_newlines.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# change to root directory +cd $(dirname $0)/../.. + +function find_without_newline() { + find . -type f -not -path './third_party/*' -and \( \ + -name '*.c' \ + -or -name '*.cc' \ + -or -name '*.proto' \ + -or -name '*.rb' \ + -or -name '*.py' \ + -or -name '*.cs' \ + -or -name '*.sh' \) -print0 \ + | while IFS= read -r -d '' f; do + if [[ ! -z $f ]]; then + if [[ $(tail -c 1 "$f") != $NEWLINE ]]; then + echo "Error: file '$f' is missing a trailing newline character." + if $2; then # fix + sed -i -e '$a\' $f + echo 'Fixed!' + fi + fi + fi + done +} + +if [[ $# == 1 && $1 == '--fix' ]]; then + ERRORS=$(find_without_newline true) +else + ERRORS=$(find_without_newline false) +fi + +if [[ "$ERRORS" != '' ]]; then + echo "$ERRORS" + if ! $FIX; then + exit 1 + fi +fi diff --git a/grpc/tools/distrib/check_upb_output.sh b/grpc/tools/distrib/check_upb_output.sh new file mode 100755 index 00000000..6febd0e9 --- /dev/null +++ b/grpc/tools/distrib/check_upb_output.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2019 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +readonly UPB_GENERATED_SRC=src/core/ext/upb-generated +readonly UPB_TMP_OUTPUT="$(mktemp -d)" + +tools/codegen/core/gen_upb_api.sh "$UPB_TMP_OUTPUT" + +diff -rq "$UPB_GENERATED_SRC" "$UPB_TMP_OUTPUT" diff --git a/grpc/tools/distrib/check_windows_dlls.sh b/grpc/tools/distrib/check_windows_dlls.sh new file mode 100755 index 00000000..c25091ed --- /dev/null +++ b/grpc/tools/distrib/check_windows_dlls.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to root directory +cd "$(dirname "$0")/../.." + +bundle +rake dlls diff --git a/grpc/tools/distrib/clang_format_code.sh b/grpc/tools/distrib/clang_format_code.sh new file mode 100755 index 00000000..cd7553e9 --- /dev/null +++ b/grpc/tools/distrib/clang_format_code.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to root directory +cd $(dirname $0)/../.. +REPO_ROOT=$(pwd) + +if [ "$CLANG_FORMAT_SKIP_DOCKER" == "" ] +then + # build clang-format docker image + docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format + + # run clang-format against the checked out codebase + # when modifying the checked-out files, the current user will be impersonated + # so that the updated files don't end up being owned by "root". + docker run -e TEST="$TEST" -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_FORMAT_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code --user "$(id -u):$(id -g)" -t grpc_clang_format /clang_format_all_the_things.sh +else + CLANG_FORMAT_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh +fi diff --git a/grpc/tools/distrib/clang_tidy_code.sh b/grpc/tools/distrib/clang_tidy_code.sh new file mode 100755 index 00000000..9262b6bd --- /dev/null +++ b/grpc/tools/distrib/clang_tidy_code.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "NOTE: to automagically apply fixes, invoke with --fix" + +set -ex + +# change to root directory +cd $(dirname $0)/../.. +REPO_ROOT=$(pwd) + +if [ "$CLANG_TIDY_SKIP_DOCKER" == "" ] +then + # build clang-tidy docker image + docker build -t grpc_clang_tidy tools/dockerfile/grpc_clang_tidy + + # run clang-tidy against the checked out codebase + # when modifying the checked-out files, the current user will be impersonated + # so that the updated files don't end up being owned by "root". + docker run -e TEST="$TEST" -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_TIDY_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code --user "$(id -u):$(id -g)" -t grpc_clang_tidy /clang_tidy_all_the_things.sh "$@" +else + CLANG_TIDY_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_tidy/clang_tidy_all_the_things.sh "$@" +fi diff --git a/grpc/tools/distrib/format_bazel.sh b/grpc/tools/distrib/format_bazel.sh new file mode 100755 index 00000000..9aa98cbf --- /dev/null +++ b/grpc/tools/distrib/format_bazel.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2019 The gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set=-ex + +VIRTUAL_ENV=bazel_format_virtual_environment + +CONFIG_PATH="$(dirname ${0})/bazel_style.cfg" + +python -m virtualenv ${VIRTUAL_ENV} +PYTHON=${VIRTUAL_ENV}/bin/python +"$PYTHON" -m pip install --upgrade pip==19.3.1 +"$PYTHON" -m pip install --upgrade futures +"$PYTHON" -m pip install yapf==0.28.0 + +pushd "$(dirname "${0}")/../.." +FILES=$(find . -path ./third_party -prune -o -name '*.bzl' -print) +echo "${FILES}" | xargs "$PYTHON" -m yapf -i --style="${CONFIG_PATH}" + +if ! which buildifier &>/dev/null; then + echo 'buildifer must be installed.' >/dev/stderr + exit 1 +fi + +echo "${FILES}" | xargs buildifier --type=bzl + +popd diff --git a/grpc/tools/distrib/generate_boringssl_prefix_header.sh b/grpc/tools/distrib/generate_boringssl_prefix_header.sh new file mode 100755 index 00000000..f8d60b87 --- /dev/null +++ b/grpc/tools/distrib/generate_boringssl_prefix_header.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate the list of boringssl symbols that need to be renamed based on the +# current boringssl submodule. The script should be run after a boringssl +# upgrade in third_party/boringssl-with-bazel. Note that after the script is +# run, you will typically need to manually upgrade the BoringSSL-GRPC podspec +# (templates/src/objective-c/BoringSSL-GRPC.podspec.template) version and the +# corresponding version number in gRPC-Core podspec +# (templates/gRPC-Core.podspec.template). + +set -ev + +BORINGSSL_ROOT=third_party/boringssl-with-bazel/src + +cd "$(dirname $0)" +cd ../../$BORINGSSL_ROOT + +BORINGSSL_COMMIT=$(git rev-parse HEAD) +BORINGSSL_PREFIX_HEADERS_DIR=src/boringssl + +rm -rf build +mkdir -p build +cd build +cmake .. +make -j +[ -f ssl/libssl.a ] || { echo "Failed to build libssl.a" ; exit 1 ; } +[ -f crypto/libcrypto.a ] || { echo "Failed to build libcrypto.a" ; exit 1 ; } + +# Generates boringssl_prefix_symbols.h. The prefix header is generated by +# BoringSSL's build system as instructed by BoringSSL build guide (see +# https://github.com/google/boringssl/blob/367d64f84c3c1d01381c18c5a239b85eef47633c/BUILDING.md#building-with-prefixed-symbols). +go run ../util/read_symbols.go ssl/libssl.a > ./symbols.txt +go run ../util/read_symbols.go crypto/libcrypto.a >> ./symbols.txt +cmake .. -DBORINGSSL_PREFIX=GRPC -DBORINGSSL_PREFIX_SYMBOLS=symbols.txt +make boringssl_prefix_symbols +[ -f symbol_prefix_include/boringssl_prefix_symbols.h ] || { echo "Failed to build boringssl_prefix_symbols.sh" ; exit 1 ; } + +cd ../../../.. +mkdir -p $BORINGSSL_PREFIX_HEADERS_DIR +echo "// generated by generate_boringssl_prefix_header.sh on BoringSSL commit: $BORINGSSL_COMMIT" > $BORINGSSL_PREFIX_HEADERS_DIR/boringssl_prefix_symbols.h +echo "" >> $BORINGSSL_PREFIX_HEADERS_DIR/boringssl_prefix_symbols.h +cat "$BORINGSSL_ROOT/build/symbol_prefix_include/boringssl_prefix_symbols.h" >> $BORINGSSL_PREFIX_HEADERS_DIR/boringssl_prefix_symbols.h + +# Regenerated the project +tools/buildgen/generate_projects.sh + +exit 0 diff --git a/grpc/tools/distrib/guard_headers.sh b/grpc/tools/distrib/guard_headers.sh new file mode 100755 index 00000000..fb66e55f --- /dev/null +++ b/grpc/tools/distrib/guard_headers.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +cd `dirname $0`/../.. + +function process_dir { + base_dir=$1 + prefix=$2 + comment_language=$3 + ( + cd $base_dir + find . -name "*.h" | while read f ; do + guard=${prefix}_`echo ${f#*/} | tr '[:lower:]/.-' '[:upper:]___'` + if [ "$comment_language" = "c++" ] ; then + comment="// $guard" + else + comment="/* $guard */" + fi + awk ' + BEGIN { + guard = "'${guard}'"; + comment_language = "'${comment_language}'"; + } + prev ~ /^#ifndef/ && !got_first_ifndef { + got_first_ifndef = 1; + prev = "#ifndef " guard; + } + prev ~ /^#define/ && !got_first_define { + got_first_define = 1; + prev = "#define " guard; + } + NR > 1 { print prev; } + { prev = $0; } + END { + if (prev ~ /^#endif/) { + if (comment_language ~ /^c$/) { + print "#endif /* " guard " */"; + } else if (comment_language ~ /^c\+\+$/) { + print "#endif // " guard; + } else { + print "ERROR: unknown comment language: " comment_language; + } + } else { + print "ERROR: file does not end with #endif"; + } + } + ' "${f}" > "${f}.rewritten" + mv "${f}.rewritten" "${f}" + done + ) +} + +process_dir include/grpc GRPC c +process_dir include/grpc++ GRPCXX c++ +process_dir src/core GRPC_INTERNAL_CORE c +process_dir src/cpp GRPC_INTERNAL_CPP c++ +process_dir src/compiler GRPC_INTERNAL_COMPILER c++ +process_dir test/core GRPC_TEST_CORE c +process_dir test/cpp GRPC_TEST_CPP c++ +process_dir examples GRPC_EXAMPLES c++ diff --git a/grpc/tools/distrib/pull_requests_interval.sh b/grpc/tools/distrib/pull_requests_interval.sh new file mode 100755 index 00000000..7a6c702d --- /dev/null +++ b/grpc/tools/distrib/pull_requests_interval.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$1" = "x" ] ; then + echo "Usage: $0 <first ref> [second ref]" + exit 1 +else + first=$1 +fi + +if [ -n $2 ] ; then + second=HEAD +fi + +if [ -e ~/github-credentials.vars ] ; then + . ~/github-credentials.vars +fi + +if [ "x$github_client_id" = "x" ] || [ "x$github_client_secret" = "x" ] ; then + echo "Warning: you don't have github credentials set." + echo + echo "You may end up exceeding guest quota quickly." + echo "You can create an application for yourself," + echo "and get its credentials. Go to" + echo + echo " https://github.com/settings/developers" + echo + echo "and click 'Register a new application'." + echo + echo "From the application's information, copy/paste" + echo "its Client ID and Client Secret, into the file" + echo + echo " ~/github-credentials.vars" + echo + echo "with the following format:" + echo + echo "github_client_id=0123456789abcdef0123" + echo "github_client_secret=0123456789abcdef0123456789abcdef" + echo + echo + addendum="" +else + addendum="?client_id=$github_client_id&client_secret=$github_client_secret" +fi + +unset notfirst +echo "[" +git log --pretty=oneline $1..$2 | + grep '[^ ]\+ Merge pull request #[0-9]\{4,6\} ' | + cut -f 2 -d# | + cut -f 1 -d\ | + sort -u | + while read id ; do + if [ "x$notfirst" = "x" ] ; then + notfirst=true + else + echo "," + fi + echo -n " {\"url\": \"https://github.com/grpc/grpc/pull/$id\"," + out=`mktemp` + curl -s "https://api.github.com/repos/grpc/grpc/pulls/$id$addendum" > $out + echo -n " "`grep '"title"' $out` + echo -n " "`grep '"login"' $out | head -1` + echo -n " \"pr\": $id }" + rm $out + done +echo +echo "]" diff --git a/grpc/tools/distrib/pylint_code.sh b/grpc/tools/distrib/pylint_code.sh new file mode 100755 index 00000000..5e6b818d --- /dev/null +++ b/grpc/tools/distrib/pylint_code.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2017 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# NOTE(rbellevi): We ignore generated code. +IGNORE_PATTERNS=--ignore-patterns='.*pb2\.py,.*pb2_grpc\.py' + +# change to root directory +cd "$(dirname "$0")/../.." + +DIRS=( + 'src/python/grpcio/grpc' + 'src/python/grpcio_channelz/grpc_channelz' + 'src/python/grpcio_health_checking/grpc_health' + 'src/python/grpcio_reflection/grpc_reflection' + 'src/python/grpcio_testing/grpc_testing' + 'src/python/grpcio_status/grpc_status' +) + +TEST_DIRS=( + 'src/python/grpcio_tests/tests' +) + +VIRTUALENV=python_pylint_venv +python3 -m virtualenv $VIRTUALENV -p $(which python3) + +PYTHON=$VIRTUALENV/bin/python + +$PYTHON -m pip install --upgrade pip==19.3.1 +$PYTHON -m pip install --upgrade astroid==2.3.3 pylint==2.2.2 + +EXIT=0 +for dir in "${DIRS[@]}"; do + $PYTHON -m pylint --rcfile=.pylintrc -rn "$dir" ${IGNORE_PATTERNS} || EXIT=1 +done + +for dir in "${TEST_DIRS[@]}"; do + $PYTHON -m pylint --rcfile=.pylintrc-tests -rn "$dir" ${IGNORE_PATTERNS} || EXIT=1 +done + +find examples/python \ + -iname "*.py" \ + -not -name "*_pb2.py" \ + -not -name "*_pb2_grpc.py" \ + | xargs $PYTHON -m pylint --rcfile=.pylintrc-examples -rn ${IGNORE_PATTERNS} + +exit $EXIT diff --git a/grpc/tools/distrib/python/.gitignore b/grpc/tools/distrib/python/.gitignore new file mode 100644 index 00000000..8ff3b086 --- /dev/null +++ b/grpc/tools/distrib/python/.gitignore @@ -0,0 +1 @@ +distrib_virtualenv/ diff --git a/grpc/tools/distrib/python/bazel_deps.sh b/grpc/tools/distrib/python/bazel_deps.sh new file mode 100755 index 00000000..ce2c1147 --- /dev/null +++ b/grpc/tools/distrib/python/bazel_deps.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd $(dirname $0)/../../../ + +cd third_party/protobuf +# use tools/bazel wrapper to make sure we use the right version of bazel +../../tools/bazel query 'deps('$1')' diff --git a/grpc/tools/distrib/python/check_grpcio_tools.py b/grpc/tools/distrib/python/check_grpcio_tools.py new file mode 100755 index 00000000..25751ca4 --- /dev/null +++ b/grpc/tools/distrib/python/check_grpcio_tools.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import make_grpcio_tools as _make + +OUT_OF_DATE_MESSAGE = """file {} is out of date + +Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protobuf?""" + +submodule_commit_hash = _make.protobuf_submodule_commit_hash() + +with open(_make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as _protoc_lib_deps_file: + content = _protoc_lib_deps_file.read().splitlines() + +testString = (_make.COMMIT_HASH_PREFIX + submodule_commit_hash + + _make.COMMIT_HASH_SUFFIX) + +if testString not in content: + print(OUT_OF_DATE_MESSAGE.format(_make.GRPC_PYTHON_PROTOC_LIB_DEPS)) + raise SystemExit(1) diff --git a/grpc/tools/distrib/python/docgen.py b/grpc/tools/distrib/python/docgen.py new file mode 100755 index 00000000..9c9e5910 --- /dev/null +++ b/grpc/tools/distrib/python/docgen.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import os +import os.path +import shutil +import subprocess +import sys +import tempfile + +parser = argparse.ArgumentParser() +parser.add_argument('--config', + metavar='c', + type=str, + nargs=1, + help='GRPC/GPR libraries build configuration', + default='opt') +parser.add_argument('--submit', action='store_true') +parser.add_argument('--repo-owner', + type=str, + help=('Owner of the GitHub repository to be pushed')) +parser.add_argument('--doc-branch', type=str) +args = parser.parse_args() + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..', '..')) + +CONFIG = args.config +SETUP_PATH = os.path.join(PROJECT_ROOT, 'setup.py') +REQUIREMENTS_PATH = os.path.join(PROJECT_ROOT, 'requirements.bazel.txt') +DOC_PATH = os.path.join(PROJECT_ROOT, 'doc/build') +INCLUDE_PATH = os.path.join(PROJECT_ROOT, 'include') +LIBRARY_PATH = os.path.join(PROJECT_ROOT, 'libs/{}'.format(CONFIG)) +VIRTUALENV_DIR = os.path.join(SCRIPT_DIR, 'distrib_virtualenv') +VIRTUALENV_PYTHON_PATH = os.path.join(VIRTUALENV_DIR, 'bin', 'python') +VIRTUALENV_PIP_PATH = os.path.join(VIRTUALENV_DIR, 'bin', 'pip') + +environment = os.environ.copy() +environment.update({ + 'CONFIG': CONFIG, + 'CFLAGS': '-I{}'.format(INCLUDE_PATH), + 'LDFLAGS': '-L{}'.format(LIBRARY_PATH), + 'LD_LIBRARY_PATH': LIBRARY_PATH, + 'GRPC_PYTHON_BUILD_WITH_CYTHON': '1', + 'GRPC_PYTHON_ENABLE_DOCUMENTATION_BUILD': '1', +}) + +subprocess_arguments_list = [ + { + 'args': ['virtualenv', VIRTUALENV_DIR], + 'env': environment + }, + { + 'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip==19.3.1'], + 'env': environment + }, + { + 'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH], + 'env': environment + }, + { + 'args': [VIRTUALENV_PIP_PATH, 'install', 'Sphinx~=1.8.1'], + 'env': environment + }, + { + 'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'doc'], + 'env': environment + }, +] + +for subprocess_arguments in subprocess_arguments_list: + print('Running command: {}'.format(subprocess_arguments['args'])) + subprocess.check_call(**subprocess_arguments) + +if not args.submit: + print('Please check generated Python doc inside doc/build') +elif args.submit: + assert args.repo_owner + assert args.doc_branch + github_repository_owner = args.repo_owner + # Create a temporary directory out of tree, checkout gh-pages from the + # specified repository, edit it, and push it. It's up to the user to then go + # onto GitHub and make a PR against grpc/grpc:gh-pages. + repo_parent_dir = tempfile.mkdtemp() + print('Documentation parent directory: {}'.format(repo_parent_dir)) + repo_dir = os.path.join(repo_parent_dir, 'grpc') + python_doc_dir = os.path.join(repo_dir, 'python') + doc_branch = args.doc_branch + + print('Cloning your repository...') + subprocess.check_call([ + 'git', + 'clone', + '--branch', + 'gh-pages', + 'https://github.com/grpc/grpc', + ], + cwd=repo_parent_dir) + subprocess.check_call(['git', 'checkout', '-b', doc_branch], cwd=repo_dir) + subprocess.check_call([ + 'git', 'remote', 'add', 'ssh-origin', + 'git@github.com:%s/grpc.git' % (github_repository_owner) + ], + cwd=repo_dir) + print('Updating documentation...') + shutil.rmtree(python_doc_dir, ignore_errors=True) + shutil.copytree(DOC_PATH, python_doc_dir) + print('Attempting to push documentation...') + try: + subprocess.check_call(['git', 'add', '--all'], cwd=repo_dir) + subprocess.check_call( + ['git', 'commit', '-m', 'Auto-update Python documentation'], + cwd=repo_dir) + subprocess.check_call( + ['git', 'push', '--set-upstream', 'ssh-origin', doc_branch], + cwd=repo_dir) + except subprocess.CalledProcessError: + print('Failed to push documentation. Examine this directory and push ' + 'manually: {}'.format(repo_parent_dir)) + sys.exit(1) + shutil.rmtree(repo_parent_dir) diff --git a/grpc/tools/distrib/python/grpcio_tools/.gitignore b/grpc/tools/distrib/python/grpcio_tools/.gitignore new file mode 100644 index 00000000..9f3a7360 --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/.gitignore @@ -0,0 +1,8 @@ +build/ +protobuf/ +grpc_plugin/ +grpc_root/ +*.c +*.cpp +*.egg-info +*.so diff --git a/grpc/tools/distrib/python/grpcio_tools/MANIFEST.in b/grpc/tools/distrib/python/grpcio_tools/MANIFEST.in new file mode 100644 index 00000000..49437518 --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/MANIFEST.in @@ -0,0 +1,8 @@ +include _parallel_compile_patch.py +include grpc_version.py +include protoc_deps.py +include protoc_lib_deps.py +include README.rst +graft grpc_tools +graft grpc_root +graft third_party diff --git a/grpc/tools/distrib/python/grpcio_tools/README.rst b/grpc/tools/distrib/python/grpcio_tools/README.rst new file mode 100644 index 00000000..15a76416 --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/README.rst @@ -0,0 +1,186 @@ +gRPC Python Tools +================= + +Package for gRPC Python tools. + +Supported Python Versions +------------------------- +Python >= 3.5 + +Deprecated Python Versions +-------------------------- +Python == 2.7. Python 2.7 support will be removed on January 1, 2020. + +Installation +------------ + +The gRPC Python tools package is available for Linux, Mac OS X, and Windows +running Python 2.7. + +Installing From PyPI +~~~~~~~~~~~~~~~~~~~~ + +If you are installing locally... + +:: + + $ pip install grpcio-tools + +Else system wide (on Ubuntu)... + +:: + + $ sudo pip install grpcio-tools + +If you're on Windows make sure that you installed the :code:`pip.exe` component +when you installed Python (if not go back and install it!) then invoke: + +:: + + $ pip.exe install grpcio-tools + +Windows users may need to invoke :code:`pip.exe` from a command line ran as +administrator. + +n.b. On Windows and on Mac OS X one *must* have a recent release of :code:`pip` +to retrieve the proper wheel from PyPI. Be sure to upgrade to the latest +version! + +You might also need to install Cython to handle installation via the source +distribution if gRPC Python's system coverage with wheels does not happen to +include your system. + +Installing From Source +~~~~~~~~~~~~~~~~~~~~~~ + +Building from source requires that you have the Python headers (usually a +package named :code:`python-dev`) and Cython installed. It further requires a +GCC-like compiler to go smoothly; you can probably get it to work without +GCC-like stuff, but you may end up having a bad time. + +:: + + $ export REPO_ROOT=grpc # REPO_ROOT can be any directory of your choice + $ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc $REPO_ROOT + $ cd $REPO_ROOT + $ git submodule update --init + + $ cd tools/distrib/python/grpcio_tools + $ python ../make_grpcio_tools.py + + # For the next command do `sudo pip install` if you get permission-denied errors + $ GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install . + +You cannot currently install Python from source on Windows. Things might work +out for you in MSYS2 (follow the Linux instructions), but it isn't officially +supported at the moment. + +Troubleshooting +~~~~~~~~~~~~~~~ + +Help, I ... + +* **... see a** :code:`pkg_resources.VersionConflict` **when I try to install + grpc** + + This is likely because :code:`pip` doesn't own the offending dependency, + which in turn is likely because your operating system's package manager owns + it. You'll need to force the installation of the dependency: + + :code:`pip install --ignore-installed $OFFENDING_DEPENDENCY` + + For example, if you get an error like the following: + + :: + + Traceback (most recent call last): + File "<string>", line 17, in <module> + ... + File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 509, in find + raise VersionConflict(dist, req) + pkg_resources.VersionConflict: (six 1.8.0 (/usr/lib/python2.7/dist-packages), Requirement.parse('six>=1.10')) + + You can fix it by doing: + + :: + + sudo pip install --ignore-installed six + +* **... see compiler errors on some platforms when either installing from source or from the source distribution** + + If you see + + :: + + /tmp/pip-build-U8pSsr/cython/Cython/Plex/Scanners.c:4:20: fatal error: Python.h: No such file or directory + #include "Python.h" + ^ + compilation terminated. + + You can fix it by installing `python-dev` package. i.e + + :: + + sudo apt-get install python-dev + + If you see something similar to: + + :: + + third_party/protobuf/src/google/protobuf/stubs/mathlimits.h:173:31: note: in expansion of macro 'SIGNED_INT_MAX' + static const Type kPosMax = SIGNED_INT_MAX(Type); \\ + ^ + + And your toolchain is GCC (at the time of this writing, up through at least + GCC 6.0), this is probably a bug where GCC chokes on constant expressions + when the :code:`-fwrapv` flag is specified. You should consider setting your + environment with :code:`CFLAGS=-fno-wrapv` or using clang (:code:`CC=clang`). + +Usage +----- + +Given protobuf include directories :code:`$INCLUDE`, an output directory +:code:`$OUTPUT`, and proto files :code:`$PROTO_FILES`, invoke as: + +:: + + $ python -m grpc.tools.protoc -I$INCLUDE --python_out=$OUTPUT --grpc_python_out=$OUTPUT $PROTO_FILES + +To use as a build step in distutils-based projects, you may use the provided +command class in your :code:`setup.py`: + +:: + + setuptools.setup( + # ... + cmdclass={ + 'build_proto_modules': grpc.tools.command.BuildPackageProtos, + } + # ... + ) + +Invocation of the command will walk the project tree and transpile every +:code:`.proto` file into a :code:`_pb2.py` file in the same directory. + +Note that this particular approach requires :code:`grpcio-tools` to be +installed on the machine before the setup script is invoked (i.e. no +combination of :code:`setup_requires` or :code:`install_requires` will provide +access to :code:`grpc.tools.command.BuildPackageProtos` if it isn't already +installed). One way to work around this can be found in our +:code:`grpcio-health-checking` +`package <https://pypi.python.org/pypi/grpcio-health-checking>`_: + +:: + + class BuildPackageProtos(setuptools.Command): + """Command to generate project *_pb2.py modules from proto files.""" + # ... + def run(self): + from grpc.tools import command + command.build_package_protos(self.distribution.package_dir['']) + +Now including :code:`grpcio-tools` in :code:`setup_requires` will provide the +command on-setup as desired. + +For more information on command classes, consult :code:`distutils` and +:code:`setuptools` documentation. diff --git a/grpc/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py b/grpc/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py new file mode 100644 index 00000000..4d03ef49 --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py @@ -0,0 +1,63 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Patches the compile() to allow enable parallel compilation of C/C++. + +build_ext has lots of C/C++ files and normally them one by one. +Enabling parallel build helps a lot. +""" + +import distutils.ccompiler +import os + +try: + BUILD_EXT_COMPILER_JOBS = int( + os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1')) +except ValueError: + BUILD_EXT_COMPILER_JOBS = 1 + + +# monkey-patch for parallel compilation +def _parallel_compile(self, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + # setup the same way as distutils.ccompiler.CCompiler + # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564 + macros, objects, extra_postargs, pp_opts, build = self._setup_compile( + output_dir, macros, include_dirs, sources, depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + def _compile_single_file(obj): + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # run compilation of individual files in parallel + import multiprocessing.pool + multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map( + _compile_single_file, objects) + return objects + + +def monkeypatch_compile_maybe(): + """Monkeypatching is dumb, but the build speed gain is worth it.""" + if BUILD_EXT_COMPILER_JOBS > 1: + distutils.ccompiler.CCompiler.compile = _parallel_compile diff --git a/grpc/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py new file mode 100644 index 00000000..5772620b --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/grpc/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx new file mode 100644 index 00000000..481b5a4f --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/_protoc_compiler.pyx @@ -0,0 +1,24 @@ +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libc cimport stdlib + +cdef extern from "grpc_tools/main.h": + int protoc_main(int argc, char *argv[]) + +def run_main(list args not None): + cdef char **argv = <char **>stdlib.malloc(len(args)*sizeof(char *)) + for i in range(len(args)): + argv[i] = args[i] + return protoc_main(len(args), argv) diff --git a/grpc/tools/distrib/python/grpcio_tools/grpc_tools/command.py b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/command.py new file mode 100644 index 00000000..b1669db3 --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/command.py @@ -0,0 +1,70 @@ +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pkg_resources +import sys + +import setuptools + +from grpc_tools import protoc + + +def build_package_protos(package_root, strict_mode=False): + proto_files = [] + inclusion_root = os.path.abspath(package_root) + for root, _, files in os.walk(inclusion_root): + for filename in files: + if filename.endswith('.proto'): + proto_files.append(os.path.abspath(os.path.join(root, + filename))) + + well_known_protos_include = pkg_resources.resource_filename( + 'grpc_tools', '_proto') + + for proto_file in proto_files: + command = [ + 'grpc_tools.protoc', + '--proto_path={}'.format(inclusion_root), + '--proto_path={}'.format(well_known_protos_include), + '--python_out={}'.format(inclusion_root), + '--grpc_python_out={}'.format(inclusion_root), + ] + [proto_file] + if protoc.main(command) != 0: + if strict_mode: + raise Exception('error: {} failed'.format(command)) + else: + sys.stderr.write('warning: {} failed'.format(command)) + + +class BuildPackageProtos(setuptools.Command): + """Command to generate project *_pb2.py modules from proto files.""" + + description = 'build grpc protobuf modules' + user_options = [('strict-mode', 's', + 'exit with non-zero value if the proto compiling fails.')] + + def initialize_options(self): + self.strict_mode = False + + def finalize_options(self): + pass + + def run(self): + # due to limitations of the proto generator, we require that only *one* + # directory is provided as an 'include' directory. We assume it's the '' key + # to `self.distribution.package_dir` (and get a key error if it's not + # there). + build_package_protos(self.distribution.package_dir[''], + self.strict_mode) diff --git a/grpc/tools/distrib/python/grpcio_tools/grpc_tools/main.cc b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/main.cc new file mode 100644 index 00000000..65933865 --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/main.cc @@ -0,0 +1,38 @@ +// Copyright 2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <google/protobuf/compiler/command_line_interface.h> +#include <google/protobuf/compiler/python/python_generator.h> + +#include "src/compiler/python_generator.h" + +#include "grpc_tools/main.h" + +int protoc_main(int argc, char* argv[]) { + google::protobuf::compiler::CommandLineInterface cli; + cli.AllowPlugins("protoc-"); + + // Proto2 Python + google::protobuf::compiler::python::Generator py_generator; + cli.RegisterGenerator("--python_out", &py_generator, + "Generate Python source file."); + + // gRPC Python + grpc_python_generator::GeneratorConfiguration grpc_py_config; + grpc_python_generator::PythonGrpcGenerator grpc_py_generator(grpc_py_config); + cli.RegisterGenerator("--grpc_python_out", &grpc_py_generator, + "Generate Python source file."); + + return cli.Run(argc, argv); +} diff --git a/grpc/tools/distrib/python/grpcio_tools/grpc_tools/main.h b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/main.h new file mode 100644 index 00000000..9a1df202 --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/main.h @@ -0,0 +1,18 @@ +// Copyright 2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// We declare `protoc_main` here since we want access to it from Cython as an +// extern but *without* triggering a dllimport declspec when on Windows. +int protoc_main(int argc, char *argv[]); diff --git a/grpc/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py new file mode 100644 index 00000000..582cba0e --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pkg_resources +import sys + +from grpc_tools import _protoc_compiler + + +def main(command_arguments): + """Run the protocol buffer compiler with the given command-line arguments. + + Args: + command_arguments: a list of strings representing command line arguments to + `protoc`. + """ + command_arguments = [argument.encode() for argument in command_arguments] + return _protoc_compiler.run_main(command_arguments) + + +if __name__ == '__main__': + proto_include = pkg_resources.resource_filename('grpc_tools', '_proto') + sys.exit(main(sys.argv + ['-I{}'.format(proto_include)])) diff --git a/grpc/tools/distrib/python/grpcio_tools/grpc_version.py b/grpc/tools/distrib/python/grpcio_tools/grpc_version.py new file mode 100644 index 00000000..2aeab682 --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/grpc_version.py @@ -0,0 +1,17 @@ +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! + +VERSION = '1.29.1' diff --git a/grpc/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/grpc/tools/distrib/python/grpcio_tools/protoc_lib_deps.py new file mode 100644 index 00000000..a557be55 --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -0,0 +1,23 @@ + +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AUTO-GENERATED BY make_grpcio_tools.py! +CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/parse_context.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/io_win32.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/implicit_weak_message.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/generated_enum_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arena.cc', 'google/protobuf/any_lite.cc'] +PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto'] + +CC_INCLUDE='third_party/protobuf/src' +PROTO_INCLUDE='third_party/protobuf/src' + +PROTOBUF_SUBMODULE_VERSION="fe1790ca0df67173702f70d5646b82f48f412b99" diff --git a/grpc/tools/distrib/python/grpcio_tools/setup.py b/grpc/tools/distrib/python/grpcio_tools/setup.py new file mode 100644 index 00000000..90347b52 --- /dev/null +++ b/grpc/tools/distrib/python/grpcio_tools/setup.py @@ -0,0 +1,226 @@ +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from distutils import cygwinccompiler +from distutils import extension +from distutils import util +import errno +import os +import os.path +import pkg_resources +import platform +import re +import shlex +import shutil +import sys +import sysconfig + +import setuptools +from setuptools.command import build_ext + +import subprocess +from subprocess import PIPE + +# TODO(atash) add flag to disable Cython use + +_PACKAGE_PATH = os.path.realpath(os.path.dirname(__file__)) +_README_PATH = os.path.join(_PACKAGE_PATH, 'README.rst') + +os.chdir(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.abspath('.')) + +import _parallel_compile_patch +import protoc_lib_deps +import grpc_version + +_parallel_compile_patch.monkeypatch_compile_maybe() + +CLASSIFIERS = [ + 'Development Status :: 5 - Production/Stable', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'License :: OSI Approved :: Apache Software License', +] + +PY3 = sys.version_info.major == 3 + +# Environment variable to determine whether or not the Cython extension should +# *use* Cython or use the generated C files. Note that this requires the C files +# to have been generated by building first *with* Cython support. +BUILD_WITH_CYTHON = os.environ.get('GRPC_PYTHON_BUILD_WITH_CYTHON', False) + + +def check_linker_need_libatomic(): + """Test if linker on system needs libatomic.""" + code_test = (b'#include <atomic>\n' + + b'int main() { return std::atomic<int64_t>{}; }') + cc_test = subprocess.Popen(['cc', '-x', 'c++', '-std=c++11', '-'], + stdin=PIPE, + stdout=PIPE, + stderr=PIPE) + cc_test.communicate(input=code_test) + return cc_test.returncode != 0 + + +# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are +# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. +# We use these environment variables to thus get around that without locking +# ourselves in w.r.t. the multitude of operating systems this ought to build on. +# We can also use these variables as a way to inject environment-specific +# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a +# reasonable default. +EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None) +EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None) +if EXTRA_ENV_COMPILE_ARGS is None: + EXTRA_ENV_COMPILE_ARGS = '-std=c++11' + if 'win32' in sys.platform: + if sys.version_info < (3, 5): + # We use define flags here and don't directly add to DEFINE_MACROS below to + # ensure that the expert user/builder has a way of turning it off (via the + # envvars) without adding yet more GRPC-specific envvars. + # See https://sourceforge.net/p/mingw-w64/bugs/363/ + if '32' in platform.architecture()[0]: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s -D_hypot=hypot' + else: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64 -D_hypot=hypot' + else: + # We need to statically link the C++ Runtime, only the C runtime is + # available dynamically + EXTRA_ENV_COMPILE_ARGS += ' /MT' + elif "linux" in sys.platform or "darwin" in sys.platform: + EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv -frtti' +if EXTRA_ENV_LINK_ARGS is None: + EXTRA_ENV_LINK_ARGS = '' + if "linux" in sys.platform or "darwin" in sys.platform: + EXTRA_ENV_LINK_ARGS += ' -lpthread' + if check_linker_need_libatomic(): + EXTRA_ENV_LINK_ARGS += ' -latomic' + elif "win32" in sys.platform and sys.version_info < (3, 5): + msvcr = cygwinccompiler.get_msvcr()[0] + EXTRA_ENV_LINK_ARGS += ( + ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr}' + ' -static -lshlwapi'.format(msvcr=msvcr)) + +EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) +EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) + +CC_FILES = [os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES] +PROTO_FILES = [ + os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES +] +CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE) +PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE) + +GRPC_PYTHON_TOOLS_PACKAGE = 'grpc_tools' +GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto' + +DEFINE_MACROS = () +if "win32" in sys.platform: + DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1),) + if '64bit' in platform.architecture()[0]: + DEFINE_MACROS += (('MS_WIN64', 1),) +elif "linux" in sys.platform or "darwin" in sys.platform: + DEFINE_MACROS += (('HAVE_PTHREAD', 1),) + +# By default, Python3 distutils enforces compatibility of +# c plugins (.so files) with the OSX version Python3 was built with. +# For Python3.4, this is OSX 10.6, but we need Thread Local Support (__thread) +if 'darwin' in sys.platform and PY3: + mac_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if mac_target and (pkg_resources.parse_version(mac_target) < + pkg_resources.parse_version('10.9.0')): + os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9' + os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( + r'macosx-[0-9]+\.[0-9]+-(.+)', r'macosx-10.9-\1', + util.get_platform()) + + +def package_data(): + tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep) + proto_resources_path = os.path.join(tools_path, + GRPC_PYTHON_PROTO_RESOURCES_NAME) + proto_files = [] + for proto_file in PROTO_FILES: + source = os.path.join(PROTO_INCLUDE, proto_file) + target = os.path.join(proto_resources_path, proto_file) + relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, + proto_file) + try: + os.makedirs(os.path.dirname(target)) + except OSError as error: + if error.errno == errno.EEXIST: + pass + else: + raise + shutil.copy(source, target) + proto_files.append(relative_target) + return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files} + + +def extension_modules(): + if BUILD_WITH_CYTHON: + plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.pyx')] + else: + plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.cpp')] + + plugin_sources += [ + os.path.join('grpc_tools', 'main.cc'), + os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc') + ] + [os.path.join(CC_INCLUDE, cc_file) for cc_file in CC_FILES] + + plugin_ext = extension.Extension( + name='grpc_tools._protoc_compiler', + sources=plugin_sources, + include_dirs=[ + '.', + 'grpc_root', + os.path.join('grpc_root', 'include'), + CC_INCLUDE, + ], + language='c++', + define_macros=list(DEFINE_MACROS), + extra_compile_args=list(EXTRA_COMPILE_ARGS), + extra_link_args=list(EXTRA_LINK_ARGS), + ) + extensions = [plugin_ext] + if BUILD_WITH_CYTHON: + from Cython import Build + return Build.cythonize(extensions) + else: + return extensions + + +setuptools.setup( + name='grpcio-tools', + version=grpc_version.VERSION, + description='Protobuf code generator for gRPC', + long_description=open(_README_PATH, 'r').read(), + author='The gRPC Authors', + author_email='grpc-io@googlegroups.com', + url='https://grpc.io', + license='Apache License 2.0', + classifiers=CLASSIFIERS, + ext_modules=extension_modules(), + packages=setuptools.find_packages('.'), + install_requires=[ + 'protobuf>=3.5.0.post1', + 'grpcio>={version}'.format(version=grpc_version.VERSION), + ], + package_data=package_data(), +) diff --git a/grpc/tools/distrib/python/make_grpcio_tools.py b/grpc/tools/distrib/python/make_grpcio_tools.py new file mode 100755 index 00000000..a6fdae28 --- /dev/null +++ b/grpc/tools/distrib/python/make_grpcio_tools.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python + +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import errno +import filecmp +import glob +import os +import os.path +import shutil +import subprocess +import sys +import traceback +import uuid + +DEPS_FILE_CONTENT = """ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AUTO-GENERATED BY make_grpcio_tools.py! +CC_FILES={cc_files} +PROTO_FILES={proto_files} + +CC_INCLUDE={cc_include} +PROTO_INCLUDE={proto_include} + +{commit_hash} +""" + +COMMIT_HASH_PREFIX = 'PROTOBUF_SUBMODULE_VERSION="' +COMMIT_HASH_SUFFIX = '"' + +# Bazel query result prefix for expected source files in protobuf. +PROTOBUF_CC_PREFIX = '//:src/' +PROTOBUF_PROTO_PREFIX = '//:src/' + +GRPC_ROOT = os.path.abspath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..')) + +GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', + 'grpcio_tools') + +GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = os.path.join('third_party', 'protobuf', + 'src') +GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT) +GRPC_PROTOBUF_SUBMODULE_ROOT = os.path.join(GRPC_ROOT, 'third_party', + 'protobuf') +GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src', 'compiler') +GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, 'third_party', 'protobuf', + 'src') +GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root', 'src', + 'compiler') +GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_PYTHON_ROOT, + 'protoc_lib_deps.py') + +GRPC_INCLUDE = os.path.join(GRPC_ROOT, 'include') +GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root', 'include') + +BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', + 'bazel_deps.sh') +BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib' +BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos' + + +def protobuf_submodule_commit_hash(): + """Gets the commit hash for the HEAD of the protobuf submodule currently + checked out.""" + cwd = os.getcwd() + os.chdir(GRPC_PROTOBUF_SUBMODULE_ROOT) + output = subprocess.check_output(['git', 'rev-parse', 'HEAD']) + os.chdir(cwd) + return output.splitlines()[0].strip() + + +def bazel_query(query): + print('Running "bazel query %s"' % query) + output = subprocess.check_output([BAZEL_DEPS, query]) + return output.splitlines() + + +def get_deps(): + """Write the result of the bazel query `query` against protobuf to + `out_file`.""" + cc_files_output = bazel_query(BAZEL_DEPS_PROTOC_LIB_QUERY) + cc_files = [ + name[len(PROTOBUF_CC_PREFIX):] + for name in cc_files_output + if name.endswith('.cc') and name.startswith(PROTOBUF_CC_PREFIX) + ] + proto_files_output = bazel_query(BAZEL_DEPS_COMMON_PROTOS_QUERY) + proto_files = [ + name[len(PROTOBUF_PROTO_PREFIX):] + for name in proto_files_output + if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX) + ] + commit_hash = protobuf_submodule_commit_hash() + deps_file_content = DEPS_FILE_CONTENT.format( + cc_files=cc_files, + proto_files=proto_files, + cc_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT), + proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT), + commit_hash=COMMIT_HASH_PREFIX + commit_hash + COMMIT_HASH_SUFFIX) + return deps_file_content + + +def long_path(path): + if os.name == 'nt': + return '\\\\?\\' + path + else: + return path + + +def main(): + os.chdir(GRPC_ROOT) + + for source, target in [(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF), + (GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS), + (GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)]: + for source_dir, _, files in os.walk(source): + target_dir = os.path.abspath( + os.path.join(target, os.path.relpath(source_dir, source))) + try: + os.makedirs(target_dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise + for relative_file in files: + source_file = os.path.abspath( + os.path.join(source_dir, relative_file)) + target_file = os.path.abspath( + os.path.join(target_dir, relative_file)) + shutil.copyfile(source_file, target_file) + + try: + print('Invoking "bazel query" to gather the protobuf dependencies.') + protoc_lib_deps_content = get_deps() + except Exception as error: + # We allow this script to succeed even if we couldn't get the dependencies, + # as then we can assume that even without a successful bazel run the + # dependencies currently in source control are 'good enough'. + sys.stderr.write("Got non-fatal error:\n") + traceback.print_exc(file=sys.stderr) + return + # If we successfully got the dependencies, truncate and rewrite the deps file. + with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file: + deps_file.write(protoc_lib_deps_content) + print('File "%s" updated.' % GRPC_PYTHON_PROTOC_LIB_DEPS) + print('Done.') + + +if __name__ == '__main__': + main() diff --git a/grpc/tools/distrib/python/submit.py b/grpc/tools/distrib/python/submit.py new file mode 100755 index 00000000..6ce6462b --- /dev/null +++ b/grpc/tools/distrib/python/submit.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import shutil +import subprocess + +parser = argparse.ArgumentParser( + description='Submit the package to a PyPI repository.') +parser.add_argument('--repository', + '-r', + metavar='r', + type=str, + default='pypi', + help='The repository to push the package to. ' + 'Ensure the value appears in your .pypirc file. ' + 'Defaults to "pypi".') +parser.add_argument('--identity', + '-i', + metavar='i', + type=str, + help='GPG identity to sign the files with.') +parser.add_argument( + '--username', + '-u', + metavar='u', + type=str, + help='Username to authenticate with the repository. Not needed if you have ' + 'configured your .pypirc to include your username.') +parser.add_argument( + '--password', + '-p', + metavar='p', + type=str, + help='Password to authenticate with the repository. Not needed if you have ' + 'configured your .pypirc to include your password.') +parser.add_argument( + '--bdist', + '-b', + action='store_true', + help='Generate a binary distribution (wheel) for the current OS.') +parser.add_argument( + '--dist-args', + type=str, + help='Additional arguments to pass to the *dist setup.py command.') +args = parser.parse_args() + +# Move to the root directory of Python GRPC. +pkgdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../') +# Remove previous distributions; they somehow confuse twine. +try: + shutil.rmtree(os.path.join(pkgdir, 'dist/')) +except: + pass + +# Build the Cython C files +build_env = os.environ.copy() +build_env['GRPC_PYTHON_BUILD_WITH_CYTHON'] = "1" +cmd = ['python', 'setup.py', 'build_ext', '--inplace'] +subprocess.call(cmd, cwd=pkgdir, env=build_env) + +# Make the push. +if args.bdist: + cmd = ['python', 'setup.py', 'bdist_wheel'] +else: + cmd = ['python', 'setup.py', 'sdist'] +if args.dist_args: + cmd += args.dist_args.split() +subprocess.call(cmd, cwd=pkgdir) + +cmd = ['twine', 'upload', '-r', args.repository] +if args.identity is not None: + cmd.extend(['-i', args.identity]) +if args.username is not None: + cmd.extend(['-u', args.username]) +if args.password is not None: + cmd.extend(['-p', args.password]) +cmd.append('dist/*') + +subprocess.call(cmd, cwd=pkgdir) diff --git a/grpc/tools/distrib/python_wrapper.sh b/grpc/tools/distrib/python_wrapper.sh new file mode 100755 index 00000000..a099b2f1 --- /dev/null +++ b/grpc/tools/distrib/python_wrapper.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +for p in python2.7 python2.6 python2 python not_found ; do + + python=$(which $p || echo not_found) + + if [ -x "$python" ] ; then + break + fi + +done + +if [ -x "$python" ] ; then + exec "$python" "$@" +else + echo "No acceptable version of python found on the system" + exit 1 +fi diff --git a/grpc/tools/distrib/run_clang_tidy.py b/grpc/tools/distrib/run_clang_tidy.py new file mode 100755 index 00000000..6e54a759 --- /dev/null +++ b/grpc/tools/distrib/run_clang_tidy.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python2.7 +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import subprocess +import argparse +import multiprocessing + +sys.path.append( + os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests', + 'python_utils')) +import jobset + +extra_args = [ + '-x', + 'c++', + '-std=c++11', +] +with open('.clang_complete') as f: + for line in f: + line = line.strip() + if line.startswith('-I'): + extra_args.append(line) + +clang_tidy = os.environ.get('CLANG_TIDY', 'clang-tidy') + +argp = argparse.ArgumentParser(description='Run clang-tidy against core') +argp.add_argument('files', nargs='+', help='Files to tidy') +argp.add_argument('--fix', dest='fix', action='store_true') +argp.add_argument('-j', + '--jobs', + type=int, + default=multiprocessing.cpu_count(), + help='Number of CPUs to use') +argp.set_defaults(fix=False) +args = argp.parse_args() + +cmdline = [ + clang_tidy, +] + ['--extra-arg-before=%s' % arg for arg in extra_args] + +if args.fix: + cmdline.append('--fix') + +jobs = [] +for filename in args.files: + jobs.append(jobset.JobSpec( + cmdline + [filename], + shortname=filename, + )) #verbose_success=True)) + +num_fails, res_set = jobset.run(jobs, maxjobs=args.jobs) +sys.exit(num_fails) diff --git a/grpc/tools/distrib/sanitize.sh b/grpc/tools/distrib/sanitize.sh new file mode 100755 index 00000000..3319bcdc --- /dev/null +++ b/grpc/tools/distrib/sanitize.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd $(dirname $0)/../.. + +tools/buildgen/generate_projects.sh +tools/distrib/clang_format_code.sh +tools/distrib/check_copyright.py +tools/distrib/check_trailing_newlines.sh + diff --git a/grpc/tools/distrib/yapf_code.sh b/grpc/tools/distrib/yapf_code.sh new file mode 100755 index 00000000..4c9639ba --- /dev/null +++ b/grpc/tools/distrib/yapf_code.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +ACTION=${1:---in-place} +[[ $ACTION == '--in-place' ]] || [[ $ACTION == '--diff' ]] + +# change to root directory +cd "$(dirname "${0}")/../.." + +DIRS=( + 'examples' + 'src' + 'test' + 'tools' +) + +VIRTUALENV=yapf_virtual_environment + +python3 -m virtualenv $VIRTUALENV -p $(which python3) +PYTHON=${VIRTUALENV}/bin/python +"$PYTHON" -m pip install --upgrade pip==19.3.1 +"$PYTHON" -m pip install --upgrade futures +"$PYTHON" -m pip install yapf==0.28.0 + +$PYTHON -m yapf $ACTION --recursive --style=setup.cfg "${DIRS[@]}" |