Merge branch 'master' into migrate-client-context

11 files changed, 74 insertions, 78 deletions

diff --git a/tools/bazel.sh b/tools/bazel
index 37fd2a242b..bacfeaddfd 100755
--- a/tools/bazel.sh
+++ b/tools/bazel
@@ -19,17 +19,28 @@
 # supported version, and then calling it. This way, we can make sure
 # that running bazel will always get meaningful results, at least
 # until Bazel 1.0 is released.
+# NOTE: This script relies on bazel's feature where //tools/bazel
+# script can be used to hijack "bazel" invocations in given workspace.
 
 set -e
 
+# First of all, if DISABLE_BAZEL_WRAPPER is set, just use BAZEL_REAL as set by
+# https://github.com/bazelbuild/bazel/blob/master/scripts/packages/bazel.sh
+# that originally invoked this script.
+if [ "${BAZEL_REAL}" != "" ] && [ "${DISABLE_BAZEL_WRAPPER}" != "" ]
+then
+  exec -a "$0" "${BAZEL_REAL}" "$@"
+fi
+
 VERSION=0.24.1
 
-CWD=`pwd`
+echo "INFO: Running bazel wrapper (see //tools/bazel for details), bazel version $VERSION will be used instead of system-wide bazel installation."
+
 BASEURL=https://github.com/bazelbuild/bazel/releases/download/
-cd `dirname $0`
-TOOLDIR=`pwd`
+pushd "$(dirname "$0")" >/dev/null
+TOOLDIR=$(pwd)
 
-case `uname -sm` in
+case $(uname -sm) in
   "Linux x86_64")
     suffix=linux-x86_64
     ;;
@@ -37,17 +48,17 @@ case `uname -sm` in
     suffix=darwin-x86_64
     ;;
   *)
-    echo "Unsupported architecture: `uname -sm`"
+    echo "Unsupported architecture: $(uname -sm)"
     exit 1
     ;;
 esac
 
-filename=bazel-$VERSION-$suffix
+filename="bazel-$VERSION-$suffix"
 
-if [ ! -x $filename ] ; then
-  curl -L $BASEURL/$VERSION/$filename > $filename
-  chmod a+x $filename
+if [ ! -x "$filename" ] ; then
+  curl -L "$BASEURL/$VERSION/$filename" > "$filename"
+  chmod a+x "$filename"
 fi
 
-cd $CWD
-$TOOLDIR/$filename $@
+popd >/dev/null
+exec "$TOOLDIR/$filename" "$@"
diff --git a/tools/dockerfile/test/bazel/Dockerfile b/tools/dockerfile/test/bazel/Dockerfile
index b9fc409d93..df7a7a2f32 100644
--- a/tools/dockerfile/test/bazel/Dockerfile
+++ b/tools/dockerfile/test/bazel/Dockerfile
@@ -51,9 +51,12 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t
 #========================
 # Bazel installation
 
-# Must be in sync with tools/bazel.sh
+# Must be in sync with tools/bazel
 ENV BAZEL_VERSION 0.24.1
 
+# The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper.
+ENV DISABLE_BAZEL_WRAPPER 1
+
 RUN apt-get update && apt-get install -y wget && apt-get clean
 RUN wget "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh" && \
   bash ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile
index 4ef4a0a945..9a080d0efb 100644
--- a/tools/dockerfile/test/sanity/Dockerfile
+++ b/tools/dockerfile/test/sanity/Dockerfile
@@ -97,9 +97,12 @@ ENV CLANG_TIDY=clang-tidy
 #========================
 # Bazel installation
 
-# Must be in sync with tools/bazel.sh
+# Must be in sync with tools/bazel
 ENV BAZEL_VERSION 0.24.1
 
+# The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper.
+ENV DISABLE_BAZEL_WRAPPER 1
+
 RUN apt-get update && apt-get install -y wget && apt-get clean
 RUN wget "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh" && \
   bash ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh
index bcde79ecd8..5a358acc09 100755
--- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh
+++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh
@@ -21,7 +21,7 @@ cd $(dirname $0)/../../..
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
 # make sure bazel is available
-tools/bazel.sh version
+tools/bazel version
 
 # to get "bazel" link for kokoro build, we need to generate
 # invocation UUID, set a flag for bazel to use it
@@ -29,7 +29,7 @@ tools/bazel.sh version
 BAZEL_INVOCATION_ID="$(uuidgen)"
 echo "${BAZEL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids"
 
-tools/bazel.sh \
+tools/bazel \
   --bazelrc=tools/remote_build/kokoro.bazelrc \
   test \
   --invocation_id="${BAZEL_INVOCATION_ID}" \
diff --git a/tools/internal_ci/linux/grpc_bazel_rbe_incompatible_changes.sh b/tools/internal_ci/linux/grpc_bazel_rbe_incompatible_changes.sh
index 9c3712a07d..85b82443bc 100644
--- a/tools/internal_ci/linux/grpc_bazel_rbe_incompatible_changes.sh
+++ b/tools/internal_ci/linux/grpc_bazel_rbe_incompatible_changes.sh
@@ -15,9 +15,14 @@
 
 set -ex
 
-# TODO(jtattermusch): use the latest version of bazel
+# Use bazelisk to download the right bazel version
+wget https://github.com/bazelbuild/bazelisk/releases/download/v0.0.7/bazelisk-linux-amd64
+chmod u+x bazelisk-linux-amd64
 
-# Use --all_incompatible_changes to give an early warning about future
-# bazel incompatibilities.
-EXTRA_FLAGS="--config=opt --cache_test_results=no --all_incompatible_changes"
+# We want bazelisk to run the latest stable version
+export USE_BAZEL_VERSION=latest
+# Use bazelisk instead of our usual //tools/bazel wrapper
+mv bazelisk-linux-amd64 github/grpc/tools/bazel
+
+EXTRA_FLAGS="--config=opt --cache_test_results=no"
 github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}"
diff --git a/tools/internal_ci/macos/grpc_interop.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_build_only.cfg
index 434ecd19c4..0eb27206ee 100644
--- a/tools/internal_ci/macos/grpc_interop.cfg
+++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_build_only.cfg
@@ -1,4 +1,4 @@
-# Copyright 2017 gRPC authors.
+# Copyright 2019 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,12 +15,16 @@
 # Config file for the internal CI (in protobuf text format)
 
 # Location of the continuous shell script in repository.
-build_file: "grpc/tools/internal_ci/macos/grpc_interop.sh"
-gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
-timeout_mins: 240
+build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh"
+timeout_mins: 60
 action {
   define_artifacts {
     regex: "**/*sponge_log.*"
     regex: "github/grpc/reports/**"
   }
 }
+
+env_vars {
+  key: "RUN_TESTS_FLAGS"
+  value: "-f basictests linux corelang --inner_jobs 16 -j 1 --internal_ci --build_only"
+}
diff --git a/tools/internal_ci/macos/grpc_interop.sh b/tools/internal_ci/macos/grpc_interop.sh
deleted file mode 100755
index e290ed60c4..0000000000
--- a/tools/internal_ci/macos/grpc_interop.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-# Copyright 2017 gRPC authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-# change to grpc repo root
-cd $(dirname $0)/../../..
-
-source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc
-source tools/internal_ci/helper_scripts/prepare_build_macos_rc
-
-tools/run_tests/run_interop_tests.py -l objc -s all --use_docker -t -j 1 || FAILED="true"
-
-tools/internal_ci/helper_scripts/delete_nonartifacts.sh || true
-
-if [ "$FAILED" != "" ]
-then
-  exit 1
-fi
diff --git a/tools/internal_ci/macos/grpc_interop_toprod.sh b/tools/internal_ci/macos/grpc_interop_toprod.sh
index 36e5a2103d..13bf58ad1b 100755
--- a/tools/internal_ci/macos/grpc_interop_toprod.sh
+++ b/tools/internal_ci/macos/grpc_interop_toprod.sh
@@ -34,6 +34,7 @@ tools/run_tests/run_interop_tests.py -l c++ \
     --google_default_creds_use_key_file \
     --prod_servers default gateway_v4 \
     --service_account_key_file="${KOKORO_KEYSTORE_DIR}/73836_interop_to_prod_tests_service_account_key" \
+    --default_service_account="interop-to-prod-tests@grpc-testing.iam.gserviceaccount.com" \
     --skip_compute_engine_creds --internal_ci -t -j 4 || FAILED="true"
 
 tools/internal_ci/helper_scripts/delete_nonartifacts.sh || true
diff --git a/tools/internal_ci/windows/bazel_rbe.bat b/tools/internal_ci/windows/bazel_rbe.bat
index 30c66896ed..8abab687ed 100644
--- a/tools/internal_ci/windows/bazel_rbe.bat
+++ b/tools/internal_ci/windows/bazel_rbe.bat
@@ -13,7 +13,7 @@
 @rem limitations under the License.
 
 @rem TODO(jtattermusch): make this generate less output
-@rem TODO(jtattermusch): use tools/bazel.sh script to keep the versions in sync
+@rem TODO(jtattermusch): use tools/bazel script to keep the versions in sync
 choco install bazel -y --version 0.24.1 --limit-output
 
 cd github/grpc
diff --git a/tools/remote_build/README.md b/tools/remote_build/README.md
index 4cc3c7a0ea..2cd5f03daf 100644
--- a/tools/remote_build/README.md
+++ b/tools/remote_build/README.md
@@ -17,27 +17,22 @@ and tests run by Kokoro CI.
 
 ## Running remote build manually from dev workstation
 
-*At the time being, tools/bazel.sh is used instead of invoking "bazel" directly
-to overcome the bazel versioning problem (our BUILD files currently only work with
-a specific window of bazel version and bazel.sh wrapper makes sure that version
-is used).*
-
 Run from repository root (opt, dbg):
 ```
 # manual run of bazel tests remotely on Foundry
-tools/bazel.sh --bazelrc=tools/remote_build/manual.bazelrc test --config=opt //test/...
+bazel --bazelrc=tools/remote_build/manual.bazelrc test --config=opt //test/...
 ```
 
 Sanitizer runs (asan, msan, tsan, ubsan):
 ```
 # manual run of bazel tests remotely on Foundry with given sanitizer
-tools/bazel.sh --bazelrc=tools/remote_build/manual.bazelrc test --config=asan //test/...
+bazel --bazelrc=tools/remote_build/manual.bazelrc test --config=asan //test/...
 ```
 
 Run on Windows MSVC:
 ```
 # RBE manual run only for c-core (must be run on a Windows host machine)
-tools/bazel.sh --bazelrc=tools/remote_build/windows.bazelrc build :all [--credentials_json=(path to service account credentials)]
+bazel --bazelrc=tools/remote_build/windows.bazelrc build :all [--credentials_json=(path to service account credentials)]
 ```
 
 Available command line options can be found in
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 96e5ee340f..e71d308dcb 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -807,23 +807,17 @@ def compute_engine_creds_required(language, test_case):
     return False
 
 
-def auth_options(language,
-                 test_case,
-                 google_default_creds_use_key_file,
-                 service_account_key_file=None):
+def auth_options(language, test_case, google_default_creds_use_key_file,
+                 service_account_key_file, default_service_account):
     """Returns (cmdline, env) tuple with cloud_to_prod_auth test options."""
 
     language = str(language)
     cmdargs = []
     env = {}
 
-    if not service_account_key_file:
-        # this file path only works inside docker
-        service_account_key_file = '/root/service_account/grpc-testing-ebe7c1ac7381.json'
     oauth_scope_arg = '--oauth_scope=https://www.googleapis.com/auth/xapi.zoo'
     key_file_arg = '--service_account_key_file=%s' % service_account_key_file
-    # default compute engine credentials associated with the testing VMs in "grpc-testing" cloud project
-    default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com'
+    default_account_arg = '--default_service_account=%s' % default_service_account
 
     if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']:
         if language in [
@@ -874,6 +868,7 @@ def cloud_to_prod_jobspec(language,
                           auth=False,
                           manual_cmd_log=None,
                           service_account_key_file=None,
+                          default_service_account=None,
                           transport_security='tls'):
     """Creates jobspec for cloud-to-prod interop test"""
     container_name = None
@@ -901,9 +896,9 @@ def cloud_to_prod_jobspec(language,
     cmdargs = cmdargs + transport_security_options
     environ = dict(language.cloud_to_prod_env(), **language.global_env())
     if auth:
-        auth_cmdargs, auth_env = auth_options(language, test_case,
-                                              google_default_creds_use_key_file,
-                                              service_account_key_file)
+        auth_cmdargs, auth_env = auth_options(
+            language, test_case, google_default_creds_use_key_file,
+            service_account_key_file, default_service_account)
         cmdargs += auth_cmdargs
         environ.update(auth_env)
     cmdline = bash_cmdline(language.client_cmd(cmdargs))
@@ -1212,12 +1207,17 @@ argp.add_argument(
     help=
     'Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000',
     default=[])
+# TODO(jtattermusch): the default service_account_key_file only works when --use_docker is used.
 argp.add_argument(
     '--service_account_key_file',
     type=str,
-    help=
-    'Override the default service account key file to use for auth interop tests.',
-    default=None)
+    help='The service account key file to use for some auth interop tests.',
+    default='/root/service_account/grpc-testing-ebe7c1ac7381.json')
+argp.add_argument(
+    '--default_service_account',
+    type=str,
+    help='Default GCE service account email to use for some auth interop tests.',
+    default='830293263384-compute@developer.gserviceaccount.com')
 argp.add_argument(
     '-t', '--travis', default=False, action='store_const', const=True)
 argp.add_argument(
@@ -1470,6 +1470,8 @@ try:
                                     manual_cmd_log=client_manual_cmd_log,
                                     service_account_key_file=args.
                                     service_account_key_file,
+                                    default_service_account=args.
+                                    default_service_account,
                                     transport_security=transport_security)
                                 jobs.append(test_job)
             if args.http2_interop:
@@ -1484,6 +1486,7 @@ try:
                         docker_image=docker_images.get(str(http2Interop)),
                         manual_cmd_log=client_manual_cmd_log,
                         service_account_key_file=args.service_account_key_file,
+                        default_service_account=args.default_service_account,
                         transport_security=args.transport_security)
                     jobs.append(test_job)
 
@@ -1517,6 +1520,8 @@ try:
                                 manual_cmd_log=client_manual_cmd_log,
                                 service_account_key_file=args.
                                 service_account_key_file,
+                                default_service_account=args.
+                                default_service_account,
                                 transport_security=transport_security)
                             jobs.append(test_job)
     for server in args.override_server: