#!/usr/bin/env python3 # Copyright 2016 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Runs selected gRPC test/build tasks.""" from __future__ import print_function import argparse import multiprocessing import sys import artifacts.artifact_targets as artifact_targets import artifacts.distribtest_targets as distribtest_targets import artifacts.package_targets as package_targets import python_utils.jobset as jobset import python_utils.report_utils as report_utils _TARGETS = [] _TARGETS += artifact_targets.targets() _TARGETS += distribtest_targets.targets() _TARGETS += package_targets.targets() def _create_build_map(): """Maps task names and labels to list of tasks to be built.""" target_build_map = dict([(target.name, [target]) for target in _TARGETS]) if len(_TARGETS) > len(list(target_build_map.keys())): raise Exception("Target names need to be unique") label_build_map = {} label_build_map["all"] = [t for t in _TARGETS] # to build all targets for target in _TARGETS: for label in target.labels: if label in label_build_map: label_build_map[label].append(target) else: label_build_map[label] = [target] if set(target_build_map.keys()).intersection(list(label_build_map.keys())): raise Exception("Target names need to be distinct from label names") return dict(list(target_build_map.items()) + list(label_build_map.items())) _BUILD_MAP = _create_build_map() argp = argparse.ArgumentParser(description="Runs build/test targets.") argp.add_argument( "-b", "--build", choices=sorted(_BUILD_MAP.keys()), nargs="+", default=["all"], help="Target name or target label to build.", ) argp.add_argument( "-f", "--filter", choices=sorted(_BUILD_MAP.keys()), nargs="+", default=[], help="Filter targets to build with AND semantics.", ) argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int) argp.add_argument( "-x", "--xml_report", default="report_taskrunner_sponge_log.xml", type=str, help="Filename for the JUnit-compatible XML report", ) argp.add_argument( "--dry_run", default=False, action="store_const", const=True, help="Only print what would be run.", ) argp.add_argument( "--inner_jobs", default=None, type=int, help=( "Number of parallel jobs to use by each target. Passed as" " build_jobspec(inner_jobs=N) to each target." ), ) args = argp.parse_args() # Figure out which targets to build targets = [] for label in args.build: targets += _BUILD_MAP[label] # Among targets selected by -b, filter out those that don't match the filter targets = [t for t in targets if all(f in t.labels for f in args.filter)] print("Will build %d targets:" % len(targets)) for target in targets: print(" %s, labels %s" % (target.name, target.labels)) print() if args.dry_run: print("--dry_run was used, exiting") sys.exit(1) # Execute pre-build phase prebuild_jobs = [] for target in targets: prebuild_jobs += target.pre_build_jobspecs() if prebuild_jobs: num_failures, _ = jobset.run( prebuild_jobs, newline_on_success=True, maxjobs=args.jobs ) if num_failures != 0: jobset.message("FAILED", "Pre-build phase failed.", do_newline=True) sys.exit(1) build_jobs = [] for target in targets: build_jobs.append(target.build_jobspec(inner_jobs=args.inner_jobs)) if not build_jobs: print("Nothing to build.") sys.exit(1) jobset.message("START", "Building targets.", do_newline=True) num_failures, resultset = jobset.run( build_jobs, newline_on_success=True, maxjobs=args.jobs ) report_utils.render_junit_xml_report( resultset, args.xml_report, suite_name="tasks" ) if num_failures == 0: jobset.message( "SUCCESS", "All targets built successfully.", do_newline=True ) else: jobset.message("FAILED", "Failed to build targets.", do_newline=True) sys.exit(1)