diff options
Diffstat (limited to 'grpc/tools/buildgen/generate_projects.py')
| -rwxr-xr-x | grpc/tools/buildgen/generate_projects.py | 198 |
1 files changed, 115 insertions, 83 deletions
diff --git a/grpc/tools/buildgen/generate_projects.py b/grpc/tools/buildgen/generate_projects.py index 841abbdc..2e64484f 100755 --- a/grpc/tools/buildgen/generate_projects.py +++ b/grpc/tools/buildgen/generate_projects.py @@ -14,99 +14,131 @@ import argparse import glob +import yaml +import pickle import os import shutil import sys import tempfile import multiprocessing -sys.path.append( - os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests', - 'python_utils')) +from typing import Union, Dict, List + +import _utils + +PROJECT_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", + "..") +os.chdir(PROJECT_ROOT) +# TODO(lidiz) find a better way for plugins to reference each other +sys.path.append(os.path.join(PROJECT_ROOT, 'tools', 'buildgen', 'plugins')) + +# from tools.run_tests.python_utils import jobset +jobset = _utils.import_python_module( + os.path.join(PROJECT_ROOT, 'tools', 'run_tests', 'python_utils', + 'jobset.py')) + +PREPROCESSED_BUILD = '.preprocessed_build' +test = {} if os.environ.get('TEST', 'false') == 'true' else None assert sys.argv[1:], 'run generate_projects.sh instead of this directly' +parser = argparse.ArgumentParser() +parser.add_argument('build_files', + nargs='+', + default=[], + help="build files describing build specs") +parser.add_argument('--templates', + nargs='+', + default=[], + help="mako template files to render") +parser.add_argument('--output_merged', + '-m', + default='', + type=str, + help="merge intermediate results to a file") +parser.add_argument('--jobs', + '-j', + default=multiprocessing.cpu_count(), + type=int, + help="maximum parallel jobs") +parser.add_argument('--base', + default='.', + type=str, + help="base path for generated files") +args = parser.parse_args() -import jobset -os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) +def preprocess_build_files() -> _utils.Bunch: + """Merges build yaml into a one dictionary then pass it to plugins.""" + build_spec = dict() + for build_file in args.build_files: + with open(build_file, 'r') as f: + _utils.merge_json(build_spec, + yaml.load(f.read(), Loader=yaml.FullLoader)) + # Executes plugins. Plugins update the build spec in-place. + for py_file in sorted(glob.glob('tools/buildgen/plugins/*.py')): + plugin = _utils.import_python_module(py_file) + plugin.mako_plugin(build_spec) + if args.output_merged: + with open(args.output_merged, 'w') as f: + f.write(yaml.dump(build_spec)) + # Makes build_spec sort of immutable and dot-accessible + return _utils.to_bunch(build_spec) -argp = argparse.ArgumentParser() -argp.add_argument('build_files', nargs='+', default=[]) -argp.add_argument('--templates', nargs='+', default=[]) -argp.add_argument('--output_merged', default=None, type=str) -argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int) -argp.add_argument('--base', default='.', type=str) -args = argp.parse_args() -json = args.build_files +def generate_template_render_jobs(templates: List[str]) -> List[jobset.JobSpec]: + """Generate JobSpecs for each one of the template rendering work.""" + jobs = [] + base_cmd = [sys.executable, 'tools/buildgen/_mako_renderer.py'] + for template in sorted(templates, reverse=True): + root, f = os.path.split(template) + if os.path.splitext(f)[1] == '.template': + out_dir = args.base + root[len('templates'):] + out = os.path.join(out_dir, os.path.splitext(f)[0]) + if not os.path.exists(out_dir): + os.makedirs(out_dir) + cmd = base_cmd[:] + cmd.append('-P') + cmd.append(PREPROCESSED_BUILD) + cmd.append('-o') + if test is None: + cmd.append(out) + else: + tf = tempfile.mkstemp() + test[out] = tf[1] + os.close(tf[0]) + cmd.append(test[out]) + cmd.append(args.base + '/' + root + '/' + f) + jobs.append(jobset.JobSpec(cmd, shortname=out, + timeout_seconds=None)) + return jobs + + +def main() -> None: + templates = args.templates + if not templates: + for root, _, files in os.walk('templates'): + for f in files: + templates.append(os.path.join(root, f)) + + build_spec = preprocess_build_files() + with open(PREPROCESSED_BUILD, 'wb') as f: + pickle.dump(build_spec, f) + + err_cnt, _ = jobset.run(generate_template_render_jobs(templates), + maxjobs=args.jobs) + if err_cnt != 0: + print('ERROR: %s error(s) found while generating projects.' % err_cnt, + file=sys.stderr) + sys.exit(1) + + if test is not None: + for s, g in test.items(): + if os.path.isfile(g): + assert 0 == os.system('diff %s %s' % (s, g)), s + os.unlink(g) + else: + assert 0 == os.system('diff -r %s %s' % (s, g)), s + shutil.rmtree(g, ignore_errors=True) -test = {} if os.environ.get('TEST', 'false') == 'true' else None -plugins = sorted(glob.glob('tools/buildgen/plugins/*.py')) - -templates = args.templates -if not templates: - for root, dirs, files in os.walk('templates'): - for f in files: - templates.append(os.path.join(root, f)) - -pre_jobs = [] -base_cmd = [sys.executable, 'tools/buildgen/mako_renderer.py'] -cmd = base_cmd[:] -for plugin in plugins: - cmd.append('-p') - cmd.append(plugin) -for js in json: - cmd.append('-d') - cmd.append(js) -cmd.append('-w') -preprocessed_build = '.preprocessed_build' -cmd.append(preprocessed_build) -if args.output_merged is not None: - cmd.append('-M') - cmd.append(args.output_merged) -pre_jobs.append( - jobset.JobSpec(cmd, shortname='preprocess', timeout_seconds=None)) - -jobs = [] -for template in reversed(sorted(templates)): - root, f = os.path.split(template) - if os.path.splitext(f)[1] == '.template': - out_dir = args.base + root[len('templates'):] - out = out_dir + '/' + os.path.splitext(f)[0] - if not os.path.exists(out_dir): - os.makedirs(out_dir) - cmd = base_cmd[:] - cmd.append('-P') - cmd.append(preprocessed_build) - cmd.append('-o') - if test is None: - cmd.append(out) - else: - tf = tempfile.mkstemp() - test[out] = tf[1] - os.close(tf[0]) - cmd.append(test[out]) - cmd.append(args.base + '/' + root + '/' + f) - jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) - -err_cnt, _ = jobset.run(pre_jobs, maxjobs=args.jobs) -if err_cnt != 0: - print('ERROR: {count} error(s) encountered during preprocessing.'.format( - count=err_cnt), - file=sys.stderr) - sys.exit(1) -err_cnt, _ = jobset.run(jobs, maxjobs=args.jobs) -if err_cnt != 0: - print('ERROR: {count} error(s) found while generating projects.'.format( - count=err_cnt), - file=sys.stderr) - sys.exit(1) - -if test is not None: - for s, g in test.items(): - if os.path.isfile(g): - assert 0 == os.system('diff %s %s' % (s, g)), s - os.unlink(g) - else: - assert 0 == os.system('diff -r %s %s' % (s, g)), s - shutil.rmtree(g, ignore_errors=True) +if __name__ == "__main__": + main() |