diff options
Diffstat (limited to 'infra/build/functions/build_and_run_coverage.py')
| -rwxr-xr-x[-rw-r--r--] | infra/build/functions/build_and_run_coverage.py | 194 |
1 files changed, 78 insertions, 116 deletions
diff --git a/infra/build/functions/build_and_run_coverage.py b/infra/build/functions/build_and_run_coverage.py index cc2de5a32..1195776d9 100644..100755 --- a/infra/build/functions/build_and_run_coverage.py +++ b/infra/build/functions/build_and_run_coverage.py @@ -13,11 +13,11 @@ # limitations under the License. # ################################################################################ -#!/usr/bin/python2 +#!/usr/bin/env python3 """Starts and runs coverage build on Google Cloud Builder. -Usage: build_and_run_coverage.py <project_dir> + +Usage: build_and_run_coverage.py <project>. """ -import datetime import json import logging import os @@ -27,119 +27,105 @@ import build_lib import build_project SANITIZER = 'coverage' -CONFIGURATION = ['FUZZING_ENGINE=libfuzzer', 'SANITIZER=%s' % SANITIZER] +FUZZING_ENGINE = 'libfuzzer' +ARCHITECTURE = 'x86_64' + PLATFORM = 'linux' -COVERAGE_BUILD_TAG = 'coverage' +COVERAGE_BUILD_TYPE = 'coverage' # Where code coverage reports need to be uploaded to. COVERAGE_BUCKET_NAME = 'oss-fuzz-coverage' -# Link to the code coverage report in HTML format. -HTML_REPORT_URL_FORMAT = (build_lib.GCS_URL_BASENAME + COVERAGE_BUCKET_NAME + - '/{project}/reports/{date}/{platform}/index.html') - # This is needed for ClusterFuzz to pick up the most recent reports data. -LATEST_REPORT_INFO_URL = ('/' + COVERAGE_BUCKET_NAME + - '/latest_report_info/{project}.json') -LATEST_REPORT_INFO_CONTENT_TYPE = 'application/json' -# Link where to upload code coverage report files to. -UPLOAD_URL_FORMAT = 'gs://' + COVERAGE_BUCKET_NAME + '/{project}/{type}/{date}' +LATEST_REPORT_INFO_CONTENT_TYPE = 'application/json' # Languages from project.yaml that have code coverage support. -LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'rust'] +LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'jvm', 'rust', 'swift'] + + +class Bucket: # pylint: disable=too-few-public-methods + """Class representing the coverage GCS bucket.""" + def __init__(self, project, date, platform, testing): + self.coverage_bucket_name = 'oss-fuzz-coverage' + if testing: + self.coverage_bucket_name += '-testing' -def usage(): - """Exit with code 1 and display syntax to use this file.""" - sys.stderr.write("Usage: " + sys.argv[0] + " <project_dir>\n") - sys.exit(1) + self.date = date + self.project = project + self.html_report_url = ( + f'{build_lib.GCS_URL_BASENAME}{self.coverage_bucket_name}/{project}' + f'/reports/{date}/{platform}/index.html') + self.latest_report_info_url = (f'/{COVERAGE_BUCKET_NAME}' + f'/latest_report_info/{project}.json') + def get_upload_url(self, upload_type): + """Returns an upload url for |upload_type|.""" + return (f'gs://{self.coverage_bucket_name}/{self.project}' + f'/{upload_type}/{self.date}') -# pylint: disable=too-many-locals -def get_build_steps(project_name, project_yaml_file, dockerfile_lines, - image_project, base_images_project): + +def get_build_steps( # pylint: disable=too-many-locals, too-many-arguments + project_name, project_yaml_contents, dockerfile_lines, image_project, + base_images_project, config): """Returns build steps for project.""" - project_yaml = build_project.load_project_yaml(project_name, - project_yaml_file, - image_project) - if project_yaml['disabled']: - logging.info('Project "%s" is disabled.', project_name) + project = build_project.Project(project_name, project_yaml_contents, + dockerfile_lines, image_project) + if project.disabled: + logging.info('Project "%s" is disabled.', project.name) return [] - if project_yaml['language'] not in LANGUAGES_WITH_COVERAGE_SUPPORT: + if project.fuzzing_language not in LANGUAGES_WITH_COVERAGE_SUPPORT: logging.info( 'Project "%s" is written in "%s", coverage is not supported yet.', - project_name, project_yaml['language']) + project.name, project.fuzzing_language) return [] - name = project_yaml['name'] - image = project_yaml['image'] - language = project_yaml['language'] - report_date = datetime.datetime.now().strftime('%Y%m%d') - - build_steps = build_lib.project_image_steps(name, image, language) + report_date = build_project.get_datetime_now().strftime('%Y%m%d') + bucket = Bucket(project.name, report_date, PLATFORM, config.testing) - env = CONFIGURATION[:] - out = '/workspace/out/' + SANITIZER - env.append('OUT=' + out) - env.append('FUZZING_LANGUAGE=' + language) + build_steps = build_lib.project_image_steps( + project.name, + project.image, + project.fuzzing_language, + branch=config.branch, + test_image_suffix=config.test_image_suffix) - workdir = build_project.workdir_from_dockerfile(dockerfile_lines) - if not workdir: - workdir = '/src' - - failure_msg = ('*' * 80 + '\nCoverage build failed.\nTo reproduce, run:\n' - 'python infra/helper.py build_image {name}\n' - 'python infra/helper.py build_fuzzers --sanitizer coverage ' - '{name}\n' + '*' * 80).format(name=name) - - # Compilation step. - build_steps.append({ - 'name': - image, - 'env': - env, - 'args': [ - 'bash', - '-c', - # Remove /out to make sure there are non instrumented binaries. - # `cd /src && cd {workdir}` (where {workdir} is parsed from the - # Dockerfile). Container Builder overrides our workdir so we need - # to add this step to set it back. - ('rm -r /out && cd /src && cd {workdir} && mkdir -p {out} && ' - 'compile || (echo "{failure_msg}" && false)' - ).format(workdir=workdir, out=out, failure_msg=failure_msg), - ], - }) - - download_corpora_steps = build_lib.download_corpora_steps(project_name) + build = build_project.Build('libfuzzer', 'coverage', 'x86_64') + env = build_project.get_env(project.fuzzing_language, build) + build_steps.append( + build_project.get_compile_step(project, build, env, config.parallel)) + download_corpora_steps = build_lib.download_corpora_steps( + project.name, testing=config.testing) if not download_corpora_steps: - logging.info('Skipping code coverage build for %s.', project_name) + logging.info('Skipping code coverage build for %s.', project.name) return [] build_steps.extend(download_corpora_steps) failure_msg = ('*' * 80 + '\nCode coverage report generation failed.\n' 'To reproduce, run:\n' - 'python infra/helper.py build_image {name}\n' + f'python infra/helper.py build_image {project.name}\n' 'python infra/helper.py build_fuzzers --sanitizer coverage ' - '{name}\n' - 'python infra/helper.py coverage {name}\n' + - '*' * 80).format(name=name) + f'{project.name}\n' + f'python infra/helper.py coverage {project.name}\n' + '*' * 80) # Unpack the corpus and run coverage script. coverage_env = env + [ 'HTTP_PORT=', - 'COVERAGE_EXTRA_ARGS=%s' % project_yaml['coverage_extra_args'].strip(), + f'COVERAGE_EXTRA_ARGS={project.coverage_extra_args.strip()}', ] - if 'dataflow' in project_yaml['fuzzing_engines']: + if 'dataflow' in project.fuzzing_engines: coverage_env.append('FULL_SUMMARY_PER_TARGET=1') build_steps.append({ - 'name': 'gcr.io/{0}/base-runner'.format(base_images_project), - 'env': coverage_env, + 'name': + build_project.get_runner_image_name(base_images_project, + config.test_image_suffix), + 'env': + coverage_env, 'args': [ 'bash', '-c', ('for f in /corpus/*.zip; do unzip -q $f -d ${f%%.*} || (' @@ -158,9 +144,7 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines, }) # Upload the report. - upload_report_url = UPLOAD_URL_FORMAT.format(project=project_name, - type='reports', - date=report_date) + upload_report_url = bucket.get_upload_url('reports') # Delete the existing report as gsutil cannot overwrite it in a useful way due # to the lack of `-T` option (it creates a subdir in the destination dir). @@ -172,15 +156,14 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines, '-m', 'cp', '-r', - os.path.join(out, 'report'), + os.path.join(build.out, 'report'), upload_report_url, ], }) # Upload the fuzzer stats. Delete the old ones just in case. - upload_fuzzer_stats_url = UPLOAD_URL_FORMAT.format(project=project_name, - type='fuzzer_stats', - date=report_date) + upload_fuzzer_stats_url = bucket.get_upload_url('fuzzer_stats') + build_steps.append(build_lib.gsutil_rm_rf_step(upload_fuzzer_stats_url)) build_steps.append({ 'name': @@ -189,15 +172,13 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines, '-m', 'cp', '-r', - os.path.join(out, 'fuzzer_stats'), + os.path.join(build.out, 'fuzzer_stats'), upload_fuzzer_stats_url, ], }) # Upload the fuzzer logs. Delete the old ones just in case - upload_fuzzer_logs_url = UPLOAD_URL_FORMAT.format(project=project_name, - type='logs', - date=report_date) + upload_fuzzer_logs_url = bucket.get_upload_url('logs') build_steps.append(build_lib.gsutil_rm_rf_step(upload_fuzzer_logs_url)) build_steps.append({ 'name': @@ -206,15 +187,13 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines, '-m', 'cp', '-r', - os.path.join(out, 'logs'), + os.path.join(build.out, 'logs'), upload_fuzzer_logs_url, ], }) # Upload srcmap. - srcmap_upload_url = UPLOAD_URL_FORMAT.format(project=project_name, - type='srcmap', - date=report_date) + srcmap_upload_url = bucket.get_upload_url('srcmap') srcmap_upload_url = srcmap_upload_url.rstrip('/') + '.json' build_steps.append({ 'name': 'gcr.io/cloud-builders/gsutil', @@ -227,15 +206,13 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines, # Update the latest report information file for ClusterFuzz. latest_report_info_url = build_lib.get_signed_url( - LATEST_REPORT_INFO_URL.format(project=project_name), + bucket.latest_report_info_url, content_type=LATEST_REPORT_INFO_CONTENT_TYPE) latest_report_info_body = json.dumps({ 'fuzzer_stats_dir': upload_fuzzer_stats_url, 'html_report_url': - HTML_REPORT_URL_FORMAT.format(project=project_name, - date=report_date, - platform=PLATFORM), + bucket.html_report_url, 'report_date': report_date, 'report_summary_path': @@ -251,25 +228,10 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines, def main(): """Build and run coverage for projects.""" - if len(sys.argv) != 2: - usage() - - image_project = 'oss-fuzz' - base_images_project = 'oss-fuzz-base' - project_dir = sys.argv[1].rstrip(os.path.sep) - project_name = os.path.basename(project_dir) - dockerfile_path = os.path.join(project_dir, 'Dockerfile') - project_yaml_path = os.path.join(project_dir, 'project.yaml') - - with open(dockerfile_path) as docker_file: - dockerfile_lines = docker_file.readlines() - - with open(project_yaml_path) as project_yaml_file: - steps = get_build_steps(project_name, project_yaml_file, dockerfile_lines, - image_project, base_images_project) - - build_project.run_build(steps, project_name, COVERAGE_BUILD_TAG) + return build_project.build_script_main( + 'Generates coverage report for project.', get_build_steps, + COVERAGE_BUILD_TYPE) -if __name__ == "__main__": - main() +if __name__ == '__main__': + sys.exit(main()) |