diff options
Diffstat (limited to 'infra/build/functions/build_lib.py')
| -rw-r--r-- | infra/build/functions/build_lib.py | 102 |
1 files changed, 85 insertions, 17 deletions
diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index 007579ef9..292ef682f 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -83,11 +83,23 @@ def get_targets_list_url(bucket, project, sanitizer): return url -def _get_targets_list(project_name): +def get_upload_bucket(engine, architecture, testing): + """Returns the upload bucket for |engine| and architecture. Returns the + testing bucket if |testing|.""" + bucket = ENGINE_INFO[engine].upload_bucket + if architecture != 'x86_64': + bucket += '-' + architecture + if testing: + bucket += '-testing' + return bucket + + +def _get_targets_list(project_name, testing): """Returns target list.""" - # libFuzzer ASan is the default configuration, get list of targets from it. - url = get_targets_list_url(ENGINE_INFO['libfuzzer'].upload_bucket, - project_name, 'address') + # libFuzzer ASan 'x86_84' is the default configuration, get list of targets + # from it. + bucket = get_upload_bucket('libfuzzer', 'x86_64', testing) + url = get_targets_list_url(bucket, project_name, 'address') url = urlparse.urljoin(GCS_URL_BASENAME, url) response = requests.get(url) @@ -104,7 +116,7 @@ def _get_targets_list(project_name): def get_signed_url(path, method='PUT', content_type=''): """Returns signed url.""" timestamp = int(time.time() + BUILD_TIMEOUT) - blob = '{0}\n\n{1}\n{2}\n{3}'.format(method, content_type, timestamp, path) + blob = f'{method}\n\n{content_type}\n{timestamp}\n{path}' service_account_path = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') if service_account_path: @@ -119,7 +131,7 @@ def get_signed_url(path, method='PUT', content_type=''): credentials=credentials, cache_discovery=False) client_id = project + '@appspot.gserviceaccount.com' - service_account = 'projects/-/serviceAccounts/{0}'.format(client_id) + service_account = f'projects/-/serviceAccounts/{client_id}' response = iam.projects().serviceAccounts().signBlob( name=service_account, body={ @@ -133,14 +145,13 @@ def get_signed_url(path, method='PUT', content_type=''): 'Expires': timestamp, 'Signature': signature, } - return ('https://storage.googleapis.com{0}?'.format(path) + - urlparse.urlencode(values)) + return f'https://storage.googleapis.com{path}?{urlparse.urlencode(values)}' -def download_corpora_steps(project_name): +def download_corpora_steps(project_name, testing): """Returns GCB steps for downloading corpora backups for the given project. """ - fuzz_targets = _get_targets_list(project_name) + fuzz_targets = _get_targets_list(project_name, testing) if not fuzz_targets: sys.stderr.write('No fuzz targets found for project "%s".\n' % project_name) return None @@ -206,15 +217,72 @@ def gsutil_rm_rf_step(url): return step -def project_image_steps(name, image, language): +def get_pull_test_images_steps(test_image_suffix): + """Returns steps to pull testing versions of base-images and tag them so that + they are used in builds.""" + images = [ + 'gcr.io/oss-fuzz-base/base-builder', + 'gcr.io/oss-fuzz-base/base-builder-swift', + 'gcr.io/oss-fuzz-base/base-builder-jvm', + 'gcr.io/oss-fuzz-base/base-builder-go', + 'gcr.io/oss-fuzz-base/base-builder-python', + 'gcr.io/oss-fuzz-base/base-builder-rust', + ] + steps = [] + for image in images: + test_image = image + '-' + test_image_suffix + steps.append({ + 'name': 'gcr.io/cloud-builders/docker', + 'args': [ + 'pull', + test_image, + ], + 'waitFor': '-' # Start this immediately, don't wait for previous step. + }) + + # This step is hacky but gives us great flexibility. OSS-Fuzz has hardcoded + # references to gcr.io/oss-fuzz-base/base-builder (in dockerfiles, for + # example) and gcr.io/oss-fuzz-base-runner (in this build code). But the + # testing versions of those images are called e.g. + # gcr.io/oss-fuzz-base/base-builder-testing and + # gcr.io/oss-fuzz-base/base-runner-testing. How can we get the build to use + # the testing images instead of the real ones? By doing this step: tagging + # the test image with the non-test version, so that the test version is used + # instead of pulling the real one. + steps.append({ + 'name': 'gcr.io/cloud-builders/docker', + 'args': ['tag', test_image, image], + }) + return steps + + +def get_srcmap_step_id(): + """Returns the id for the srcmap step.""" + return 'srcmap' + + +def project_image_steps(name, + image, + language, + branch=None, + test_image_suffix=None): """Returns GCB steps to build OSS-Fuzz project image.""" - steps = [{ + clone_step = { 'args': [ - 'clone', - 'https://github.com/google/oss-fuzz.git', + 'clone', 'https://github.com/google/oss-fuzz.git', '--depth', '1' ], 'name': 'gcr.io/cloud-builders/git', - }, { + } + if branch: + # Do this to support testing other branches. + clone_step['args'].extend(['--branch', branch]) + + steps = [clone_step] + if test_image_suffix: + steps.extend(get_pull_test_images_steps(test_image_suffix)) + + srcmap_step_id = get_srcmap_step_id() + steps += [{ 'name': 'gcr.io/cloud-builders/docker', 'args': [ 'build', @@ -224,8 +292,7 @@ def project_image_steps(name, image, language): ], 'dir': 'oss-fuzz/projects/' + name, }, { - 'name': - image, + 'name': image, 'args': [ 'bash', '-c', 'srcmap > /workspace/srcmap.json && cat /workspace/srcmap.json' @@ -234,6 +301,7 @@ def project_image_steps(name, image, language): 'OSSFUZZ_REVISION=$REVISION_ID', 'FUZZING_LANGUAGE=%s' % language, ], + 'id': srcmap_step_id }] return steps |