Snap for 6176706 from fc25fcf0519e5da3d5acfea27eadaa9ccffaef55 to rvc-releaseandroid-vts-11.0_r9android-vts-11.0_r8android-vts-11.0_r7android-vts-11.0_r6android-vts-11.0_r5android-vts-11.0_r4android-vts-11.0_r3android-vts-11.0_r2android-vts-11.0_r16android-vts-11.0_r15android-vts-11.0_r14android-vts-11.0_r13android-vts-11.0_r12android-vts-11.0_r11android-vts-11.0_r10android-vts-11.0_r1android-security-11.0.0_r76android-security-11.0.0_r75android-security-11.0.0_r74android-security-11.0.0_r73android-security-11.0.0_r72android-security-11.0.0_r71android-security-11.0.0_r70android-security-11.0.0_r69android-security-11.0.0_r68android-security-11.0.0_r67android-security-11.0.0_r66android-security-11.0.0_r65android-security-11.0.0_r64android-security-11.0.0_r63android-security-11.0.0_r62android-security-11.0.0_r61android-security-11.0.0_r60android-security-11.0.0_r59android-security-11.0.0_r58android-security-11.0.0_r57android-security-11.0.0_r56android-security-11.0.0_r55android-security-11.0.0_r54android-security-11.0.0_r53android-security-11.0.0_r52android-security-11.0.0_r51android-security-11.0.0_r50android-security-11.0.0_r49android-security-11.0.0_r1android-platform-11.0.0_r9android-platform-11.0.0_r8android-platform-11.0.0_r7android-platform-11.0.0_r6android-platform-11.0.0_r5android-platform-11.0.0_r40android-platform-11.0.0_r4android-platform-11.0.0_r39android-platform-11.0.0_r38android-platform-11.0.0_r37android-platform-11.0.0_r36android-platform-11.0.0_r35android-platform-11.0.0_r34android-platform-11.0.0_r33android-platform-11.0.0_r32android-platform-11.0.0_r31android-platform-11.0.0_r30android-platform-11.0.0_r3android-platform-11.0.0_r29android-platform-11.0.0_r28android-platform-11.0.0_r27android-platform-11.0.0_r26android-platform-11.0.0_r25android-platform-11.0.0_r24android-platform-11.0.0_r23android-platform-11.0.0_r22android-platform-11.0.0_r21android-platform-11.0.0_r20android-platform-11.0.0_r2android-platform-11.0.0_r19android-platform-11.0.0_r18android-platform-11.0.0_r17android-platform-11.0.0_r16android-platform-11.0.0_r15android-platform-11.0.0_r14android-platform-11.0.0_r13android-platform-11.0.0_r12android-platform-11.0.0_r11android-platform-11.0.0_r10android-platform-11.0.0_r1android-cts-11.0_r9android-cts-11.0_r8android-cts-11.0_r7android-cts-11.0_r6android-cts-11.0_r5android-cts-11.0_r4android-cts-11.0_r3android-cts-11.0_r2android-cts-11.0_r16android-cts-11.0_r15android-cts-11.0_r14android-cts-11.0_r13android-cts-11.0_r12android-cts-11.0_r11android-cts-11.0_r10android-cts-11.0_r1android-11.0.0_r6android-11.0.0_r5android-11.0.0_r4android-11.0.0_r3android-11.0.0_r25android-11.0.0_r2android-11.0.0_r17android-11.0.0_r1android11-tests-releaseandroid11-security-releaseandroid11-s1-releaseandroid11-releaseandroid11-platform-releaseandroid11-gsi

Change-Id: If51f7e2f73df599e130e06b179c360fd2941ec8a

1 files changed, 156 insertions, 172 deletions

diff --git a/infra/gcb/build_and_run_coverage.py b/infra/gcb/build_and_run_coverage.py
index 7c6ea5b04..b94fe3558 100644
--- a/infra/gcb/build_and_run_coverage.py
+++ b/infra/gcb/build_and_run_coverage.py
@@ -11,41 +11,31 @@ import requests
 import sys
 import urlparse
 
+import build_lib
 import build_project
 
 SANITIZER = 'coverage'
 CONFIGURATION = ['FUZZING_ENGINE=libfuzzer', 'SANITIZER=%s' % SANITIZER]
 PLATFORM = 'linux'
 
-# Where corpus backups can be downloaded from.
-CORPUS_BACKUP_URL = ('/{project}-backup.clusterfuzz-external.appspot.com/'
-                     'corpus/libFuzzer/{fuzzer}/latest.zip')
-
-# Cloud Builder has a limit of 100 build steps and 100 arguments for each step.
-CORPUS_DOWNLOAD_BATCH_SIZE = 100
-
 COVERAGE_BUILD_TAG = 'coverage'
 
-# Needed for reading public target.list.* files.
-GCS_URL_BASENAME = 'https://storage.googleapis.com/'
-
 # Where code coverage reports need to be uploaded to.
 COVERAGE_BUCKET_NAME = 'oss-fuzz-coverage'
 
 # Link to the code coverage report in HTML format.
-HTML_REPORT_URL_FORMAT = (
-    GCS_URL_BASENAME + COVERAGE_BUCKET_NAME +
-    '/{project}/reports/{date}/{platform}/index.html')
+HTML_REPORT_URL_FORMAT = (build_lib.GCS_URL_BASENAME + COVERAGE_BUCKET_NAME +
+                          '/{project}/reports/{date}/{platform}/index.html')
 
 # This is needed for ClusterFuzz to pick up the most recent reports data.
-LATEST_REPORT_INFO_URL = (
-    '/' + COVERAGE_BUCKET_NAME + '/latest_report_info/{project}.json')
+LATEST_REPORT_INFO_URL = ('/' + COVERAGE_BUCKET_NAME +
+                          '/latest_report_info/{project}.json')
 
 # Link where to upload code coverage report files to.
 UPLOAD_URL_FORMAT = 'gs://' + COVERAGE_BUCKET_NAME + '/{project}/{type}/{date}'
 
-# TODO(#2817): Support code coverage for Go projects.
-GO_FUZZ_BUILD = 'go-fuzz-build -libfuzzer'
+# Languages from project.yaml that have code coverage support.
+LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'cpp']
 
 
 def skip_build(message):
@@ -58,8 +48,7 @@ def skip_build(message):
 
 
 def usage():
-  sys.stderr.write(
-    "Usage: " + sys.argv[0] + " <project_dir>\n")
+  sys.stderr.write("Usage: " + sys.argv[0] + " <project_dir>\n")
   exit(1)
 
 
@@ -70,14 +59,13 @@ def get_build_steps(project_dir):
     skip_build('Project "%s" is disabled.' % project_name)
 
   build_script_path = os.path.join(project_dir, 'build.sh')
-  with open(build_script_path) as fh:
-    if GO_FUZZ_BUILD in fh.read():
-      skip_build('Project "%s" uses go-fuzz, coverage is not supported yet.' %
-                 project_name)
-
-  fuzz_targets = get_targets_list(project_name)
-  if not fuzz_targets:
-    skip_build('No fuzz targets found for project "%s".' % project_name)
+  if os.path.exists(build_script_path):
+    with open(build_script_path) as fh:
+      if project_yaml['language'] not in LANGUAGES_WITH_COVERAGE_SUPPORT:
+        skip_build(('Project "{project_name}" is written in "{language}", '
+                    'coverage is not supported yet.').format(
+                        project_name=project_name,
+                        language=project_yaml['language']))
 
   dockerfile_path = os.path.join(project_dir, 'Dockerfile')
   name = project_yaml['name']
@@ -87,7 +75,8 @@ def get_build_steps(project_dir):
   build_steps = [
       {
           'args': [
-              'clone', 'https://github.com/google/oss-fuzz.git',
+              'clone',
+              'https://github.com/google/oss-fuzz.git',
           ],
           'name': 'gcr.io/cloud-builders/git',
       },
@@ -102,8 +91,7 @@ def get_build_steps(project_dir):
           'dir': 'oss-fuzz/projects/' + name,
       },
       {
-          'name':
-              image,
+          'name': image,
           'args': [
               'bash', '-c',
               'srcmap > /workspace/srcmap.json && cat /workspace/srcmap.json'
@@ -120,170 +108,166 @@ def get_build_steps(project_dir):
   if not workdir:
     workdir = '/src'
 
+  failure_msg = ('*' * 80 + '\nCoverage build failed.\nTo reproduce, run:\n'
+                 'python infra/helper.py build_image {name}\n'
+                 'python infra/helper.py build_fuzzers --sanitizer coverage '
+                 '{name}\n' + '*' * 80).format(name=name)
+
   # Compilation step.
-  build_steps.append(
-      {
-          'name': image,
-          'env': env,
-          'args': [
-              'bash',
-              '-c',
-               # Remove /out to make sure there are non instrumented binaries.
-               # `cd /src && cd {workdir}` (where {workdir} is parsed from the
-               # Dockerfile). Container Builder overrides our workdir so we need
-               # to add this step to set it back.
-               'rm -r /out && cd /src && cd {1} && mkdir -p {0} && compile'.format(out, workdir),
-          ],
-      }
-  )
-
-  # Split fuzz targets into batches of CORPUS_DOWNLOAD_BATCH_SIZE.
-  for i in xrange(0,  len(fuzz_targets), CORPUS_DOWNLOAD_BATCH_SIZE):
-    download_corpus_args = []
-    for binary_name in fuzz_targets[i : i+CORPUS_DOWNLOAD_BATCH_SIZE]:
-      qualified_name = binary_name
-      qualified_name_prefix = '%s_' % project_name
-      if not binary_name.startswith(qualified_name_prefix):
-        qualified_name = qualified_name_prefix + binary_name
-
-      url = build_project.get_signed_url(
-          CORPUS_BACKUP_URL.format(
-              project=project_name, fuzzer=qualified_name),
-          method='GET')
-
-      corpus_archive_path = os.path.join('/corpus', binary_name + '.zip')
-      download_corpus_args.append('%s %s' % (corpus_archive_path, url))
-
-    # Download corpus.
-    build_steps.append(
-        {
-            'name': 'gcr.io/oss-fuzz-base/base-runner',
-            'entrypoint': 'download_corpus',
-            'args': download_corpus_args,
-            'volumes': [{'name': 'corpus', 'path': '/corpus'}],
-        }
-    )
+  build_steps.append({
+      'name':
+          image,
+      'env':
+          env,
+      'args': [
+          'bash',
+          '-c',
+          # Remove /out to make sure there are non instrumented binaries.
+          # `cd /src && cd {workdir}` (where {workdir} is parsed from the
+          # Dockerfile). Container Builder overrides our workdir so we need
+          # to add this step to set it back.
+          ('rm -r /out && cd /src && cd {workdir} && mkdir -p {out} && '
+           'compile || (echo "{failure_msg}" && false)'
+          ).format(workdir=workdir, out=out, failure_msg=failure_msg),
+      ],
+  })
+
+  download_corpora_step = build_lib.download_corpora_step(project_name)
+  if not download_corpora_step:
+    skip_build("Skipping code coverage build for %s.\n" % project_name)
+
+  build_steps.append(download_corpora_step)
+
+  failure_msg = ('*' * 80 + '\nCode coverage report generation failed.\n'
+                 'To reproduce, run:\n'
+                 'python infra/helper.py build_image {name}\n'
+                 'python infra/helper.py build_fuzzers --sanitizer coverage '
+                 '{name}\n'
+                 'python infra/helper.py coverage {name}\n' +
+                 '*' * 80).format(name=name)
 
   # Unpack the corpus and run coverage script.
-  build_steps.append(
-      {
-          'name': 'gcr.io/oss-fuzz-base/base-runner',
-          'env': env + [
+  build_steps.append({
+      'name':
+          'gcr.io/oss-fuzz-base/base-runner',
+      'env':
+          env + [
               'HTTP_PORT=',
-              'COVERAGE_EXTRA_ARGS=%s' % project_yaml['coverage_extra_args'].strip()
-          ],
-          'args': [
-              'bash',
-              '-c',
-              'for f in /corpus/*.zip; do unzip -q $f -d ${f%%.*}; done && coverage',
+              'COVERAGE_EXTRA_ARGS=%s' %
+              project_yaml['coverage_extra_args'].strip()
           ],
-          'volumes': [{'name': 'corpus', 'path': '/corpus'}],
-      }
-  )
+      'args': [
+          'bash', '-c',
+          ('for f in /corpus/*.zip; do unzip -q $f -d ${f%%.*} || ('
+           'echo "Failed to unpack the corpus for $(basename ${f%%.*}). '
+           'This usually means that corpus backup for a particular fuzz '
+           'target does not exist. If a fuzz target was added in the last '
+           '24 hours, please wait one more day. Otherwise, something is '
+           'wrong with the fuzz target or the infrastructure, and corpus '
+           'pruning task does not finish successfully." && exit 1'
+           '); done && coverage || (echo "' + failure_msg + '" && false)')
+      ],
+      'volumes': [{
+          'name': 'corpus',
+          'path': '/corpus'
+      }],
+  })
 
   # Upload the report.
-  upload_report_url = UPLOAD_URL_FORMAT.format(
-      project=project_name, type='reports', date=report_date)
-  build_steps.append(
-      {
-          'name': 'gcr.io/cloud-builders/gsutil',
-          'args': [
-              '-m', 'cp', '-r',
-              os.path.join(out, 'report'),
-              upload_report_url,
-          ],
-      }
-  )
+  upload_report_url = UPLOAD_URL_FORMAT.format(project=project_name,
+                                               type='reports',
+                                               date=report_date)
+  build_steps.append({
+      'name':
+          'gcr.io/cloud-builders/gsutil',
+      'args': [
+          '-m',
+          'cp',
+          '-r',
+          os.path.join(out, 'report'),
+          upload_report_url,
+      ],
+  })
 
   # Upload the fuzzer stats.
-  upload_fuzzer_stats_url = UPLOAD_URL_FORMAT.format(
-      project=project_name, type='fuzzer_stats', date=report_date)
-  build_steps.append(
-      {
-          'name': 'gcr.io/cloud-builders/gsutil',
-          'args': [
-              '-m', 'cp', '-r',
-              os.path.join(out, 'fuzzer_stats'),
-              upload_fuzzer_stats_url,
-          ],
-      }
-  )
+  upload_fuzzer_stats_url = UPLOAD_URL_FORMAT.format(project=project_name,
+                                                     type='fuzzer_stats',
+                                                     date=report_date)
+  build_steps.append({
+      'name':
+          'gcr.io/cloud-builders/gsutil',
+      'args': [
+          '-m',
+          'cp',
+          '-r',
+          os.path.join(out, 'fuzzer_stats'),
+          upload_fuzzer_stats_url,
+      ],
+  })
 
   # Upload the fuzzer logs.
-  build_steps.append(
-      {
-          'name': 'gcr.io/cloud-builders/gsutil',
-          'args': [
-              '-m', 'cp', '-r',
-              os.path.join(out, 'logs'),
-              UPLOAD_URL_FORMAT.format(
-                  project=project_name, type='logs', date=report_date),
-          ],
-      }
-  )
+  build_steps.append({
+      'name':
+          'gcr.io/cloud-builders/gsutil',
+      'args': [
+          '-m',
+          'cp',
+          '-r',
+          os.path.join(out, 'logs'),
+          UPLOAD_URL_FORMAT.format(project=project_name,
+                                   type='logs',
+                                   date=report_date),
+      ],
+  })
 
   # Upload srcmap.
-  srcmap_upload_url = UPLOAD_URL_FORMAT.format(
-      project=project_name, type='srcmap', date=report_date)
+  srcmap_upload_url = UPLOAD_URL_FORMAT.format(project=project_name,
+                                               type='srcmap',
+                                               date=report_date)
   srcmap_upload_url = srcmap_upload_url.rstrip('/') + '.json'
-  build_steps.append(
-      {
-          'name': 'gcr.io/cloud-builders/gsutil',
-          'args': [
-              'cp',
-              '/workspace/srcmap.json',
-              srcmap_upload_url,
-          ],
-      }
-  )
+  build_steps.append({
+      'name': 'gcr.io/cloud-builders/gsutil',
+      'args': [
+          'cp',
+          '/workspace/srcmap.json',
+          srcmap_upload_url,
+      ],
+  })
 
   # Update the latest report information file for ClusterFuzz.
-  latest_report_info_url = build_project.get_signed_url(
+  latest_report_info_url = build_lib.get_signed_url(
       LATEST_REPORT_INFO_URL.format(project=project_name),
       method='PUT',
       content_type='application/json')
-  latest_report_info_body = json.dumps(
-      {
-          'fuzzer_stats_dir': upload_fuzzer_stats_url,
-          'html_report_url': HTML_REPORT_URL_FORMAT.format(
-              project=project_name, date=report_date, platform=PLATFORM),
-          'report_date': report_date,
-          'report_summary_path': os.path.join(
-              upload_report_url, PLATFORM, 'summary.json'),
-      }
-  )
-
-  build_steps.append(
-      {
-          'name': 'gcr.io/cloud-builders/curl',
-          'args': [
-              '-H', 'Content-Type: application/json',
-              '-X', 'PUT',
-              '-d', latest_report_info_body,
-              latest_report_info_url,
-          ],
-      }
-  )
+  latest_report_info_body = json.dumps({
+      'fuzzer_stats_dir':
+          upload_fuzzer_stats_url,
+      'html_report_url':
+          HTML_REPORT_URL_FORMAT.format(project=project_name,
+                                        date=report_date,
+                                        platform=PLATFORM),
+      'report_date':
+          report_date,
+      'report_summary_path':
+          os.path.join(upload_report_url, PLATFORM, 'summary.json'),
+  })
+
+  build_steps.append({
+      'name':
+          'gcr.io/cloud-builders/curl',
+      'args': [
+          '-H',
+          'Content-Type: application/json',
+          '-X',
+          'PUT',
+          '-d',
+          latest_report_info_body,
+          latest_report_info_url,
+      ],
+  })
   return build_steps
 
 
-def get_targets_list(project_name):
-  # libFuzzer ASan is the default configuration, get list of targets from it.
-  url = build_project.get_targets_list_url(
-      build_project.ENGINE_INFO['libfuzzer'].upload_bucket,
-      project_name,
-      'address')
-
-  url = urlparse.urljoin(GCS_URL_BASENAME , url)
-  r = requests.get(url)
-  if not r.status_code == 200:
-    sys.stderr.write('Failed to get list of targets from "%s".\n' % url)
-    sys.stderr.write('Status code: %d \t\tText:\n%s\n' % (r.status_code, r.text))
-    return None
-
-  return r.text.split()
-
-
 def main():
   if len(sys.argv) != 2:
     usage()