diff options
| -rwxr-xr-x | infra/ci/build.py | 10 | ||||
| -rw-r--r-- | infra/cifuzz/config_utils.py | 29 | ||||
| -rw-r--r-- | infra/cifuzz/config_utils_test.py | 4 | ||||
| -rw-r--r-- | infra/cifuzz/docker.py | 14 | ||||
| -rw-r--r-- | infra/constants.py | 36 | ||||
| -rwxr-xr-x | infra/helper.py | 64 | ||||
| -rw-r--r-- | infra/helper_test.py | 5 | ||||
| -rwxr-xr-x | infra/presubmit.py | 22 |
8 files changed, 100 insertions, 84 deletions
diff --git a/infra/ci/build.py b/infra/ci/build.py index 8de73636f..5919e445b 100755 --- a/infra/ci/build.py +++ b/infra/ci/build.py @@ -25,15 +25,17 @@ import sys import subprocess import yaml +# pylint: disable=wrong-import-position,import-error +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import constants + CANARY_PROJECT = 'skcms' DEFAULT_ARCHITECTURES = ['x86_64'] DEFAULT_ENGINES = ['afl', 'honggfuzz', 'libfuzzer'] DEFAULT_SANITIZERS = ['address', 'undefined'] -# Languages from project.yaml that have code coverage support. -LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'jvm', 'rust'] - def get_changed_files_output(): """Returns the output of a git command that discovers changed files.""" @@ -112,7 +114,7 @@ def should_build_coverage(project_yaml): return False language = project_yaml.get('language') - if language not in LANGUAGES_WITH_COVERAGE_SUPPORT: + if language not in constants.LANGUAGES_WITH_COVERAGE_SUPPORT: print(('Project is written in "{language}", ' 'coverage is not supported yet.').format(language=language)) return False diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py index d9a8f0c45..cac076212 100644 --- a/infra/cifuzz/config_utils.py +++ b/infra/cifuzz/config_utils.py @@ -24,26 +24,10 @@ import environment # pylint: disable=wrong-import-position,import-error sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import helper +import constants RUN_FUZZERS_MODES = ['batch', 'ci', 'coverage'] - -# TODO(metzman): Make one source of truth for these in helper.py SANITIZERS = ['address', 'memory', 'undefined', 'coverage'] -LANGUAGES = [ - 'c', - 'c++', - 'go', - 'jvm', - 'python', - 'rust', - 'swift', -] - -DEFAULT_ENGINE = 'libfuzzer' -DEFAULT_ARCHITECTURE = 'x86_64' -DEFAULT_LANGUAGE = 'c++' -DEFAULT_SANITIZER = 'address' # This module deals a lot with env variables. Many of these will be set by users # and others beyond CIFuzz's control. Thus, you should be careful about using @@ -60,7 +44,7 @@ def _get_pr_ref(event): def _get_sanitizer(): - return os.getenv('SANITIZER', DEFAULT_SANITIZER).lower() + return os.getenv('SANITIZER', constants.DEFAULT_SANITIZER).lower() def _is_dry_run(): @@ -75,7 +59,7 @@ def _get_language(): # getting it from the project.yaml) is outweighed by the complexity in # implementing this. A lot of the complexity comes from our unittests not # setting a proper projet at this point. - return os.getenv('LANGUAGE', DEFAULT_LANGUAGE) + return os.getenv('LANGUAGE', constants.DEFAULT_LANGUAGE) # pylint: disable=too-few-public-methods,too-many-instance-attributes @@ -211,7 +195,8 @@ class BaseConfig: self.sanitizer = _get_sanitizer() - self.build_integration_path = helper.DEFAULT_RELATIVE_BUILD_INTEGRATION_PATH + self.build_integration_path = ( + constants.DEFAULT_EXTERNAL_BUILD_INTEGRATION_PATH) self.language = _get_language() self.low_disk_space = environment.get_bool('LOW_DISK_SPACE', False) @@ -239,9 +224,9 @@ class BaseConfig: self.sanitizer, SANITIZERS) return False - if self.language not in LANGUAGES: + if self.language not in constants.LANGUAGES: logging.error('Invalid LANGUAGE: %s. Must be one of: %s.', self.language, - LANGUAGES) + constants.LANGUAGES) return False return True diff --git a/infra/cifuzz/config_utils_test.py b/infra/cifuzz/config_utils_test.py index d904fbfbb..a8da59ad8 100644 --- a/infra/cifuzz/config_utils_test.py +++ b/infra/cifuzz/config_utils_test.py @@ -17,6 +17,7 @@ import unittest from unittest import mock import config_utils +import constants import test_helpers # pylint: disable=no-self-use,protected-access @@ -72,8 +73,7 @@ class BaseConfigTest(unittest.TestCase): config = self._create_config() self.assertFalse(config.validate()) mocked_error.assert_called_with('Invalid LANGUAGE: %s. Must be one of: %s.', - os.environ['LANGUAGE'], - config_utils.LANGUAGES) + os.environ['LANGUAGE'], constants.LANGUAGES) @mock.patch('logging.error') def test_validate_invalid_sanitizer(self, mocked_error): diff --git a/infra/cifuzz/docker.py b/infra/cifuzz/docker.py index 15858b458..e20e5ca00 100644 --- a/infra/cifuzz/docker.py +++ b/infra/cifuzz/docker.py @@ -16,10 +16,10 @@ import logging import os import sys -import config_utils # pylint: disable=wrong-import-position,import-error sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import constants import utils BASE_BUILDER_TAG = 'gcr.io/oss-fuzz-base/base-builder' @@ -29,8 +29,8 @@ PROJECT_TAG_PREFIX = 'gcr.io/oss-fuzz/' _DEFAULT_DOCKER_RUN_ARGS = [ '--cap-add', 'SYS_PTRACE', '-e', - 'FUZZING_ENGINE=' + config_utils.DEFAULT_ENGINE, '-e', - 'ARCHITECTURE=' + config_utils.DEFAULT_ARCHITECTURE, '-e', 'CIFUZZ=True' + 'FUZZING_ENGINE=' + constants.DEFAULT_ENGINE, '-e', + 'ARCHITECTURE=' + constants.DEFAULT_ARCHITECTURE, '-e', 'CIFUZZ=True' ] EXTERNAL_PROJECT_IMAGE = 'external-project' @@ -69,8 +69,8 @@ def delete_images(images): def get_base_docker_run_args(workspace, - sanitizer=config_utils.DEFAULT_SANITIZER, - language=config_utils.DEFAULT_LANGUAGE): + sanitizer=constants.DEFAULT_SANITIZER, + language=constants.DEFAULT_LANGUAGE): """Returns arguments that should be passed to every invocation of 'docker run'.""" docker_args = _DEFAULT_DOCKER_RUN_ARGS.copy() @@ -92,8 +92,8 @@ def get_base_docker_run_args(workspace, def get_base_docker_run_command(workspace, - sanitizer=config_utils.DEFAULT_SANITIZER, - language=config_utils.DEFAULT_LANGUAGE): + sanitizer=constants.DEFAULT_SANITIZER, + language=constants.DEFAULT_LANGUAGE): """Returns part of the command that should be used everytime 'docker run' is invoked.""" docker_args, docker_container = get_base_docker_run_args( diff --git a/infra/constants.py b/infra/constants.py new file mode 100644 index 000000000..2fd1836e9 --- /dev/null +++ b/infra/constants.py @@ -0,0 +1,36 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +"""Constants for OSS-Fuzz.""" + +DEFAULT_EXTERNAL_BUILD_INTEGRATION_PATH = '.clusterfuzzlite' + +DEFAULT_LANGUAGE = 'c++' +DEFAULT_SANITIZER = 'address' +DEFAULT_ARCHITECTURE = 'x86_64' +DEFAULT_ENGINE = 'libfuzzer' +LANGUAGES = [ + 'c', + 'c++', + 'go', + 'jvm', + 'python', + 'rust', + 'swift', +] +LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'jvm', 'rust'] +SANITIZERS = ['address', 'none', 'memory', 'undefined', 'dataflow', 'thread'] +ARCHITECTURES = ['i386', 'x86_64'] +ENGINES = ['libfuzzer', 'afl', 'honggfuzz', 'dataflow', 'none'] diff --git a/infra/helper.py b/infra/helper.py index 69e9e2da6..db97698fa 100755 --- a/infra/helper.py +++ b/infra/helper.py @@ -30,6 +30,8 @@ import subprocess import sys import templates +import constants + OSS_FUZZ_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) BUILD_DIR = os.path.join(OSS_FUZZ_DIR, 'build') @@ -46,9 +48,6 @@ BASE_IMAGES = [ VALID_PROJECT_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_-]+$') MAX_PROJECT_NAME_LENGTH = 26 -if sys.version_info[0] >= 3: - raw_input = input # pylint: disable=invalid-name - CORPUS_URL_FORMAT = ( 'gs://{project_name}-corpus.clusterfuzz-external.appspot.com/libFuzzer/' '{fuzz_target}/') @@ -58,12 +57,10 @@ CORPUS_BACKUP_URL_FORMAT = ( PROJECT_LANGUAGE_REGEX = re.compile(r'\s*language\s*:\s*([^\s]+)') -# Languages from project.yaml that have code coverage support. -LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'jvm', 'rust'] - WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)') -DEFAULT_RELATIVE_BUILD_INTEGRATION_PATH = '.clusterfuzzlite' +if sys.version_info[0] >= 3: + raw_input = input # pylint: disable=invalid-name # pylint: disable=too-many-lines @@ -72,10 +69,11 @@ class Project: """Class representing a project that is in OSS-Fuzz or an external project (ClusterFuzzLite user).""" - def __init__(self, - project_name_or_path, - is_external=False, - build_integration_path=DEFAULT_RELATIVE_BUILD_INTEGRATION_PATH): + def __init__( + self, + project_name_or_path, + is_external=False, + build_integration_path=constants.DEFAULT_EXTERNAL_BUILD_INTEGRATION_PATH): self.is_external = is_external if self.is_external: self.name = os.path.basename(os.path.abspath(project_name_or_path)) @@ -97,7 +95,7 @@ class Project: """Returns project language.""" if self.is_external: # TODO(metzman): Handle this properly. - return 'c++' + return constants.DEFAULT_LANGUAGE project_yaml_path = os.path.join(self.path, 'project.yaml') with open(project_yaml_path) as file_handle: @@ -144,7 +142,7 @@ def main(): # pylint: disable=too-many-branches,too-many-return-statements if args.engine == 'dataflow': args.sanitizer = 'dataflow' else: - args.sanitizer = 'address' + args.sanitizer = constants.DEFAULT_SANITIZER if args.command == 'generate': result = generate(args) @@ -260,12 +258,8 @@ def get_parser(): # pylint: disable=too-many-statements check_build_parser = subparsers.add_parser( 'check_build', help='Checks that fuzzers execute without errors.') _add_architecture_args(check_build_parser) - _add_engine_args( - check_build_parser, - choices=['libfuzzer', 'afl', 'honggfuzz', 'dataflow', 'none']) - _add_sanitizer_args( - check_build_parser, - choices=['address', 'memory', 'undefined', 'dataflow', 'thread']) + _add_engine_args(check_build_parser, choices=constants.ENGINES) + _add_sanitizer_args(check_build_parser, choices=constants.SANITIZERS) _add_environment_args(check_build_parser) check_build_parser.add_argument('project', help='name of the project or path (external)') @@ -412,22 +406,28 @@ def _get_out_dir(project=''): return _get_project_build_subdir(project, 'out') -def _add_architecture_args(parser, choices=('x86_64', 'i386')): +def _add_architecture_args(parser, choices=None): """Adds common architecture args.""" - parser.add_argument('--architecture', default='x86_64', choices=choices) + if choices is None: + choices = constants.ARCHITECTURES + parser.add_argument('--architecture', + default=constants.DEFAULT_ARCHITECTURE, + choices=choices) -def _add_engine_args(parser, - choices=('libfuzzer', 'afl', 'honggfuzz', 'dataflow', - 'none')): +def _add_engine_args(parser, choices=None): """Adds common engine args.""" - parser.add_argument('--engine', default='libfuzzer', choices=choices) + if choices is None: + choices = constants.ENGINES + parser.add_argument('--engine', + default=constants.DEFAULT_ENGINE, + choices=choices) -def _add_sanitizer_args(parser, - choices=('address', 'memory', 'undefined', 'coverage', - 'dataflow', 'thread')): +def _add_sanitizer_args(parser, choices=None): """Adds common sanitizer args.""" + if choices is None: + choices = constants.SANITIZERS parser.add_argument( '--sanitizer', default=None, @@ -706,9 +706,9 @@ def check_build(args): fuzzing_language = args.project.language if not fuzzing_language: - logging.warning( - 'Language not specified in project.yaml. Defaulting to C++.') - fuzzing_language = 'c++' + fuzzing_language = constants.DEFAULT_LANGUAGE + logging.warning('Language not specified in project.yaml. Defaulting to %s.', + fuzzing_language) env = [ 'FUZZING_ENGINE=' + args.engine, @@ -845,7 +845,7 @@ def coverage(args): if not check_project_exists(args.project): return False - if args.project.language not in LANGUAGES_WITH_COVERAGE_SUPPORT: + if args.project.language not in constants.LANGUAGES_WITH_COVERAGE_SUPPORT: logging.error( 'Project is written in %s, coverage for it is not supported yet.', args.project.language) diff --git a/infra/helper_test.py b/infra/helper_test.py index 572a76762..6f0bc88cf 100644 --- a/infra/helper_test.py +++ b/infra/helper_test.py @@ -21,6 +21,7 @@ from unittest import mock from pyfakefs import fake_filesystem_unittest +import constants import helper import templates @@ -159,7 +160,7 @@ class ProjectTest(fake_filesystem_unittest.TestCase): self.assertEqual( self.external_project.build_integration_path, os.path.join(self.external_project_path, - helper.DEFAULT_RELATIVE_BUILD_INTEGRATION_PATH)) + constants.DEFAULT_EXTERNAL_BUILD_INTEGRATION_PATH)) def test_init_internal_project(self): """Tests __init__ method for internal projects.""" @@ -180,7 +181,7 @@ class ProjectTest(fake_filesystem_unittest.TestCase): self.assertEqual( self.external_project.dockerfile_path, os.path.join(self.external_project_path, - helper.DEFAULT_RELATIVE_BUILD_INTEGRATION_PATH, + constants.DEFAULT_EXTERNAL_BUILD_INTEGRATION_PATH, 'Dockerfile')) def test_out(self): diff --git a/infra/presubmit.py b/infra/presubmit.py index bd3e25258..8b72b3aec 100755 --- a/infra/presubmit.py +++ b/infra/presubmit.py @@ -23,6 +23,8 @@ import sys import unittest import yaml +import constants + _SRC_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -73,9 +75,9 @@ class ProjectYamlChecker: # Sections in a project.yaml and the constant values that they are allowed # to have. SECTIONS_AND_CONSTANTS = { - 'sanitizers': {'address', 'none', 'memory', 'undefined', 'dataflow'}, - 'architectures': {'i386', 'x86_64'}, - 'fuzzing_engines': {'afl', 'libfuzzer', 'honggfuzz', 'dataflow', 'none'}, + 'sanitizers': constants.SANITIZERS, + 'architectures': constants.ARCHITECTURES, + 'fuzzing_engines': constants.ENGINES, } # Note: this list must be updated when we allow new sections. @@ -100,16 +102,6 @@ class ProjectYamlChecker: 'view_restrictions', ] - SUPPORTED_LANGUAGES = [ - 'c', - 'c++', - 'go', - 'jvm', - 'python', - 'rust', - 'swift', - ] - # Note that some projects like boost only have auto-ccs. However, forgetting # primary contact is probably a mistake. REQUIRED_SECTIONS = ['primary_contact', 'main_repo'] @@ -226,10 +218,10 @@ class ProjectYamlChecker: language = self.data.get('language') if not language: self.error('Missing "language" attribute in project.yaml.') - elif language not in self.SUPPORTED_LANGUAGES: + elif language not in constants.LANGUAGES: self.error( '"language: {language}" is not supported ({supported}).'.format( - language=language, supported=self.SUPPORTED_LANGUAGES)) + language=language, supported=constants.LANGUAGES)) def _check_one_project_yaml(project_yaml_filename): |