aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLeo Neat <leosneat@gmail.com>2020-01-15 13:30:57 -0800
committerjonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com>2020-01-15 13:30:57 -0800
commit14582175d07c8a66adf3f9087a464612f2a3f630 (patch)
tree831d57d5a406af6be2dcb3bd0ca7ba1970956cea
parent79860344beaf5b228c0f619451d860ec0f2adcfa (diff)
downloadoss-fuzz-14582175d07c8a66adf3f9087a464612f2a3f630.tar.gz
[infra] Scripts for building fuzzers with CIFuzz (#3207)
-rw-r--r--infra/base-images/base-builder/detect_repo.py81
-rw-r--r--infra/base-images/base-builder/detect_repo_test.py64
-rw-r--r--infra/bisector.py118
-rw-r--r--infra/bisector_test.py49
-rw-r--r--infra/build_specified_commit.py110
-rw-r--r--infra/build_specified_commit_test.py74
-rw-r--r--infra/cifuzz.py88
-rwxr-xr-xinfra/helper.py22
8 files changed, 412 insertions, 194 deletions
diff --git a/infra/base-images/base-builder/detect_repo.py b/infra/base-images/base-builder/detect_repo.py
index d7974ca5e..f57947e7e 100644
--- a/infra/base-images/base-builder/detect_repo.py
+++ b/infra/base-images/base-builder/detect_repo.py
@@ -16,9 +16,11 @@ inside of an OSS-Fuzz project.
Example Usage:
- python detect_repo.py --src_dir /src --example_commit b534f03eecd8a109db2b085ab24d419b6486de97
+ python detect_repo.py --src_dir /src --example_commit
+ b534f03eecd8a109db2b085ab24d419b6486de97
-Prints the location of the git remote repo as well as the repos name seperated by a space.
+Prints the location of the git remote repo as well as the repo's name
+seperated by a space.
https://github.com/VirusTotal/yara.git yara
@@ -29,26 +31,36 @@ import subprocess
def main():
- """Function to get a git repos information based on its commit."""
+ """Function to get a git repo's url and name referenced by OSS-Fuzz
+ Dockerfile.
+
+ Raises:
+ ValueError when a commit or a ref is not provided.
+ """
parser = argparse.ArgumentParser(
- description='Finds a specific git repo in an oss-fuzz projects docker file.'
- )
+ description=
+ 'Finds a specific git repo in an oss-fuzz project\'s docker file.')
parser.add_argument(
'--src_dir',
- help='The location of the oss-fuzz projects source directory',
- required=True)
- parser.add_argument(
- '--example_commit',
- help='A commit SHA refrencing the projects main repo',
+ help='The location of an oss-fuzz project\'s source directory.',
required=True)
+ parser.add_argument('--repo_name', help='The name of the git repo.')
+ parser.add_argument('--example_commit',
+ help='A commit SHA referencing the project\'s main repo.')
+
args = parser.parse_args()
+ if not args.repo_name and not args.example_commit:
+ raise ValueError(
+ 'Requires an example commit or a repo name to find repo location.')
for single_dir in os.listdir(args.src_dir):
full_path = os.path.join(args.src_dir, single_dir)
if not os.path.isdir(full_path):
continue
- if check_for_commit(
- os.path.join(args.src_dir, full_path), args.example_commit):
- print('Detected repo: %s %s' % (get_repo(full_path), single_dir.rstrip()))
+ if args.example_commit and check_for_commit(full_path, args.example_commit):
+ print('Detected repo:', get_repo(full_path), single_dir)
+ return
+ if args.repo_name and check_for_repo_name(full_path, args.repo_name):
+ print('Detected repo:', get_repo(full_path), single_dir)
return
print('No git repos with specific commit: %s found in %s' %
(args.example_commit, args.src_dir))
@@ -61,26 +73,41 @@ def get_repo(repo_path):
repo_path: The directory on the image where the git repo exists.
Returns:
- The repo location or None
+ The repo location or None.
"""
- output, return_code = execute(
- ['git', 'config', '--get', 'remote.origin.url'],
- location=repo_path,
- check_result=True)
+ output, return_code = execute(['git', 'config', '--get', 'remote.origin.url'],
+ location=repo_path,
+ check_result=True)
if return_code == 0 and output:
return output.rstrip()
return None
+def check_for_repo_name(repo_path, repo_name):
+ """Check to see if the repo_name matches the remote repository repo name.
+
+ Args:
+ repo_path: The directory of the git repo.
+ repo_name: The name of the target git repo.
+ """
+ if not os.path.exists(os.path.join(repo_path, '.git')):
+ return False
+
+ out, _ = execute(['git', 'config', '--get', 'remote.origin.url'],
+ location=repo_path)
+ out = out.split('/')[-1].replace('.git', '').rstrip()
+ return out == repo_name
+
+
def check_for_commit(repo_path, commit):
"""Checks a directory for a specific commit.
Args:
- repo_path: The name of the directory to test for the commit
- commit: The commit SHA to check for
+ repo_path: The name of the directory to test for the commit.
+ commit: The commit SHA to check for.
Returns:
- True if directory contains that commit
+ True if directory contains that commit.
"""
# Check if valid git repo.
@@ -93,7 +120,7 @@ def check_for_commit(repo_path, commit):
# Check if commit is in history.
_, return_code = execute(['git', 'cat-file', '-e', commit],
- location=repo_path)
+ location=repo_path)
return return_code == 0
@@ -101,15 +128,15 @@ def execute(command, location, check_result=False):
"""Runs a shell command in the specified directory location.
Args:
- command: The command as a list to be run
- location: The directory the command is run in
- check_result: Should an exception be thrown on failed command
+ command: The command as a list to be run.
+ location: The directory the command is run in.
+ check_result: Should an exception be thrown on failed command.
Returns:
- The stdout of the command, the error code
+ The stdout of the command, the error code.
Raises:
- RuntimeError: running a command resulted in an error
+ RuntimeError: running a command resulted in an error.
"""
process = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=location)
output, err = process.communicate()
diff --git a/infra/base-images/base-builder/detect_repo_test.py b/infra/base-images/base-builder/detect_repo_test.py
index cac0f8821..e9029b7ff 100644
--- a/infra/base-images/base-builder/detect_repo_test.py
+++ b/infra/base-images/base-builder/detect_repo_test.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test the functionality of the detect_repo module.
-The will consist of the following functional test
+This will consist of the following functional test:
1. Determine if a OSS-Fuzz projects main repo can be accurately deduce
from example commits.
"""
@@ -23,11 +23,14 @@ import tempfile
import unittest
import detect_repo
+
# Appending to path for access to repo_manager module.
+# pylint: disable=wrong-import-position
sys.path.append(
- os.path.dirname(
- os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
+ os.path.dirname(os.path.dirname(os.path.dirname(
+ os.path.abspath(__file__)))))
import repo_manager
+# pylint: enable=wrong-import-position
class DetectRepoTest(unittest.TestCase):
@@ -62,25 +65,66 @@ class DetectRepoTest(unittest.TestCase):
self.check_commit_with_repo(None, None,
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', tmp_dir)
- def check_commit_with_repo(self, repo_origin, repo_name, commit, tmp_dir):
+ def test_infer_main_repo_from_name(self):
+ """Tests that the main project repo can be inferred from a repo name."""
+
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ # Construct example repos to check for name.
+ repo_manager.RepoManager('https://github.com/curl/curl.git', tmp_dir)
+ repo_manager.RepoManager('https://github.com/ntop/nDPI.git', tmp_dir)
+ repo_manager.RepoManager('https://github.com/libarchive/libarchive.git',
+ tmp_dir)
+ self.check_ref_with_repo('https://github.com/curl/curl.git', 'curl',
+ tmp_dir)
+ self.check_ref_with_repo('https://github.com/ntop/nDPI.git', 'nDPI',
+ tmp_dir)
+ self.check_ref_with_repo('https://github.com/libarchive/libarchive.git',
+ 'libarchive', tmp_dir)
+
+ def check_ref_with_repo(self, repo_origin, repo_name, tmp_dir):
"""Checks the detect repo's main method for a specific set of inputs.
+ Args:
+ repo_origin: URL of the git repo.
+ repo_name: The name of the directory it is cloned to.
+ tmp_dir: The location of the directory of git repos to be searched.
+ """
+ command = [
+ 'python3', 'detect_repo.py', '--src_dir', tmp_dir, '--repo_name',
+ repo_name
+ ]
+ out, _ = detect_repo.execute(command,
+ location=os.path.dirname(
+ os.path.realpath(__file__)))
+ match = re.search(r'\bDetected repo: ([^ ]+) ([^ ]+)', out.rstrip())
+ if match and match.group(1) and match.group(2):
+ self.assertEqual(match.group(1), repo_origin)
+ else:
+ self.assertIsNone(repo_origin)
+
+ def check_commit_with_repo(self, repo_origin, repo_name, commit, tmp_dir):
+ """Checks the detect repos main method for a specific set of inputs.
+
Args:
- repo_origin: The location of where the git repo is stored
- repo_name: The name of the directory it is cloned to
- commit: The commit that should be used to look up the repo
- tmp_dir: The location of the directory of git repos to be searched
+ repo_origin: URL of the git repo.
+ repo_name: The name of the directory it is cloned to.
+ commit: The commit that should be used to look up the repo.
+ tmp_dir: The location of the directory of git repos to be searched.
"""
command = [
'python3', 'detect_repo.py', '--src_dir', tmp_dir, '--example_commit',
commit
]
- out, _ = detect_repo.execute(
- command, location=os.path.dirname(os.path.realpath(__file__)))
+ out, _ = detect_repo.execute(command,
+ location=os.path.dirname(
+ os.path.abspath(__file__)))
match = re.search(r'\bDetected repo: ([^ ]+) ([^ ]+)', out.rstrip())
if match and match.group(1) and match.group(2):
self.assertEqual(match.group(1), repo_origin)
self.assertEqual(match.group(2), repo_name)
+ else:
+ self.assertIsNone(repo_origin)
+ self.assertIsNone(repo_name)
if __name__ == '__main__':
diff --git a/infra/bisector.py b/infra/bisector.py
index afb5a4cef..557b92beb 100644
--- a/infra/bisector.py
+++ b/infra/bisector.py
@@ -18,7 +18,7 @@ where the bug was introduced. It also looks for where the bug was fixed.
This is done with the following steps:
- NOTE: NEEDS TO BE RUN FROM THE OSS-Fuzz HOME directory
+ NOTE: Needs to be run from root of the OSS-Fuzz source checkout.
Typical usage example:
python3 infra/bisector.py
@@ -31,7 +31,6 @@ This is done with the following steps:
"""
import argparse
-from dataclasses import dataclass
import os
import tempfile
@@ -40,22 +39,6 @@ import helper
import repo_manager
-@dataclass
-class BuildData():
- """List of data requried for bisection of errors in OSS-Fuzz projects.
-
- Attributes:
- project_name: The name of the OSS-Fuzz project that is being checked
- engine: The fuzzing engine to be used
- sanitizer: The sanitizer to be used
- architecture: CPU architecture to build the fuzzer for
- """
- project_name: str
- engine: str
- sanitizer: str
- architecture: str
-
-
def main():
"""Finds the commit SHA where an error was initally introduced."""
oss_fuzz_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
@@ -65,29 +48,34 @@ def main():
parser = argparse.ArgumentParser(
description='git bisection for finding introduction of bugs')
- parser.add_argument(
- '--project_name',
- help='The name of the project where the bug occured',
- required=True)
- parser.add_argument(
- '--commit_new',
- help='The newest commit SHA to be bisected',
- required=True)
- parser.add_argument(
- '--commit_old',
- help='The oldest commit SHA to be bisected',
- required=True)
- parser.add_argument(
- '--fuzz_target', help='the name of the fuzzer to be built', required=True)
- parser.add_argument('--testcase', help='path to test case', required=True)
- parser.add_argument(
- '--engine', help='the default is "libfuzzer"', default='libfuzzer')
- parser.add_argument(
- '--sanitizer', default='address', help='the default is "address"')
+ parser.add_argument('--project_name',
+ help='The name of the project where the bug occurred.',
+ required=True)
+ parser.add_argument('--commit_new',
+ help='The newest commit SHA to be bisected.',
+ required=True)
+ parser.add_argument('--commit_old',
+ help='The oldest commit SHA to be bisected.',
+ required=True)
+ parser.add_argument('--fuzz_target',
+ help='The name of the fuzzer to be built.',
+ required=True)
+ parser.add_argument('--testcase',
+ help='The path to test case.',
+ required=True)
+ parser.add_argument('--engine',
+ help='The default is "libfuzzer".',
+ default='libfuzzer')
+ parser.add_argument('--sanitizer',
+ default='address',
+ help='The default is "address".')
parser.add_argument('--architecture', default='x86_64')
args = parser.parse_args()
- build_data = BuildData(args.project_name, args.engine, args.sanitizer,
- args.architecture)
+ build_data = build_specified_commit.BuildData()
+ build_data.project_name = args.project_name
+ build_data.engine = args.engine
+ build_data.sanitizer = args.sanitizer
+ build_data.architecture = args.architecture
error_sha = bisect(args.commit_old, args.commit_new, args.testcase,
args.fuzz_target, build_data)
if not error_sha:
@@ -107,54 +95,52 @@ def bisect(commit_old, commit_new, testcase, fuzz_target, build_data):
specific error from a fuzz testcase.
Args:
- commit_old: The oldest commit in the error regression range
- commit_new: The newest commit in the error regression range
+ commit_old: The oldest commit in the error regression range.
+ commit_new: The newest commit in the error regression range.
testcase: The file path of the test case that triggers the error
- fuzz_target: The name of the fuzzer to be tested
- build_data: a class holding all of the input parameters for bisection
+ fuzz_target: The name of the fuzzer to be tested.
+ build_data: a class holding all of the input parameters for bisection.
Returns:
- The commit SHA that introduced the error or None
+ The commit SHA that introduced the error or None.
Raises:
- ValueError: when a repo url can't be determine from the project
+ ValueError: when a repo url can't be determine from the project.
"""
with tempfile.TemporaryDirectory() as tmp_dir:
- repo_url, repo_name = build_specified_commit.detect_main_repo_from_docker(
- build_data.project_name, commit_old)
+ repo_url, repo_name = build_specified_commit.detect_main_repo(
+ build_data.project_name, commit=commit_old)
if not repo_url or not repo_name:
raise ValueError('Main git repo can not be determined.')
- bisect_repo_manager = repo_manager.RepoManager(
- repo_url, tmp_dir, repo_name=repo_name)
+ bisect_repo_manager = repo_manager.RepoManager(repo_url,
+ tmp_dir,
+ repo_name=repo_name)
commit_list = bisect_repo_manager.get_commit_list(commit_old, commit_new)
old_idx = len(commit_list) - 1
new_idx = 0
- build_specified_commit.build_fuzzer_from_commit(
- build_data.project_name, commit_list[new_idx],
- bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer,
- build_data.architecture, bisect_repo_manager)
+
+ build_specified_commit.build_fuzzers_from_commit(build_data,
+ commit_list[new_idx],
+ bisect_repo_manager)
expected_error_code = helper.reproduce_impl(build_data.project_name,
fuzz_target, False, [], [],
testcase)
# Check if the error is persistent through the commit range
- build_specified_commit.build_fuzzer_from_commit(
- build_data.project_name, commit_list[old_idx],
- bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer,
- build_data.architecture, bisect_repo_manager)
- oldest_error_code = helper.reproduce_impl(build_data.project_name,
- fuzz_target, False, [], [],
- testcase)
-
- if expected_error_code == oldest_error_code:
+ build_specified_commit.build_fuzzers_from_commit(build_data,
+ commit_list[old_idx],
+ bisect_repo_manager)
+
+ if expected_error_code == helper.reproduce_impl(build_data.project_name,
+ fuzz_target, False, [], [],
+ testcase):
return commit_list[old_idx]
while old_idx - new_idx > 1:
curr_idx = (old_idx + new_idx) // 2
- build_specified_commit.build_fuzzer_from_commit(
- build_data.project_name, commit_list[curr_idx],
- bisect_repo_manager.repo_dir, build_data.engine, build_data.sanitizer,
- build_data.architecture, bisect_repo_manager)
+ build_specified_commit.build_fuzzers_from_commit(build_data,
+ commit_list[curr_idx],
+ bisect_repo_manager)
error_code = helper.reproduce_impl(build_data.project_name, fuzz_target,
False, [], [], testcase)
if expected_error_code == error_code:
diff --git a/infra/bisector_test.py b/infra/bisector_test.py
index 1644e8faa..a758d30fe 100644
--- a/infra/bisector_test.py
+++ b/infra/bisector_test.py
@@ -11,27 +11,31 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing perepo_managerissions and
# limitations under the License.
-"""Test the functionality of bisection module.
-1) Test a known case where an error appears in a regression range
-2) Bisect can handle incorrect inputs
+"""Test the functionality of bisection module:
+1) Test a known case where an error appears in a regression range.
+2) Bisect can handle incorrect inputs.
"""
import os
import unittest
import bisector
+import build_specified_commit
# Necessary because __file__ changes with os.chdir
TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__))
class TestBisect(unittest.TestCase):
- """Class to test the functionality of bisection method"""
+ """Class to test the functionality of bisection method."""
def test_bisect_invalid_repo(self):
- """Test the bisection method on a project that does not exist"""
- build_data = bisector.BuildData('not-a-real-repo', 'libfuzzer', 'address',
- 'x86_64')
+ """Test the bisection method on a project that does not exist."""
+ build_data = build_specified_commit.BuildData()
+ build_data.project_name = 'not-a-real-repo'
+ build_data.engine = 'libfuzzer'
+ build_data.sanitizer = 'address'
+ build_data.architecture = 'x86_64'
commit_old = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
commit_new = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
testcase = os.path.join(TEST_DIR_PATH, 'testcases', 'usrsctp_test_data')
@@ -41,7 +45,11 @@ class TestBisect(unittest.TestCase):
def test_bisect_curl(self):
"""Test the bisect method on the curl project."""
- build_data = bisector.BuildData('curl', 'libfuzzer', 'address', 'x86_64')
+ build_data = build_specified_commit.BuildData()
+ build_data.project_name = 'curl'
+ build_data.engine = 'libfuzzer'
+ build_data.sanitizer = 'address'
+ build_data.architecture = 'x86_64'
commit_new = 'dda418266c99ceab368d723facb52069cbb9c8d5'
commit_old = 'df26f5f9c36e19cd503c0e462e9f72ad37b84c82'
fuzz_target = 'curl_fuzzer_ftp'
@@ -52,8 +60,11 @@ class TestBisect(unittest.TestCase):
def test_bisect_libarchive(self):
"""Test the bisect method on libarchive."""
- build_data = bisector.BuildData('libarchive', 'libfuzzer', 'undefined',
- 'x86_64')
+ build_data = build_specified_commit.BuildData()
+ build_data.project_name = 'libarchive'
+ build_data.engine = 'libfuzzer'
+ build_data.sanitizer = 'address'
+ build_data.architecture = 'x86_64'
commit_new = '458e49358f17ec58d65ab1c45cf299baaf3c98d1'
commit_old = '5bd2a9b6658a3a6efa20bb9ad75bd39a44d71da6'
fuzz_target = 'libarchive_fuzzer'
@@ -64,7 +75,11 @@ class TestBisect(unittest.TestCase):
def test_bisect_usrsctp(self):
"""Test the bisect method on the usrsctp."""
- build_data = bisector.BuildData('usrsctp', 'libfuzzer', 'address', 'x86_64')
+ build_data = build_specified_commit.BuildData()
+ build_data.project_name = 'usrsctp'
+ build_data.engine = 'libfuzzer'
+ build_data.sanitizer = 'address'
+ build_data.architecture = 'x86_64'
commit_old = '4886aaa49fb90e479226fcfc3241d74208908232'
commit_new = 'c710749b1053978179a027973a3ea3bccf20ee5c'
testcase = os.path.join(TEST_DIR_PATH, 'testcases', 'usrsctp_test_data')
@@ -75,7 +90,11 @@ class TestBisect(unittest.TestCase):
def test_bisect_usrsctp_single_error_exists(self):
"""Tests what happens with a single with an error."""
- build_data = bisector.BuildData('usrsctp', 'libfuzzer', 'address', 'x86_64')
+ build_data = build_specified_commit.BuildData()
+ build_data.project_name = 'usrsctp'
+ build_data.engine = 'libfuzzer'
+ build_data.sanitizer = 'address'
+ build_data.architecture = 'x86_64'
commit_old = 'c710749b1053978179a027973a3ea3bccf20ee5c'
commit_new = 'c710749b1053978179a027973a3ea3bccf20ee5c'
testcase = os.path.join(TEST_DIR_PATH, 'testcases', 'usrsctp_test_data')
@@ -86,7 +105,11 @@ class TestBisect(unittest.TestCase):
def test_bisect_usrsctp_single_no_error_exists(self):
"""Tests what happens with a single with an error."""
- build_data = bisector.BuildData('usrsctp', 'libfuzzer', 'address', 'x86_64')
+ build_data = build_specified_commit.BuildData()
+ build_data.project_name = 'usrsctp'
+ build_data.engine = 'libfuzzer'
+ build_data.sanitizer = 'address'
+ build_data.architecture = 'x86_64'
commit_old = '4886aaa49fb90e479226fcfc3241d74208908232'
commit_new = '4886aaa49fb90e479226fcfc3241d74208908232'
testcase = os.path.join(TEST_DIR_PATH, 'testcases', 'usrsctp_test_data')
diff --git a/infra/build_specified_commit.py b/infra/build_specified_commit.py
index 2452a4d91..2288cc135 100644
--- a/infra/build_specified_commit.py
+++ b/infra/build_specified_commit.py
@@ -22,72 +22,84 @@ import re
import subprocess
import helper
-import repo_manager
-class DockerExecutionError(Exception):
- """An error that occurs when running a docker command."""
+class BuildData:
+ """Data required for bisection of errors in OSS-Fuzz projects.
+
+ Attributes:
+ project_name: The name of the OSS-Fuzz project that is being checked.
+ engine: The fuzzing engine to be used.
+ sanitizer: The sanitizer to be used.
+ architecture: CPU architecture to build the fuzzer for.
+ """
+
+ # pylint: disable=too-few-public-methods
+
+ def __init__(self):
+ self.project_name = ''
+ self.engine = 'libfuzzer'
+ self.sanitizer = 'address'
+ self.architecture = 'x86_64'
-def build_fuzzer_from_commit(project_name,
- commit,
- local_store_path,
- engine='libfuzzer',
- sanitizer='address',
- architecture='x86_64',
- old_repo_manager=None):
+def build_fuzzers_from_commit(commit, build_repo_manager, build_data):
"""Builds a OSS-Fuzz fuzzer at a specific commit SHA.
Args:
- project_name: The OSS-Fuzz project name
- commit: The commit SHA to build the fuzzers at
- local_store_path: The full file path of a place where a temp git repo is stored
- engine: The fuzzing engine to be used
- sanitizer: The fuzzing sanitizer to be used
- architecture: The system architiecture to be used for fuzzing
-
+ commit: The commit SHA to build the fuzzers at.
+ build_repo_manager: The OSS-Fuzz project's repo manager to be built at.
+ build_data: A struct containing project build information.
Returns:
- 0 on successful build 1 on failure
+ 0 on successful build or error code on failure.
"""
- if not old_repo_manager:
- inferred_url, repo_name = detect_main_repo_from_docker(project_name, commit)
- old_repo_manager = repo_manager.RepoManager(
- inferred_url, local_store_path, repo_name=repo_name)
- old_repo_manager.checkout_commit(commit)
- return helper.build_fuzzers_impl(
- project_name=project_name,
- clean=True,
- engine=engine,
- sanitizer=sanitizer,
- architecture=architecture,
- env_to_add=None,
- source_path=old_repo_manager.repo_dir,
- mount_location=os.path.join('/src', old_repo_manager.repo_name))
-
-
-def detect_main_repo_from_docker(project_name, example_commit, src_dir='/src'):
+ build_repo_manager.checkout_commit(commit)
+ return helper.build_fuzzers_impl(project_name=build_data.project_name,
+ clean=True,
+ engine=build_data.engine,
+ sanitizer=build_data.sanitizer,
+ architecture=build_data.architecture,
+ env_to_add=None,
+ source_path=build_repo_manager.repo_dir,
+ mount_location=os.path.join(
+ '/src', build_repo_manager.repo_name))
+
+
+def detect_main_repo(project_name, repo_name=None, commit=None, src_dir='/src'):
"""Checks a docker image for the main repo of an OSS-Fuzz project.
+ Note: The default is to use the repo name to detect the main repo.
+
Args:
- project_name: The name of the OSS-Fuzz project
- example_commit: An associated commit SHA
- src_dir: The location of the projects source on the docker image
+ project_name: The name of the oss-fuzz project.
+ repo_name: The name of the main repo in an OSS-Fuzz project.
+ commit: A commit SHA that is associated with the main repo.
+ src_dir: The location of the projects source on the docker image.
Returns:
- The repo's origin, the repo's name
+ The repo's origin, the repo's name.
"""
+ # TODO: Add infra for non hardcoded '/src'.
+ if not repo_name and not commit:
+ print('Error: can not detect main repo without a repo_name or a commit.')
+ return None, None
+ if repo_name and commit:
+ print('Both repo name and commit specific. Using repo name for detection.')
+
helper.build_image_impl(project_name)
docker_image_name = 'gcr.io/oss-fuzz/' + project_name
command_to_run = [
- 'docker', 'run', '--rm', '-i', '-t', docker_image_name, 'python3',
- os.path.join(src_dir, 'detect_repo.py'), '--src_dir', src_dir,
- '--example_commit', example_commit
+ 'docker', 'run', '--rm', '-t', docker_image_name, 'python3',
+ os.path.join(src_dir, 'detect_repo.py'), '--src_dir', src_dir
]
+ if repo_name:
+ command_to_run.extend(['--repo_name', repo_name])
+ else:
+ command_to_run.extend(['--example_commit', commit])
out, _ = execute(command_to_run)
-
match = re.search(r'\bDetected repo: ([^ ]+) ([^ ]+)', out.rstrip())
if match and match.group(1) and match.group(2):
- return match.group(1), match.group(2).rstrip()
+ return match.group(1), match.group(2)
return None, None
@@ -95,15 +107,15 @@ def execute(command, location=None, check_result=False):
""" Runs a shell command in the specified directory location.
Args:
- command: The command as a list to be run
- location: The directory the command is run in
- check_result: Should an exception be thrown on failed command
+ command: The command as a list to be run.
+ location: The directory the command is run in.
+ check_result: Should an exception be thrown on failed command.
Returns:
- The stdout of the command, the error code
+ The stdout of the command, the error code.
Raises:
- RuntimeError: running a command resulted in an error
+ RuntimeError: running a command resulted in an error.
"""
if not location:
diff --git a/infra/build_specified_commit_test.py b/infra/build_specified_commit_test.py
index e151fe52f..ef0eb1f3f 100644
--- a/infra/build_specified_commit_test.py
+++ b/infra/build_specified_commit_test.py
@@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test the functionality of the build image from commit module.
-The will consist of the following functional tests
- 1. The inferance of the main repo for a specific project
+The will consist of the following functional tests:
+ 1. The inferance of the main repo for a specific project.
"""
import os
import tempfile
@@ -21,13 +21,14 @@ import unittest
import build_specified_commit
import helper
+import repo_manager
# Necessary because __file__ changes with os.chdir
TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__))
class BuildImageIntegrationTests(unittest.TestCase):
- """Testing if an image can be built from different states e.g. a commit"""
+ """Testing if an image can be built from different states e.g. a commit."""
def test_build_fuzzers_from_commit(self):
"""Tests if the fuzzers can build at a proper commit.
@@ -43,42 +44,79 @@ class BuildImageIntegrationTests(unittest.TestCase):
old_commit = 'f79be4f2330f4b89ea2f42e1c44ca998c59a0c0f'
new_commit = 'f50a39051ea8c7f10d6d8db9656658b49601caef'
fuzzer = 'rules_fuzzer'
- build_specified_commit.build_fuzzer_from_commit(
- project_name, old_commit, tmp_dir, sanitizer='address')
+
+ yara_repo_manager = repo_manager.RepoManager(
+ 'https://github.com/VirusTotal/yara.git', tmp_dir, repo_name='yara')
+ build_data = build_specified_commit.BuildData()
+ build_data.sanitizer = 'address'
+ build_data.architecture = 'x86_64'
+ build_data.engine = 'libfuzzer'
+ build_data.project_name = 'yara'
+ build_specified_commit.build_fuzzers_from_commit(old_commit,
+ yara_repo_manager,
+ build_data)
old_error_code = helper.reproduce_impl(project_name, fuzzer, False, [],
[], test_data)
- build_specified_commit.build_fuzzer_from_commit(
- project_name, new_commit, tmp_dir, sanitizer='address')
+ build_specified_commit.build_fuzzers_from_commit(new_commit,
+ yara_repo_manager,
+ build_data)
new_error_code = helper.reproduce_impl(project_name, fuzzer, False, [],
[], test_data)
self.assertNotEqual(new_error_code, old_error_code)
- def test_detect_main_repo(self):
- """Test the detect main repo functionality of the build specific commit module."""
- repo_origin, repo_name = build_specified_commit.detect_main_repo_from_docker(
- 'curl', 'bc5d22c3dede2f04870c37aec9a50474c4b888ad')
+ def test_detect_main_repo_from_commit(self):
+ """Test the detect main repo function from build specific commit module."""
+ repo_origin, repo_name = build_specified_commit.detect_main_repo(
+ 'curl', commit='bc5d22c3dede2f04870c37aec9a50474c4b888ad')
+ self.assertEqual(repo_origin, 'https://github.com/curl/curl.git')
+ self.assertEqual(repo_name, 'curl')
+
+ repo_origin, repo_name = build_specified_commit.detect_main_repo(
+ 'usrsctp', commit='4886aaa49fb90e479226fcfc3241d74208908232')
+ self.assertEqual(repo_origin, 'https://github.com/weinrank/usrsctp')
+ self.assertEqual(repo_name, 'usrsctp')
+
+ repo_origin, repo_name = build_specified_commit.detect_main_repo(
+ 'ndpi', commit='c4d476cc583a2ef1e9814134efa4fbf484564ed7')
+ self.assertEqual(repo_origin, 'https://github.com/ntop/nDPI.git')
+ self.assertEqual(repo_name, 'ndpi')
+
+ repo_origin, repo_name = build_specified_commit.detect_main_repo(
+ 'notproj', commit='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
+ self.assertIsNone(repo_origin)
+ self.assertIsNone(repo_name)
+
+ def test_detect_main_repo_from_name(self):
+ """Test the detect main repo function from build specific commit module."""
+ repo_origin, repo_name = build_specified_commit.detect_main_repo(
+ 'curl', repo_name='curl')
self.assertEqual(repo_origin, 'https://github.com/curl/curl.git')
self.assertEqual(repo_name, 'curl')
- repo_origin, repo_name = build_specified_commit.detect_main_repo_from_docker(
- 'usrsctp', '4886aaa49fb90e479226fcfc3241d74208908232')
+ repo_origin, repo_name = build_specified_commit.detect_main_repo(
+ 'yara', repo_name='yara')
+ self.assertEqual(repo_origin, 'https://github.com/VirusTotal/yara.git')
+ self.assertEqual(repo_name, 'yara')
+
+ repo_origin, repo_name = build_specified_commit.detect_main_repo(
+ 'usrsctp', repo_name='usrsctp')
self.assertEqual(repo_origin, 'https://github.com/weinrank/usrsctp')
self.assertEqual(repo_name, 'usrsctp')
- repo_origin, repo_name = build_specified_commit.detect_main_repo_from_docker(
- 'ndpi', 'c4d476cc583a2ef1e9814134efa4fbf484564ed7')
+ repo_origin, repo_name = build_specified_commit.detect_main_repo(
+ 'ndpi', repo_name='nDPI')
self.assertEqual(repo_origin, 'https://github.com/ntop/nDPI.git')
self.assertEqual(repo_name, 'ndpi')
- repo_origin, repo_name = build_specified_commit.detect_main_repo_from_docker(
- 'notproj', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
+ repo_origin, repo_name = build_specified_commit.detect_main_repo(
+ 'notproj', repo_name='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
self.assertIsNone(repo_origin)
self.assertIsNone(repo_name)
if __name__ == '__main__':
- # Change to oss-fuzz main directory so helper.py runs correctly
+ # Change to oss-fuzz main directory so helper.py runs correctly.
if os.getcwd() != os.path.dirname(TEST_DIR_PATH):
os.chdir(os.path.dirname(TEST_DIR_PATH))
unittest.main()
diff --git a/infra/cifuzz.py b/infra/cifuzz.py
new file mode 100644
index 000000000..161e9d953
--- /dev/null
+++ b/infra/cifuzz.py
@@ -0,0 +1,88 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module used by CI tools in order to interact with fuzzers.
+This module helps CI tools do the following:
+ 1. Build fuzzers.
+ 2. Run fuzzers.
+Eventually it will be used to help CI tools determine which fuzzers to run.
+"""
+
+import argparse
+import os
+import tempfile
+
+import build_specified_commit
+import repo_manager
+import helper
+
+
+def main():
+ """Connects Fuzzers with CI tools.
+
+ Returns:
+ True on success False on failure.
+ """
+ parser = argparse.ArgumentParser(
+ description='Help CI tools manage specific fuzzers.')
+
+ subparsers = parser.add_subparsers(dest='command')
+ build_fuzzer_parser = subparsers.add_parser(
+ 'build_fuzzers', help='Build an OSS-Fuzz projects fuzzers.')
+ build_fuzzer_parser.add_argument('project_name')
+ build_fuzzer_parser.add_argument('repo_name')
+ build_fuzzer_parser.add_argument('commit_sha')
+
+ run_fuzzer_parser = subparsers.add_parser(
+ 'run_fuzzers', help='Run an OSS-Fuzz projects fuzzers.')
+ run_fuzzer_parser.add_argument('project_name')
+ args = parser.parse_args()
+
+ # Change to oss-fuzz main directory so helper.py runs correctly.
+ if os.getcwd() != helper.OSSFUZZ_DIR:
+ os.chdir(helper.OSSFUZZ_DIR)
+
+ if args.command == 'build_fuzzers':
+ return build_fuzzers(args) == 0
+ if args.command == 'run_fuzzer':
+ print('Not implemented')
+ return False
+ print('Invalid argument option, use build_fuzzers or run_fuzzer.')
+ return False
+
+
+def build_fuzzers(args):
+ """Builds all of the fuzzers for a specific OSS-Fuzz project.
+
+ Returns:
+ True on success False on failure.
+ """
+
+ # TODO: Fix return value bubble to actually handle errors.
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ inferred_url, repo_name = build_specified_commit.detect_main_repo(
+ args.project_name, repo_name=args.repo_name)
+ build_repo_manager = repo_manager.RepoManager(inferred_url,
+ tmp_dir,
+ repo_name=repo_name)
+ build_data = build_specified_commit.BuildData()
+ build_data.project_name = args.project_name
+ build_data.sanitizer = 'address'
+ build_data.engine = 'libfuzzer'
+ build_data.architecture = 'x86_64'
+ return build_specified_commit.build_fuzzers_from_commit(
+ args.commit_sha, build_repo_manager, build_data) == 0
+
+
+if __name__ == '__main__':
+ main()
diff --git a/infra/helper.py b/infra/helper.py
index 46c34abbf..4d83175f1 100755
--- a/infra/helper.py
+++ b/infra/helper.py
@@ -342,7 +342,12 @@ def _workdir_from_dockerfile(project_name):
def docker_run(run_args, print_output=True):
"""Call `docker run`."""
- command = ['docker', 'run', '--rm', '-i', '--privileged']
+ command = ['docker', 'run', '--rm', '--privileged']
+
+ # Support environments with a TTY.
+ if sys.stdin.isatty():
+ command.append('-i')
+
command.extend(run_args)
print('Running:', _get_command_string(command))
@@ -453,9 +458,7 @@ def build_fuzzers_impl(project_name, clean, engine, sanitizer, architecture,
'bash', '-c', 'cp -r /msan /work'])
env.append('MSAN_LIBS_PATH=' + '/work/msan')
- command = (
- ['docker', 'run', '--rm', '-i', '--cap-add', 'SYS_PTRACE'] +
- _env_to_docker_args(env))
+ command = ['--cap-add', 'SYS_PTRACE'] + _env_to_docker_args(env)
if source_path:
workdir = _workdir_from_dockerfile(project_name)
if workdir == '/src':
@@ -478,13 +481,10 @@ def build_fuzzers_impl(project_name, clean, engine, sanitizer, architecture,
'-t', 'gcr.io/oss-fuzz/%s' % project_name
]
- print('Running:', _get_command_string(command))
-
- try:
- subprocess.check_call(command)
- except subprocess.CalledProcessError:
- print('Fuzzers build failed.', file=sys.stderr)
- return 1
+ result_code = docker_run(command)
+ if result_code:
+ print('Building fuzzers failed.', file=sys.stderr)
+ return result_code
# Patch MSan builds to use instrumented shared libraries.
if sanitizer == 'memory':