1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module used by CI tools in order to interact with fuzzers. This module helps
CI tools to build fuzzers."""
import logging
import os
import sys
import affected_fuzz_targets
import base_runner_utils
import clusterfuzz_deployment
import continuous_integration
import docker
import workspace_utils
# pylint: disable=wrong-import-position,import-error
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import helper
import utils
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.DEBUG)
def check_project_src_path(project_src_path):
"""Returns True if |project_src_path| exists."""
if not os.path.exists(project_src_path):
logging.error(
'PROJECT_SRC_PATH: %s does not exist. '
'Are you mounting it correctly?', project_src_path)
return False
return True
# pylint: disable=too-many-arguments
class Builder: # pylint: disable=too-many-instance-attributes
"""Class for fuzzer builders."""
def __init__(self, config, ci_system):
self.config = config
self.ci_system = ci_system
self.workspace = workspace_utils.Workspace(config)
self.workspace.initialize_dir(self.workspace.out)
self.workspace.initialize_dir(self.workspace.work)
self.clusterfuzz_deployment = (
clusterfuzz_deployment.get_clusterfuzz_deployment(
self.config, self.workspace))
self.image_repo_path = None
self.host_repo_path = None
self.repo_manager = None
def build_image_and_checkout_src(self):
"""Builds the project builder image and checkout source code for the patch
we want to fuzz (if necessary). Returns True on success."""
result = self.ci_system.prepare_for_fuzzer_build()
if not result.success:
return False
self.image_repo_path = result.image_repo_path
self.repo_manager = result.repo_manager
logging.info('repo_dir: %s.', self.repo_manager.repo_dir)
self.host_repo_path = self.repo_manager.repo_dir
return True
def build_fuzzers(self):
"""Moves the source code we want to fuzz into the project builder and builds
the fuzzers from that source code. Returns True on success."""
docker_args, docker_container = docker.get_base_docker_run_args(
self.workspace, self.config.sanitizer, self.config.language)
if not docker_container:
docker_args.extend(
_get_docker_build_fuzzers_args_not_container(self.host_repo_path))
docker_args.extend([
docker.get_project_image_name(self.config.oss_fuzz_project_name),
'/bin/bash',
'-c',
])
build_command = self.ci_system.get_build_command(self.host_repo_path,
self.image_repo_path)
docker_args.append(build_command)
logging.info('Building with %s sanitizer.', self.config.sanitizer)
# TODO(metzman): Stop using helper.docker_run so we can get rid of
# docker.get_base_docker_run_args and merge its contents into
# docker.get_base_docker_run_command.
if not helper.docker_run(docker_args):
logging.error('Building fuzzers failed.')
return False
return True
def upload_build(self):
"""Upload build."""
if self.config.upload_build:
self.clusterfuzz_deployment.upload_build(
self.repo_manager.get_current_commit())
return True
def check_fuzzer_build(self):
"""Checks the fuzzer build. Returns True on success or if config specifies
to skip check."""
if not self.config.bad_build_check:
return True
return check_fuzzer_build(self.config)
def build(self):
"""Builds the image, checkouts the source (if needed), builds the fuzzers
and then removes the unaffectted fuzzers. Returns True on success."""
methods = [
self.build_image_and_checkout_src,
self.build_fuzzers,
self.remove_unaffected_fuzz_targets,
self.check_fuzzer_build,
self.upload_build,
]
for method in methods:
if not method():
return False
return True
def remove_unaffected_fuzz_targets(self):
"""Removes the fuzzers unaffected by the patch."""
if self.config.keep_unaffected_fuzz_targets:
logging.info('Not removing unaffected fuzz targets.')
return True
logging.info('Removing unaffected fuzz targets.')
changed_files = self.ci_system.get_changed_code_under_test(
self.repo_manager)
affected_fuzz_targets.remove_unaffected_fuzz_targets(
self.clusterfuzz_deployment, self.workspace.out, changed_files,
self.image_repo_path)
return True
def build_fuzzers(config):
"""Builds all of the fuzzers for a specific OSS-Fuzz project.
Args:
project_name: The name of the OSS-Fuzz project being built.
project_repo_name: The name of the project's repo.
workspace: The location in a shared volume to store a git repo and build
artifacts.
pr_ref: The pull request reference to be built.
commit_sha: The commit sha for the project to be built at.
sanitizer: The sanitizer the fuzzers should be built with.
Returns:
True if build succeeded or False on failure.
"""
# Do some quick validation.
if config.project_src_path and not check_project_src_path(
config.project_src_path):
return False
# Get the builder and then build the fuzzers.
ci_system = continuous_integration.get_ci(config)
logging.info('ci_system: %s.', ci_system)
builder = Builder(config, ci_system)
return builder.build()
def check_fuzzer_build(config):
"""Checks the integrity of the built fuzzers.
Args:
config: The config object.
Returns:
True if fuzzers pass OSS-Fuzz's build check.
"""
workspace = workspace_utils.Workspace(config)
if not os.path.exists(workspace.out):
logging.error('Invalid out directory: %s.', workspace.out)
return False
if not os.listdir(workspace.out):
logging.error('No fuzzers found in out directory: %s.', workspace.out)
return False
env = base_runner_utils.get_env(config, workspace)
if config.allowed_broken_targets_percentage is not None:
env['ALLOWED_BROKEN_TARGETS_PERCENTAGE'] = (
config.allowed_broken_targets_percentage)
stdout, stderr, retcode = utils.execute('test_all.py', env=env)
print(f'Build check: stdout: {stdout}\nstderr: {stderr}')
if retcode == 0:
logging.info('Build check passed.')
return True
logging.error('Build check failed.')
return False
def _get_docker_build_fuzzers_args_not_container(host_repo_path):
"""Returns arguments to the docker build arguments that are needed to use
|host_repo_path| when the host of the OSS-Fuzz builder container is not
another container."""
return ['-v', f'{host_repo_path}:{host_repo_path}']
|
