1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
|
# Copyright 2013 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""An example main file running the algorithms.
Part of the Chrome build flags optimization.
An example use of the framework. It parses the input json configuration file.
Then it initiates the variables of the generation. Finally, it sets up the
processes for different modules and runs the experiment.
"""
__author__ = "yuhenglong@google.com (Yuheng Long)"
import json
import multiprocessing
from optparse import OptionParser
import sys
import flags
from genetic_algorithm import GAGeneration
from pipeline_process import PipelineProcess
import pipeline_worker
from steering import Steering
from task import BUILD_STAGE
from task import Task
from task import TEST_STAGE
import testing_batch
parser = OptionParser()
parser.add_option(
"-f",
"--file",
dest="filename",
help="configuration file FILE input",
metavar="FILE",
)
# The meta data for the genetic algorithm.
BUILD_CMD = "BUILD_CMD"
TEST_CMD = "TEST_CMD"
OUTPUT = "OUTPUT"
DEFAULT_OUTPUT = "output"
CONF = "CONF"
DEFAULT_CONF = "conf"
NUM_BUILDER = "NUM_BUILDER"
DEFAULT_NUM_BUILDER = 1
NUM_TESTER = "NUM_TESTER"
DEFAULT_NUM_TESTER = 1
STOP_THRESHOLD = "STOP_THRESHOLD"
DEFAULT_STOP_THRESHOLD = 1
NUM_CHROMOSOMES = "NUM_CHROMOSOMES"
DEFAULT_NUM_CHROMOSOMES = 20
NUM_TRIALS = "NUM_TRIALS"
DEFAULT_NUM_TRIALS = 20
MUTATION_RATE = "MUTATION_RATE"
DEFAULT_MUTATION_RATE = 0.01
def _ProcessGA(meta_data):
"""Set up the meta data for the genetic algorithm.
Args:
meta_data: the meta data for the genetic algorithm.
"""
assert BUILD_CMD in meta_data
build_cmd = meta_data[BUILD_CMD]
assert TEST_CMD in meta_data
test_cmd = meta_data[TEST_CMD]
if OUTPUT not in meta_data:
output_file = DEFAULT_OUTPUT
else:
output_file = meta_data[OUTPUT]
if CONF not in meta_data:
conf_file = DEFAULT_CONF
else:
conf_file = meta_data[CONF]
if NUM_BUILDER not in meta_data:
num_builders = DEFAULT_NUM_BUILDER
else:
num_builders = meta_data[NUM_BUILDER]
if NUM_TESTER not in meta_data:
num_testers = DEFAULT_NUM_TESTER
else:
num_testers = meta_data[NUM_TESTER]
if STOP_THRESHOLD not in meta_data:
stop_threshold = DEFAULT_STOP_THRESHOLD
else:
stop_threshold = meta_data[STOP_THRESHOLD]
if NUM_CHROMOSOMES not in meta_data:
num_chromosomes = DEFAULT_NUM_CHROMOSOMES
else:
num_chromosomes = meta_data[NUM_CHROMOSOMES]
if NUM_TRIALS not in meta_data:
num_trials = DEFAULT_NUM_TRIALS
else:
num_trials = meta_data[NUM_TRIALS]
if MUTATION_RATE not in meta_data:
mutation_rate = DEFAULT_MUTATION_RATE
else:
mutation_rate = meta_data[MUTATION_RATE]
specs = flags.ReadConf(conf_file)
# Initiate the build/test command and the log directory.
Task.InitLogCommand(build_cmd, test_cmd, output_file)
# Initiate the build/test command and the log directory.
GAGeneration.InitMetaData(
stop_threshold, num_chromosomes, num_trials, specs, mutation_rate
)
# Generate the initial generations.
generation_tasks = testing_batch.GenerateRandomGATasks(
specs, num_chromosomes, num_trials
)
generations = [GAGeneration(generation_tasks, set([]), 0)]
# Execute the experiment.
_StartExperiment(num_builders, num_testers, generations)
def _ParseJson(file_name):
"""Parse the input json file.
Parse the input json file and call the proper function to perform the
algorithms.
Args:
file_name: the input json file name.
"""
experiments = json.load(open(file_name))
for experiment in experiments:
if experiment == "GA":
# An GA experiment
_ProcessGA(experiments[experiment])
def _StartExperiment(num_builders, num_testers, generations):
"""Set up the experiment environment and execute the framework.
Args:
num_builders: number of concurrent builders.
num_testers: number of concurrent testers.
generations: the initial generation for the framework.
"""
manager = multiprocessing.Manager()
# The queue between the steering algorithm and the builder.
steering_build = manager.Queue()
# The queue between the builder and the tester.
build_test = manager.Queue()
# The queue between the tester and the steering algorithm.
test_steering = manager.Queue()
# Set up the processes for the builder, tester and steering algorithm module.
build_process = PipelineProcess(
num_builders,
"builder",
{},
BUILD_STAGE,
steering_build,
pipeline_worker.Helper,
pipeline_worker.Worker,
build_test,
)
test_process = PipelineProcess(
num_testers,
"tester",
{},
TEST_STAGE,
build_test,
pipeline_worker.Helper,
pipeline_worker.Worker,
test_steering,
)
steer_process = multiprocessing.Process(
target=Steering,
args=(set([]), generations, test_steering, steering_build),
)
# Start the processes.
build_process.start()
test_process.start()
steer_process.start()
# Wait for the processes to finish.
build_process.join()
test_process.join()
steer_process.join()
def main(argv):
(options, _) = parser.parse_args(argv)
assert options.filename
_ParseJson(options.filename)
if __name__ == "__main__":
main(sys.argv)
|
