1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
#!/usr/bin/env python3
# Copyright 2022 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import re
import subprocess
import sys
from google.cloud import bigquery
import run_buildozer
import update_flakes_query
lookback_hours = 24 * 7 * 4
def include_test(test):
if '@' in test:
return False
if test.startswith("//test/cpp/qps:"):
return False
return True
TEST_DIRS = ['test/core', 'test/cpp']
tests = {}
already_flaky = set()
for test_dir in TEST_DIRS:
for line in subprocess.check_output(
['bazel', 'query', 'tests({}/...)'.format(test_dir)]).splitlines():
test = line.strip().decode('utf-8')
if not include_test(test):
continue
tests[test] = False
for test_dir in TEST_DIRS:
for line in subprocess.check_output(
['bazel', 'query',
'attr(flaky, 1, tests({}/...))'.format(test_dir)]).splitlines():
test = line.strip().decode('utf-8')
if not include_test(test):
continue
already_flaky.add(test)
flaky_e2e = set()
client = bigquery.Client()
for row in client.query(
update_flakes_query.QUERY.format(
lookback_hours=lookback_hours)).result():
if "/macos/" in row.job_name:
continue # we know mac stuff is flaky
if row.test_binary not in tests:
m = re.match(r'^//test/core/end2end:([^@]*)@([^@]*)(.*)',
row.test_binary)
if m:
flaky_e2e.add('{}@{}{}'.format(m.group(1), m.group(2), m.group(3)))
print("will mark end2end test {} as flaky".format(row.test_binary))
else:
print("skip obsolete test {}".format(row.test_binary))
continue
print("will mark {} as flaky".format(row.test_binary))
tests[row.test_binary] = True
buildozer_commands = []
for test, flaky in sorted(tests.items()):
if flaky:
buildozer_commands.append('set flaky True|{}'.format(test))
elif test in already_flaky:
buildozer_commands.append('remove flaky|{}'.format(test))
with open('test/core/end2end/flaky.bzl', 'w') as f:
with open(sys.argv[0]) as my_source:
for line in my_source:
if line[0] != '#':
break
for line in my_source:
if line[0] == '#':
print(line.strip(), file=f)
break
for line in my_source:
if line[0] != '#':
break
print(line.strip(), file=f)
print(
"\"\"\"A list of flaky tests, consumed by generate_tests.bzl to set flaky attrs.\"\"\"",
file=f)
print("FLAKY_TESTS = [", file=f)
for line in sorted(list(flaky_e2e)):
print(" \"{}\",".format(line), file=f)
print("]", file=f)
run_buildozer.run_buildozer(buildozer_commands)
|
