1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
#! /usr/bin/env python3
# Copyright 2021 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Builds the content of xds-protos package"""
import os
import pkg_resources
from grpc_tools import protoc
# We might not want to compile all the protos
EXCLUDE_PROTO_PACKAGES_LIST = [
# Requires extra dependency to Prometheus protos
'envoy/service/metrics/v2',
'envoy/service/metrics/v3',
'envoy/service/metrics/v4alpha',
]
# Compute the pathes
WORK_DIR = os.path.dirname(os.path.abspath(__file__))
GRPC_ROOT = os.path.abspath(os.path.join(WORK_DIR, '..', '..', '..', '..'))
XDS_PROTO_ROOT = os.path.join(GRPC_ROOT, 'third_party', 'envoy-api')
UDPA_PROTO_ROOT = os.path.join(GRPC_ROOT, 'third_party', 'udpa')
GOOGLEAPIS_ROOT = os.path.join(GRPC_ROOT, 'third_party', 'googleapis')
VALIDATE_ROOT = os.path.join(GRPC_ROOT, 'third_party', 'protoc-gen-validate')
OPENCENSUS_PROTO_ROOT = os.path.join(GRPC_ROOT, 'third_party',
'opencensus-proto', 'src')
WELL_KNOWN_PROTOS_INCLUDE = pkg_resources.resource_filename(
'grpc_tools', '_proto')
OUTPUT_PATH = WORK_DIR
# Prepare the test file generation
TEST_FILE_NAME = 'generated_file_import_test.py'
TEST_IMPORTS = []
# The pkgutil-style namespace packaging __init__.py
PKGUTIL_STYLE_INIT = "__path__ = __import__('pkgutil').extend_path(__path__, __name__)\n"
NAMESPACE_PACKAGES = ["google"]
def add_test_import(proto_package_path: str,
file_name: str,
service: bool = False):
TEST_IMPORTS.append("from %s import %s\n" % (proto_package_path.replace(
'/', '.'), file_name.replace('.proto', '_pb2')))
if service:
TEST_IMPORTS.append("from %s import %s\n" % (proto_package_path.replace(
'/', '.'), file_name.replace('.proto', '_pb2_grpc')))
# Prepare Protoc command
COMPILE_PROTO_ONLY = [
'grpc_tools.protoc',
'--proto_path={}'.format(XDS_PROTO_ROOT),
'--proto_path={}'.format(UDPA_PROTO_ROOT),
'--proto_path={}'.format(GOOGLEAPIS_ROOT),
'--proto_path={}'.format(VALIDATE_ROOT),
'--proto_path={}'.format(WELL_KNOWN_PROTOS_INCLUDE),
'--proto_path={}'.format(OPENCENSUS_PROTO_ROOT),
'--python_out={}'.format(OUTPUT_PATH),
]
COMPILE_BOTH = COMPILE_PROTO_ONLY + ['--grpc_python_out={}'.format(OUTPUT_PATH)]
def has_grpc_service(proto_package_path: str) -> bool:
return proto_package_path.startswith('envoy/service')
def compile_protos(proto_root: str, sub_dir: str = '.') -> None:
for root, _, files in os.walk(os.path.join(proto_root, sub_dir)):
proto_package_path = os.path.relpath(root, proto_root)
if proto_package_path in EXCLUDE_PROTO_PACKAGES_LIST:
print(f'Skipping package {proto_package_path}')
continue
for file_name in files:
if file_name.endswith('.proto'):
# Compile proto
if has_grpc_service(proto_package_path):
return_code = protoc.main(COMPILE_BOTH +
[os.path.join(root, file_name)])
add_test_import(proto_package_path, file_name, service=True)
else:
return_code = protoc.main(COMPILE_PROTO_ONLY +
[os.path.join(root, file_name)])
add_test_import(proto_package_path,
file_name,
service=False)
if return_code != 0:
raise Exception('error: {} failed'.format(COMPILE_BOTH))
def create_init_file(path: str, package_path: str = "") -> None:
with open(os.path.join(path, "__init__.py"), 'w') as f:
# Apply the pkgutil-style namespace packaging, which is compatible for 2
# and 3. Here is the full table of namespace compatibility:
# https://github.com/pypa/sample-namespace-packages/blob/master/table.md
if package_path in NAMESPACE_PACKAGES:
f.write(PKGUTIL_STYLE_INIT)
def main():
# Compile xDS protos
compile_protos(XDS_PROTO_ROOT)
compile_protos(UDPA_PROTO_ROOT)
# We don't want to compile the entire GCP surface API, just the essential ones
compile_protos(GOOGLEAPIS_ROOT, os.path.join('google', 'api'))
compile_protos(GOOGLEAPIS_ROOT, os.path.join('google', 'rpc'))
compile_protos(GOOGLEAPIS_ROOT, os.path.join('google', 'longrunning'))
compile_protos(GOOGLEAPIS_ROOT, os.path.join('google', 'logging'))
compile_protos(GOOGLEAPIS_ROOT, os.path.join('google', 'type'))
compile_protos(VALIDATE_ROOT, 'validate')
compile_protos(OPENCENSUS_PROTO_ROOT)
# Generate __init__.py files for all modules
create_init_file(WORK_DIR)
for proto_root_module in [
'envoy', 'google', 'opencensus', 'udpa', 'validate', 'xds'
]:
for root, _, _ in os.walk(os.path.join(WORK_DIR, proto_root_module)):
package_path = os.path.relpath(root, WORK_DIR)
create_init_file(root, package_path)
# Generate test file
with open(os.path.join(WORK_DIR, TEST_FILE_NAME), 'w') as f:
f.writelines(TEST_IMPORTS)
if __name__ == "__main__":
main()
|
