diff options
| -rwxr-xr-x | pw_doctor/py/pw_doctor/doctor.py | 2 | ||||
| -rw-r--r-- | pw_env_setup/bazel/cipd_setup/internal/cipd_internal.bzl | 4 | ||||
| -rw-r--r-- | pw_env_setup/py/pw_env_setup/cipd_setup/arm.json | 28 | ||||
| -rw-r--r-- | pw_env_setup/py/pw_env_setup/cipd_setup/bazel.json | 28 | ||||
| -rw-r--r-- | pw_env_setup/py/pw_env_setup/cipd_setup/compatibility.json | 30 | ||||
| -rw-r--r-- | pw_env_setup/py/pw_env_setup/cipd_setup/kythe.json | 42 | ||||
| -rw-r--r-- | pw_env_setup/py/pw_env_setup/cipd_setup/luci.json | 158 | ||||
| -rw-r--r-- | pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json | 408 | ||||
| -rw-r--r-- | pw_env_setup/py/pw_env_setup/cipd_setup/python.json | 52 | ||||
| -rwxr-xr-x | pw_env_setup/py/pw_env_setup/cipd_setup/update.py | 28 |
10 files changed, 386 insertions, 394 deletions
diff --git a/pw_doctor/py/pw_doctor/doctor.py b/pw_doctor/py/pw_doctor/doctor.py index 5bf3ef77a..f91800520 100755 --- a/pw_doctor/py/pw_doctor/doctor.py +++ b/pw_doctor/py/pw_doctor/doctor.py @@ -333,7 +333,7 @@ def cipd_versions(ctx: DoctorContext): 'CIPD package %s is out of date, please rerun bootstrap', installed['package_name']) - for package in json.loads(json_path.read_text()): + for package in json.loads(json_path.read_text()).get('packages', ()): ctx.submit(check_cipd, package) diff --git a/pw_env_setup/bazel/cipd_setup/internal/cipd_internal.bzl b/pw_env_setup/bazel/cipd_setup/internal/cipd_internal.bzl index 150ae2e27..4c434f091 100644 --- a/pw_env_setup/bazel/cipd_setup/internal/cipd_internal.bzl +++ b/pw_env_setup/bazel/cipd_setup/internal/cipd_internal.bzl @@ -174,9 +174,9 @@ def cipd_deps_impl(repository_ctx): """ Generates a CIPD dependencies file """ pigweed_deps = json.decode( repository_ctx.read(repository_ctx.attr._pigweed_packages_json), - ) + json.decode( + )["packages"] + json.decode( repository_ctx.read(repository_ctx.attr._python_packages_json), - ) + )["packages"] repository_ctx.file("BUILD", "exports_files(glob([\"**/*\"]))\n") repository_ctx.file("cipd_init.bzl", CIPD_INIT_BZL_TEMPLATE.format( diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/arm.json b/pw_env_setup/py/pw_env_setup/cipd_setup/arm.json index 1ec4fa2a2..dbe0164bf 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/arm.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/arm.json @@ -1,13 +1,15 @@ -[ - { - "path": "pigweed/third_party/gcc-arm-none-eabi/${platform}", - "platforms": [ - "linux-amd64", - "mac-amd64", - "windows-amd64" - ], - "tags": [ - "version:10-2020-q4-major" - ] - } -] +{ + "packages": [ + { + "path": "pigweed/third_party/gcc-arm-none-eabi/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64", + "windows-amd64" + ], + "tags": [ + "version:10-2020-q4-major" + ] + } + ] +} diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/bazel.json b/pw_env_setup/py/pw_env_setup/cipd_setup/bazel.json index 657d5d04c..74a32d767 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/bazel.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/bazel.json @@ -1,13 +1,15 @@ -[ - { - "path": "pigweed/third_party/bazel/${platform}", - "platforms": [ - "linux-amd64", - "mac-amd64", - "windows-amd64" - ], - "tags": [ - "version:5.0.0" - ] - } -] +{ + "packages": [ + { + "path": "pigweed/third_party/bazel/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64", + "windows-amd64" + ], + "tags": [ + "version:5.0.0" + ] + } + ] +} diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/compatibility.json b/pw_env_setup/py/pw_env_setup/cipd_setup/compatibility.json index c16881f40..7afe4d365 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/compatibility.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/compatibility.json @@ -1,14 +1,16 @@ -[ - { - "path": "infra/3pp/tools/cpython3/${os}-${arch=amd64}", - "tags": [ - "version:3.7.7.chromium.10" - ] - }, - { - "path": "pigweed/third_party/gcc-arm-none-eabi/${os}-${arch=amd64}", - "tags": [ - "version:9-2020-q2-update" - ] - } -] +{ + "packages": [ + { + "path": "infra/3pp/tools/cpython3/${os}-${arch=amd64}", + "tags": [ + "version:3.7.7.chromium.10" + ] + }, + { + "path": "pigweed/third_party/gcc-arm-none-eabi/${os}-${arch=amd64}", + "tags": [ + "version:9-2020-q2-update" + ] + } + ] +} diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/kythe.json b/pw_env_setup/py/pw_env_setup/cipd_setup/kythe.json index 33a987559..05407096c 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/kythe.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/kythe.json @@ -1,20 +1,22 @@ -[ - { - "path": "fuchsia/third_party/kythe", - "platforms": [ - "linux-amd64" - ], - "tags": [ - "version:0.0.46" - ] - }, - { - "path": "fuchsia/third_party/kythe-libs/${platform}", - "platforms": [ - "linux-amd64" - ], - "tags": [ - "version:2020-08-05" - ] - } -] +{ + 'packages': [ + { + "path": "fuchsia/third_party/kythe", + "platforms": [ + "linux-amd64" + ], + "tags": [ + "version:0.0.46" + ] + }, + { + "path": "fuchsia/third_party/kythe-libs/${platform}", + "platforms": [ + "linux-amd64" + ], + "tags": [ + "version:2020-08-05" + ] + } + ] +} diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json b/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json index 032505365..fa30083a7 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json @@ -1,92 +1,94 @@ -[ +{ + "packages": [ { - "path": "infra/tools/bb/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "mac-arm64", - "windows-amd64" - ], - "tags": [ - "latest" - ] + "path": "infra/tools/bb/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "mac-arm64", + "windows-amd64" + ], + "tags": [ + "latest" + ] }, { - "path": "infra/tools/luci-auth/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "mac-arm64", - "windows-amd64" - ], - "tags": [ - "latest" - ] + "path": "infra/tools/luci-auth/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "mac-arm64", + "windows-amd64" + ], + "tags": [ + "latest" + ] }, { - "path": "infra/tools/luci/gerrit/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "mac-arm64", - "windows-amd64" - ], - "tags": [ - "latest" - ] + "path": "infra/tools/luci/gerrit/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "mac-arm64", + "windows-amd64" + ], + "tags": [ + "latest" + ] }, { - "path": "infra/tools/luci/gitiles/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "mac-arm64", - "windows-amd64" - ], - "tags": [ - "latest" - ] + "path": "infra/tools/luci/gitiles/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "mac-arm64", + "windows-amd64" + ], + "tags": [ + "latest" + ] }, { - "path": "infra/tools/luci/cas/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "mac-arm64", - "windows-amd64" - ], - "tags": [ - "latest" - ] + "path": "infra/tools/luci/cas/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "mac-arm64", + "windows-amd64" + ], + "tags": [ + "latest" + ] }, { - "path": "infra/tools/luci/led/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "mac-arm64", - "windows-amd64" - ], - "tags": [ - "latest" - ] + "path": "infra/tools/luci/led/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "mac-arm64", + "windows-amd64" + ], + "tags": [ + "latest" + ] }, { - "path": "infra/tools/luci/logdog/logdog/${platform}", - "platforms": [ - "linux-amd64", - "mac-amd64", - "mac-arm64", - "windows-amd64" - ], - "tags": [ - "latest" - ] + "path": "infra/tools/luci/logdog/logdog/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64", + "mac-arm64", + "windows-amd64" + ], + "tags": [ + "latest" + ] } -] + ] +} diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json b/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json index 35b7b726a..2f5e25ef9 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json @@ -1,203 +1,205 @@ -[ - { - "path": "gn/gn/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "mac-arm64", - "windows-amd64" - ], - "tags": [ - "git_revision:0725d7827575b239594fbc8fd5192873a1d62f44" - ] - }, - { - "path": "infra/3pp/tools/ninja/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "mac-arm64", - "windows-amd64" - ], - "tags": [ - "version:2@1.10.2" - ] - }, - { - "path": "fuchsia/third_party/cmake/${platform}", - "platforms": [ - "mac-amd64", - "linux-amd64", - "linux-arm64", - "windows-amd64" - ], - "tags": [ - "version:3.22.20211026-geeb4540" - ] - }, - { - "path": "pigweed/third_party/bloaty-embedded/${platform}", - "platforms": [ - "linux-amd64", - "mac-amd64" - ], - "tags": [ - "git_revision:2d87d204057b419f5290f8d38b61b9c2c5b4fb52-2" - ] - }, - { - "path": "infra/3pp/tools/protoc/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "windows-amd64" - ], - "tags": [ - "version:2@3.17.3" - ] - }, - { - "_comment": "Until there's an arm64 Mac version, get the amd64 Mac version on arm64 Macs.", - "path": "infra/3pp/tools/protoc/mac-amd64", - "platforms": [ - "mac-arm64" - ], - "tags": [ - "version:2@3.17.3" - ] - }, - { - "path": "fuchsia/third_party/clang/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "windows-amd64" - ], - "tags": [ - "git_revision:1aa59ff2f789776ebfa2d4b315fd3ea589652b4a" - ] - }, - { - "path": "infra/3pp/tools/go/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "mac-arm64", - "windows-amd64" - ], - "tags": [ - "version:2@1.17.6" - ] - }, - { - "path": "pigweed/third_party/protoc-gen-go/${platform}", - "platforms": [ - "linux-amd64", - "mac-amd64", - "windows-amd64" - ], - "tags": [ - "version:1.3.2" - ] - }, - { - "_comment": "TODO(pwbug/66) Put openocd in cipd for Windows.", - "path": "pigweed/third_party/openocd/${platform}", - "platforms": [ - "linux-amd64", - "mac-amd64" - ], - "tags": [ - "git_revision:e41c0f4906e46d1076ce62a0da5518aa1ca280b8" - ] - }, - { - "path": "pigweed/third_party/mingw64-x86_64-win32-seh/${platform}", - "platforms": [ - "windows-amd64" - ], - "tags": [ - "version:10.2.0-11" - ] - }, - { - "path": "pigweed/host_tools/${platform}", - "platforms": [ - "linux-amd64", - "mac-amd64", - "windows-amd64" - ], - "tags": [ - "git_revision:12fa59f95bf64147252f4fc60f923cfae8c75dc9" - ] - }, - { - "path": "infra/rbe/client/${platform}", - "platforms": [ - "linux-amd64", - "windows-amd64" - ], - "tags": [ - "re_client_version:0.41.4.3f0d8bb" - ] - }, - { - "path": "fuchsia/third_party/qemu/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64" - ], - "tags": [ - "git_revision:d80b96f149ebdd2ee530da1cc6245e0888080e7e" - ] - }, - { - "path": "fuchsia/third_party/kythe", - "platforms": [ - "linux-amd64" - ], - "subdir": "kythe", - "tags": [ - "version:1.0.3" - ] - }, - { - "path": "fuchsia/third_party/kythe-libs/${platform}", - "platforms": [ - "linux-amd64" - ], - "subdir": "kythe", - "tags": [ - "version:2020-08-05" - ] - }, - { - "path": "infra/3pp/tools/renode/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64" - ], - "tags": [ - "version:2@renode-1.11.0+20210306gite7897c1" - ] - }, - { - "path": "infra/3pp/tools/buildifier/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "mac-arm64", - "windows-amd64" - ], - "tags": [ - "version:2@4.2.5" - ] - } -] +{ + "packages": [ + { + "path": "gn/gn/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "mac-arm64", + "windows-amd64" + ], + "tags": [ + "git_revision:0725d7827575b239594fbc8fd5192873a1d62f44" + ] + }, + { + "path": "infra/3pp/tools/ninja/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "mac-arm64", + "windows-amd64" + ], + "tags": [ + "version:2@1.10.2" + ] + }, + { + "path": "fuchsia/third_party/cmake/${platform}", + "platforms": [ + "mac-amd64", + "linux-amd64", + "linux-arm64", + "windows-amd64" + ], + "tags": [ + "version:3.22.20211026-geeb4540" + ] + }, + { + "path": "pigweed/third_party/bloaty-embedded/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64" + ], + "tags": [ + "git_revision:2d87d204057b419f5290f8d38b61b9c2c5b4fb52-2" + ] + }, + { + "path": "infra/3pp/tools/protoc/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], + "tags": [ + "version:2@3.17.3" + ] + }, + { + "_comment": "Until there's an arm64 Mac version, get the amd64 Mac version on arm64 Macs.", + "path": "infra/3pp/tools/protoc/mac-amd64", + "platforms": [ + "mac-arm64" + ], + "tags": [ + "version:2@3.17.3" + ] + }, + { + "path": "fuchsia/third_party/clang/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], + "tags": [ + "git_revision:1aa59ff2f789776ebfa2d4b315fd3ea589652b4a" + ] + }, + { + "path": "infra/3pp/tools/go/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "mac-arm64", + "windows-amd64" + ], + "tags": [ + "version:2@1.17.6" + ] + }, + { + "path": "pigweed/third_party/protoc-gen-go/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64", + "windows-amd64" + ], + "tags": [ + "version:1.3.2" + ] + }, + { + "_comment": "TODO(pwbug/66) Put openocd in cipd for Windows.", + "path": "pigweed/third_party/openocd/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64" + ], + "tags": [ + "git_revision:e41c0f4906e46d1076ce62a0da5518aa1ca280b8" + ] + }, + { + "path": "pigweed/third_party/mingw64-x86_64-win32-seh/${platform}", + "platforms": [ + "windows-amd64" + ], + "tags": [ + "version:10.2.0-11" + ] + }, + { + "path": "pigweed/host_tools/${platform}", + "platforms": [ + "linux-amd64", + "mac-amd64", + "windows-amd64" + ], + "tags": [ + "git_revision:12fa59f95bf64147252f4fc60f923cfae8c75dc9" + ] + }, + { + "path": "infra/rbe/client/${platform}", + "platforms": [ + "linux-amd64", + "windows-amd64" + ], + "tags": [ + "re_client_version:0.41.4.3f0d8bb" + ] + }, + { + "path": "fuchsia/third_party/qemu/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64" + ], + "tags": [ + "git_revision:d80b96f149ebdd2ee530da1cc6245e0888080e7e" + ] + }, + { + "path": "fuchsia/third_party/kythe", + "platforms": [ + "linux-amd64" + ], + "subdir": "kythe", + "tags": [ + "version:1.0.3" + ] + }, + { + "path": "fuchsia/third_party/kythe-libs/${platform}", + "platforms": [ + "linux-amd64" + ], + "subdir": "kythe", + "tags": [ + "version:2020-08-05" + ] + }, + { + "path": "infra/3pp/tools/renode/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64" + ], + "tags": [ + "version:2@renode-1.11.0+20210306gite7897c1" + ] + }, + { + "path": "infra/3pp/tools/buildifier/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "mac-arm64", + "windows-amd64" + ], + "tags": [ + "version:2@4.2.5" + ] + } + ] +} diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/python.json b/pw_env_setup/py/pw_env_setup/cipd_setup/python.json index 8a71eb47f..6ecbff5f9 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/python.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/python.json @@ -1,25 +1,27 @@ -[ - { - "_comment": "When bumping the minor version (e.g., to 3.9.x) also update env_setup/virtualenv/init.py to check for the new version.", - "path": "infra/3pp/tools/cpython3/${platform}", - "platforms": [ - "linux-amd64", - "linux-arm64", - "mac-amd64", - "windows-amd64" - ], - "tags": [ - "version:2@3.9.5.chromium.19" - ] - }, - { - "_comment": "TODO(pwbug/455) Use 3.9 for Macs too.", - "path": "infra/3pp/tools/cpython3/${platform}", - "platforms": [ - "mac-arm64" - ], - "tags": [ - "version:2@3.8.10.chromium.21" - ] - } -] +{ + "packages": [ + { + "_comment": "When bumping the minor version (e.g., to 3.9.x) also update env_setup/virtualenv/init.py to check for the new version.", + "path": "infra/3pp/tools/cpython3/${platform}", + "platforms": [ + "linux-amd64", + "linux-arm64", + "mac-amd64", + "windows-amd64" + ], + "tags": [ + "version:2@3.9.5.chromium.19" + ] + }, + { + "_comment": "TODO(pwbug/455) Use 3.9 for Macs too.", + "path": "infra/3pp/tools/cpython3/${platform}", + "platforms": [ + "mac-arm64" + ], + "tags": [ + "version:2@3.8.10.chromium.21" + ] + } + ] +} diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py index aba33fca0..380ff5742 100755 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py @@ -68,11 +68,9 @@ def check_auth(cipd, package_files, spin): with open(package_file, 'r') as ins: # This is an expensive RPC, so only check the first few entries # in each file. - for i, entry in enumerate(json.load(ins)): + for i, entry in enumerate(json.load(ins).get('packages', ())): if i >= 3: break - if not isinstance(entry, dict): - continue parts = entry['path'].split('/') while '${' in parts[-1]: parts.pop(-1) @@ -193,19 +191,9 @@ def all_package_files(env_vars, package_files): result.append(package_file) with open(package_file, 'r') as ins: - entries = json.load(ins) - - # TODO(pwbug/599) Always assume isinstance(entries, dict). - if isinstance(entries, dict): - entries = entries.get('included_files', ()) + entries = json.load(ins).get('included_files', ()) for entry in entries: - # If there's an entry that's not a string it's a package and can be - # ignored here. - # TODO(pwbug/599) Don't ignore non-str entries. - if isinstance(entry, dict): - continue - entry = os.path.join(os.path.dirname(package_file), entry) if entry not in result and entry not in to_process: @@ -216,11 +204,7 @@ def all_package_files(env_vars, package_files): def write_ensure_file(package_file, ensure_file): with open(package_file, 'r') as ins: - packages = json.load(ins) - - # TODO(pwbug/599) Always assume isinstance(entries, dict). - if isinstance(packages, dict): - packages = packages.get('packages', ()) + packages = json.load(ins).get('packages', ()) with open(ensure_file, 'w') as outs: outs.write('$VerifiedPlatform linux-amd64\n' @@ -228,12 +212,6 @@ def write_ensure_file(package_file, ensure_file): '$ParanoidMode CheckPresence\n') for pkg in packages: - # Strings in package files are references to other package files. - # Ignore them here. - # TODO(pwbug/599) Error on non-dict entries. - if not isinstance(pkg, dict): - continue - # If this is a new-style package manifest platform handling must # be done here instead of by the cipd executable. if 'platforms' in pkg and _platform() not in pkg['platforms']: |