diff options
Diffstat (limited to 'pkg/private')
| -rw-r--r-- | pkg/private/BUILD | 110 | ||||
| -rw-r--r-- | pkg/private/__init__.py | 0 | ||||
| -rw-r--r-- | pkg/private/archive.py | 89 | ||||
| -rw-r--r-- | pkg/private/build_info.py | 37 | ||||
| -rw-r--r-- | pkg/private/deb/BUILD | 68 | ||||
| -rw-r--r-- | pkg/private/deb/deb.bzl | 381 | ||||
| -rw-r--r-- | pkg/private/deb/make_deb.py | 422 | ||||
| -rw-r--r-- | pkg/private/helpers.py | 89 | ||||
| -rw-r--r-- | pkg/private/install.py.tpl | 203 | ||||
| -rw-r--r-- | pkg/private/make_starlark_library.bzl | 35 | ||||
| -rw-r--r-- | pkg/private/manifest.py | 79 | ||||
| -rw-r--r-- | pkg/private/pkg_files.bzl | 610 | ||||
| -rw-r--r-- | pkg/private/tar/BUILD | 71 | ||||
| -rw-r--r-- | pkg/private/tar/build_tar.py | 459 | ||||
| -rw-r--r-- | pkg/private/tar/tar.bzl | 372 | ||||
| -rw-r--r-- | pkg/private/tar/tar_writer.py | 338 | ||||
| -rw-r--r-- | pkg/private/util.bzl | 86 | ||||
| -rw-r--r-- | pkg/private/zip/BUILD | 60 | ||||
| -rw-r--r-- | pkg/private/zip/__init__.py | 0 | ||||
| -rw-r--r-- | pkg/private/zip/build_zip.py | 305 | ||||
| -rw-r--r-- | pkg/private/zip/zip.bzl | 185 |
21 files changed, 0 insertions, 3999 deletions
diff --git a/pkg/private/BUILD b/pkg/private/BUILD deleted file mode 100644 index 8f3ca4c..0000000 --- a/pkg/private/BUILD +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""rules_pkg internal code. - -All interfaces are subject to change at any time. -""" - -load("@rules_python//python:defs.bzl", "py_library") - -package(default_applicable_licenses = ["//:license"]) - -filegroup( - name = "standard_package", - srcs = [ - "BUILD", - "install.py.tpl", - ] + glob([ - "*.bzl", - "*.py", - ]), - visibility = [ - "//distro:__pkg__", - "//pkg:__pkg__", - ], -) - -exports_files( - glob([ - "*.bzl", - ]), - visibility = [ - "//distro:__pkg__", - "//doc_build:__pkg__", - "//pkg:__pkg__", - ], -) - -# pkg_install's template script file -exports_files( - ["install.py.tpl"], - visibility = ["//visibility:public"], -) - -config_setting( - name = "private_stamp_detect", - values = {"stamp": "1"}, - # When --incompatible_config_setting_private_default_visibility is set, this fails unless this is public. - # TODO: refactor to clear up confusion that this is a "private" target with public access. - visibility = ["//visibility:public"], -) - -py_library( - name = "build_info", - srcs = [ - "build_info.py", - ], - imports = ["../.."], - srcs_version = "PY3", - visibility = [ - "//:__subpackages__", - "//tests:__pkg__", - ], -) - -py_library( - name = "archive", - srcs = [ - "__init__.py", - "archive.py", - ], - imports = ["../.."], - srcs_version = "PY3", - visibility = [ - "//:__subpackages__", - "//tests:__subpackages__", - ], -) - -py_library( - name = "helpers", - srcs = [ - "__init__.py", - "helpers.py", - ], - imports = ["../.."], - srcs_version = "PY3", - visibility = [ - "//:__subpackages__", - "//tests:__pkg__", - ], -) - -py_library( - name = "manifest", - srcs = ["manifest.py"], - imports = ["../.."], - srcs_version = "PY3", - visibility = ["//visibility:public"], -) diff --git a/pkg/private/__init__.py b/pkg/private/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/pkg/private/__init__.py +++ /dev/null diff --git a/pkg/private/archive.py b/pkg/private/archive.py deleted file mode 100644 index f88db97..0000000 --- a/pkg/private/archive.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Archive reader library for the .deb file testing.""" - -import io -import os - -class SimpleArReader(object): - """A simple AR file reader. - - This enable to read AR file (System V variant) as described - in https://en.wikipedia.org/wiki/Ar_(Unix). - - The standard usage of this class is: - - with SimpleArReader(filename) as ar: - nextFile = ar.next() - while nextFile: - print('This archive contains', nextFile.filename) - nextFile = ar.next() - - Upon error, this class will raise a ArError exception. - """ - - class ArError(Exception): - pass - - class SimpleArFileEntry(object): - """Represent one entry in a AR archive. - - Attributes: - filename: the filename of the entry, as described in the archive. - timestamp: the timestamp of the file entry. - owner_id: numeric id of the user and group owning the file. - group_id: numeric id of the user and group owning the file. - mode: unix permission mode of the file - size: size of the file - data: the content of the file. - """ - - def __init__(self, f): - self.filename = f.read(16).decode('utf-8').strip() - if self.filename.endswith('/'): # SysV variant - self.filename = self.filename[:-1] - self.timestamp = int(f.read(12).strip()) - self.owner_id = int(f.read(6).strip()) - self.group_id = int(f.read(6).strip()) - self.mode = int(f.read(8).strip(), 8) - self.size = int(f.read(10).strip()) - pad = f.read(2) - if pad != b'\x60\x0a': - raise SimpleArReader.ArError('Invalid AR file header') - self.data = f.read(self.size) - - MAGIC_STRING = b'!<arch>\n' - - def __init__(self, filename): - self.filename = filename - - def __enter__(self): - self.f = open(self.filename, 'rb') - if self.f.read(len(self.MAGIC_STRING)) != self.MAGIC_STRING: - raise self.ArError('Not a ar file: ' + self.filename) - return self - - def __exit__(self, t, v, traceback): - self.f.close() - - def next(self): - """Read the next file. Returns None when reaching the end of file.""" - # AR sections are two bit aligned using new lines. - if self.f.tell() % 2 != 0: - self.f.read(1) - # An AR sections is at least 60 bytes. Some file might contains garbage - # bytes at the end of the archive, ignore them. - if self.f.tell() > os.fstat(self.f.fileno()).st_size - 60: - return None - return self.SimpleArFileEntry(self.f) diff --git a/pkg/private/build_info.py b/pkg/private/build_info.py deleted file mode 100644 index 6780623..0000000 --- a/pkg/private/build_info.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Get BUILD_TIMESTAMP.""" - - -def get_timestamp(volatile_status_file): - """Get BUILD_TIMESTAMP as an integer. - - Reads a file of "name<space>value" pairs and returns the value - of the BUILD_TIMESTAMP. The file should be in the workspace status - format: https://docs.bazel.build/versions/master/user-manual.html#workspace_status - - Args: - volatile_status_file: path to input file. Typically ctx.version_file.path. - Returns: - int: value of BUILD_TIMESTAMP - Exceptions: - Exception: Raised if there is no BUILD_TIMESTAMP or if it is not a number. - """ - with open(volatile_status_file, 'r') as status_f: - for line in status_f: - parts = line.strip().split(' ') - if len(parts) > 1 and parts[0] == 'BUILD_TIMESTAMP': - return int(parts[1]) - raise Exception( - "Invalid status file <%s>. Expected to find BUILD_TIMESTAMP" % volatile_status_file) diff --git a/pkg/private/deb/BUILD b/pkg/private/deb/BUILD deleted file mode 100644 index 646877b..0000000 --- a/pkg/private/deb/BUILD +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""rules_pkg internal code. - -All interfaces are subject to change at any time. -""" - -load("@rules_python//python:defs.bzl", "py_binary", "py_library") - -package(default_applicable_licenses = ["//:license"]) - -filegroup( - name = "standard_package", - srcs = [ - "BUILD", - ] + glob([ - "*.bzl", - "*.py", - ]), - visibility = [ - "//distro:__pkg__", - "//pkg:__pkg__", - ], -) - -exports_files( - glob([ - "*.bzl", - ]), - visibility = [ - "//distro:__pkg__", - "//doc_build:__pkg__", - "//pkg:__pkg__", - ], -) - -py_binary( - name = "make_deb", - srcs = ["make_deb.py"], - imports = ["../../.."], - python_version = "PY3", - visibility = ["//visibility:public"], - deps = [ - "//pkg/private:helpers", - ], -) - -py_library( - name = "make_deb_lib", - srcs = ["make_deb.py"], - imports = ["../../.."], - srcs_version = "PY3", - visibility = ["//tests/deb:__pkg__"], - deps = [ - "//pkg/private:helpers", - ], -) diff --git a/pkg/private/deb/deb.bzl b/pkg/private/deb/deb.bzl deleted file mode 100644 index 5facdd7..0000000 --- a/pkg/private/deb/deb.bzl +++ /dev/null @@ -1,381 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Rule for creating Debian packages.""" - -load("//pkg:providers.bzl", "PackageVariablesInfo") -load("//pkg/private:util.bzl", "setup_output_files") - -_tar_filetype = [".tar", ".tar.gz", ".tgz", ".tar.bz2", "tar.xz", "tar.zst"] - -def _pkg_deb_impl(ctx): - """The implementation for the pkg_deb rule.""" - - package_file_name = ctx.attr.package_file_name - if not package_file_name: - package_file_name = "%s_%s_%s.deb" % ( - ctx.attr.package, - ctx.attr.version, - ctx.attr.architecture, - ) - - outputs, output_file, output_name = setup_output_files( - ctx, - package_file_name = package_file_name, - ) - - changes_file = ctx.actions.declare_file(output_name.rsplit(".", 1)[0] + ".changes") - outputs.append(changes_file) - - files = [ctx.file.data] - args = [ - "--output=" + output_file.path, - "--changes=" + changes_file.path, - "--data=" + ctx.file.data.path, - "--package=" + ctx.attr.package, - "--maintainer=" + ctx.attr.maintainer, - ] - - # Version and description can be specified by a file or inlined - if ctx.attr.architecture_file: - if ctx.attr.architecture != "all": - fail("Both architecture and architecture_file attributes were specified") - args += ["--architecture=@" + ctx.file.architecture_file.path] - files += [ctx.file.architecture_file] - else: - args += ["--architecture=" + ctx.attr.architecture] - - if ctx.attr.preinst: - args += ["--preinst=@" + ctx.file.preinst.path] - files += [ctx.file.preinst] - if ctx.attr.postinst: - args += ["--postinst=@" + ctx.file.postinst.path] - files += [ctx.file.postinst] - if ctx.attr.prerm: - args += ["--prerm=@" + ctx.file.prerm.path] - files += [ctx.file.prerm] - if ctx.attr.postrm: - args += ["--postrm=@" + ctx.file.postrm.path] - files += [ctx.file.postrm] - if ctx.attr.config: - args += ["--config=@" + ctx.file.config.path] - files += [ctx.file.config] - if ctx.attr.templates: - args += ["--templates=@" + ctx.file.templates.path] - files += [ctx.file.templates] - if ctx.attr.triggers: - args += ["--triggers=@" + ctx.file.triggers.path] - files += [ctx.file.triggers] - - # Conffiles can be specified by a file or a string list - if ctx.attr.conffiles_file: - if ctx.attr.conffiles: - fail("Both conffiles and conffiles_file attributes were specified") - args += ["--conffile=@" + ctx.file.conffiles_file.path] - files += [ctx.file.conffiles_file] - elif ctx.attr.conffiles: - args += ["--conffile=%s" % cf for cf in ctx.attr.conffiles] - - # Version and description can be specified by a file or inlined - if ctx.attr.version_file: - if ctx.attr.version: - fail("Both version and version_file attributes were specified") - args += ["--version=@" + ctx.file.version_file.path] - files += [ctx.file.version_file] - elif ctx.attr.version: - args += ["--version=" + ctx.attr.version] - else: - fail("Neither version_file nor version attribute was specified") - - if ctx.attr.description_file: - if ctx.attr.description: - fail("Both description and description_file attributes were specified") - args += ["--description=@" + ctx.file.description_file.path] - files += [ctx.file.description_file] - elif ctx.attr.description: - args += ["--description=" + ctx.attr.description] - else: - fail("Neither description_file nor description attribute was specified") - - if ctx.attr.changelog: - args += ["--changelog=@" + ctx.file.changelog.path] - files += [ctx.file.changelog] - - # Built using can also be specified by a file or inlined (but is not mandatory) - if ctx.attr.built_using_file: - if ctx.attr.built_using: - fail("Both build_using and built_using_file attributes were specified") - args += ["--built_using=@" + ctx.file.built_using_file.path] - files += [ctx.file.built_using_file] - elif ctx.attr.built_using: - args += ["--built_using=" + ctx.attr.built_using] - - if ctx.attr.depends_file: - if ctx.attr.depends: - fail("Both depends and depends_file attributes were specified") - args += ["--depends=@" + ctx.file.depends_file.path] - files += [ctx.file.depends_file] - elif ctx.attr.depends: - args += ["--depends=" + d for d in ctx.attr.depends] - - if ctx.attr.priority: - args += ["--priority=" + ctx.attr.priority] - if ctx.attr.section: - args += ["--section=" + ctx.attr.section] - if ctx.attr.homepage: - args += ["--homepage=" + ctx.attr.homepage] - if ctx.attr.license: - args += ["--license=" + ctx.attr.license] - - args += ["--distribution=" + ctx.attr.distribution] - args += ["--urgency=" + ctx.attr.urgency] - args += ["--suggests=" + d for d in ctx.attr.suggests] - args += ["--enhances=" + d for d in ctx.attr.enhances] - args += ["--conflicts=" + d for d in ctx.attr.conflicts] - args += ["--breaks=" + d for d in ctx.attr.breaks] - args += ["--pre_depends=" + d for d in ctx.attr.predepends] - args += ["--recommends=" + d for d in ctx.attr.recommends] - args += ["--replaces=" + d for d in ctx.attr.replaces] - args += ["--provides=" + d for d in ctx.attr.provides] - - ctx.actions.run( - mnemonic = "MakeDeb", - executable = ctx.executable._make_deb, - arguments = args, - inputs = files, - outputs = [output_file, changes_file], - env = { - "LANG": "en_US.UTF-8", - "LC_CTYPE": "UTF-8", - "PYTHONIOENCODING": "UTF-8", - "PYTHONUTF8": "1", - }, - ) - output_groups = { - "out": [ctx.outputs.out], - "deb": [output_file], - "changes": [changes_file], - } - return [ - OutputGroupInfo(**output_groups), - DefaultInfo( - files = depset([output_file]), - runfiles = ctx.runfiles(files = outputs), - ), - ] - -# A rule for creating a deb file, see README.md -pkg_deb_impl = rule( - implementation = _pkg_deb_impl, - doc = """ - Create a Debian package. - - This rule produces 2 artifacts: a .deb and a .changes file. The DefaultInfo will - include both. If you need downstream rule to specificially depend on only the .deb or - .changes file then you can use `filegroup` to select distinct output groups. - - **OutputGroupInfo** - - `out` the Debian package or a symlink to the actual package. - - `deb` the package with any precise file name created with `package_file_name`. - - `changes` the .changes file. - """, - attrs = { - # @unsorted-dict-items - "data": attr.label( - doc = """A tar file that contains the data for the debian package.""", - mandatory = True, - allow_single_file = _tar_filetype, - ), - "package": attr.string( - doc = "The name of the package", - mandatory = True, - ), - "architecture": attr.string( - default = "all", - doc = """Package architecture. Must not be used with architecture_file.""", - ), - "architecture_file": attr.label( - doc = """File that contains the package architecture. - Must not be used with architecture.""", - allow_single_file = True, - ), - "maintainer": attr.string( - doc = "The maintainer of the package.", - mandatory = True, - ), - "version": attr.string( - doc = """Package version. Must not be used with `version_file`.""", - ), - "version_file": attr.label( - doc = """File that contains the package version. - Must not be used with `version`.""", - allow_single_file = True, - ), - "config": attr.label( - doc = """config file used for debconf integration. - See https://www.debian.org/doc/debian-policy/ch-binary.html#prompting-in-maintainer-scripts.""", - allow_single_file = True, - ), - "changelog": attr.label( - doc = """The package changelog. - See https://www.debian.org/doc/debian-policy/ch-source.html#s-dpkgchangelog.""", - allow_single_file = True - ), - "description": attr.string( - doc = """The package description. Must not be used with `description_file`.""", - ), - "description_file": attr.label( - doc = """The package description. Must not be used with `description`.""", - allow_single_file = True - ), - "distribution": attr.string( - doc = """"distribution: See http://www.debian.org/doc/debian-policy.""", - default = "unstable", - ), - "urgency": attr.string( - doc = """"urgency: See http://www.debian.org/doc/debian-policy.""", - default = "medium", - ), - "preinst": attr.label( - doc = """"The pre-install script for the package. - See http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html.""", - allow_single_file = True, - ), - "postinst": attr.label( - doc = """The post-install script for the package. - See http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html.""", - allow_single_file = True, - ), - "prerm": attr.label( - doc = """The pre-remove script for the package. - See http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html.""", - allow_single_file = True, - ), - "postrm": attr.label( - doc = """The post-remove script for the package. - See http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html.""", - allow_single_file = True, - ), - "templates": attr.label( - doc = """templates file used for debconf integration. - See https://www.debian.org/doc/debian-policy/ch-binary.html#prompting-in-maintainer-scripts.""", - allow_single_file = True, - ), - "triggers": attr.label( - doc = """triggers file for configuring installation events exchanged by packages. - See https://wiki.debian.org/DpkgTriggers.""", - allow_single_file = True, - ), - "built_using": attr.string( - doc="""The tool that were used to build this package provided either inline (with built_using) or from a file (with built_using_file).""" - ), - "built_using_file": attr.label( - doc="""The tool that were used to build this package provided either inline (with built_using) or from a file (with built_using_file).""", - allow_single_file = True - ), - "conffiles": attr.string_list( - doc = """The list of conffiles or a file containing one conffile per line. Each item is an absolute path on the target system where the deb is installed. -See https://www.debian.org/doc/debian-policy/ch-files.html#s-config-files.""", - default = [], - ), - "conffiles_file": attr.label( - doc = """The list of conffiles or a file containing one conffile per line. Each item is an absolute path on the target system where the deb is installed. -See https://www.debian.org/doc/debian-policy/ch-files.html#s-config-files.""", - allow_single_file = True, - ), - "priority": attr.string( - doc = """The priority of the package. - See http://www.debian.org/doc/debian-policy/ch-archive.html#s-priorities.""", - ), - "section": attr.string( - doc = """The section of the package. - See http://www.debian.org/doc/debian-policy/ch-archive.html#s-subsections.""", - ), - "homepage": attr.string(doc = """The homepage of the project."""), - "license": attr.string(doc = """The license of the project."""), - - "breaks": attr.string_list( - doc = """See http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps.""", - default = [], - ), - "conflicts": attr.string_list( - doc = """See http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps.""", - default = [], - ), - "depends": attr.string_list( - doc = """See http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps.""", - default = [], - ), - "depends_file": attr.label( - doc = """File that contains a list of package dependencies. Must not be used with `depends`. - See http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps.""", - allow_single_file = True, - ), - "enhances": attr.string_list( - doc = """See http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps.""", - default = [], - ), - "provides": attr.string_list( - doc = """See http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps.""", - default = [], - ), - "predepends": attr.string_list( - doc = """See http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps.""", - default = [], - ), - "recommends": attr.string_list( - doc = """See http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps.""", - default = [], - ), - "replaces": attr.string_list( - doc = """See http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps.""", - default = [], - ), - "suggests": attr.string_list( - doc = """See http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps.""", - default = [], - ), - - # Common attributes - "out": attr.output( - doc = """See [Common Attributes](#out)""", - mandatory = True - ), - "package_file_name": attr.string( - doc = """See [Common Attributes](#package_file_name). - Default: "{package}-{version}-{architecture}.deb""", - ), - "package_variables": attr.label( - doc = """See [Common Attributes](#package_variables)""", - providers = [PackageVariablesInfo], - ), - - # Implicit dependencies. - "_make_deb": attr.label( - default = Label("//pkg/private/deb:make_deb"), - cfg = "exec", - executable = True, - allow_files = True, - ), - }, -) - -def pkg_deb(name, out = None, **kwargs): - """@wraps(pkg_deb_impl).""" - if not out: - out = name + ".deb" - pkg_deb_impl( - name = name, - out = out, - **kwargs - ) diff --git a/pkg/private/deb/make_deb.py b/pkg/private/deb/make_deb.py deleted file mode 100644 index 9d10c4b..0000000 --- a/pkg/private/deb/make_deb.py +++ /dev/null @@ -1,422 +0,0 @@ -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""A simple cross-platform helper to create a debian package.""" - -import argparse -from enum import Enum -import gzip -import hashlib -import io -import os -import sys -import tarfile -import textwrap -import time - -if sys.version_info < (3, 7): - from collections import OrderedDict -else: - OrderedDict = dict - -from pkg.private import helpers - -Multiline = Enum('Multiline', ['NO', 'YES', 'YES_ADD_NEWLINE']) - -# list of debian fields : (name, mandatory, is_multiline[, default]) -# see http://www.debian.org/doc/debian-policy/ch-controlfields.html - -DEBIAN_FIELDS = [ - ('Package', True, False), - ('Version', True, False), - ('Section', False, False, 'contrib/devel'), - ('Priority', False, False, 'optional'), - ('Architecture', False, False, 'all'), - ('Depends', False, False, []), - ('Recommends', False, False, []), - ('Replaces', False, False, []), - ('Suggests', False, False, []), - ('Enhances', False, False, []), - ('Conflicts', False, False, []), - ('Breaks', False, False, []), - ('Pre-Depends', False, False, []), - ('Provides', False, False, []), - ('Installed-Size', False, False), - ('Maintainer', True, False), - ('Description', True, True), - ('Homepage', False, False), - ('License', False, False), - ('Built-Using', False, False, None), - ('Distribution', False, False, 'unstable'), - ('Urgency', False, False, 'medium'), -] - -# size of chunks for copying package content to final .deb file -# This is a wild guess, but I am not convinced of the value of doing much work -# to tune it. -_COPY_CHUNK_SIZE = 1024 * 32 - - -def AddControlFlags(parser): - """Creates a flag for each of the control file fields.""" - for field in DEBIAN_FIELDS: - flag_name = '--' + field[0].replace('-', '_').lower() - msg = 'The value for the %s content header entry.' % field[0] - required = field[1] - if len(field) > 3: - default = field[3] - if isinstance(field[3], list): - parser.add_argument(flag_name, action='append', default=default, - required=required, help=msg) - else: - parser.add_argument(flag_name, default=default, required=required, - help=msg) - else: - parser.add_argument(flag_name, required=required, help=msg) - - -def ConvertToFileLike(content, content_len, converter): - if content_len < 0: - content_len = len(content) - content = converter(content) - return content_len, content - - -def AddArFileEntry(fileobj, filename, - content='', content_len=-1, timestamp=0, - owner_id=0, group_id=0, mode=0o644): - """Add a AR file entry to fileobj.""" - # If we got the content as a string, turn it into a file like thing. - if isinstance(content, (str, bytes)): - content_len, content = ConvertToFileLike(content, content_len, io.BytesIO) - inputs = [ - (filename + '/').ljust(16), # filename (SysV) - str(timestamp).ljust(12), # timestamp - str(owner_id).ljust(6), # owner id - str(group_id).ljust(6), # group id - str(oct(mode)).replace('0o', '0').ljust(8), # mode - str(content_len).ljust(10), # size - '\x60\x0a', # end of file entry - ] - for i in inputs: - fileobj.write(i.encode('ascii')) - size = 0 - while True: - data = content.read(_COPY_CHUNK_SIZE) - if not data: - break - size += len(data) - fileobj.write(data) - if size % 2 != 0: - fileobj.write(b'\n') # 2-byte alignment padding - - -def MakeDebianControlField(name: str, value: str, multiline:Multiline=Multiline.NO) -> str: - """Add a field to a debian control file. - - https://www.debian.org/doc/debian-policy/ch-controlfields.html#syntax-of-control-files - - Args: - name: Control field name - value: Value for that - """ - if isinstance(value, bytes): - value = value.decode('utf-8') - if isinstance(value, list): - value = u', '.join(value) - value = value.rstrip() - if multiline == Multiline.NO: - value = value.strip() - if '\n' in value: - raise ValueError( - '\\n is not allowed in simple control fields (%s)' % value) - - lines = value.split('\n') - i = 0 - if multiline != Multiline.YES_ADD_NEWLINE: - result = name + ': ' + lines[i].strip() + '\n' - i = 1 - else: - result = name + ':\n' - for line in lines[i:]: - if not line.startswith(' '): - result += ' ' - result += line - result += '\n' - return result - - -def CreateDebControl(extrafiles=None, **kwargs): - """Create the control.tar.gz file.""" - # create the control file - controlfile = u'' - for values in DEBIAN_FIELDS: - fieldname = values[0] - mandatory = values[1] - multiline = Multiline.YES if values[2] else Multiline.NO - key = fieldname[0].lower() + fieldname[1:].replace('-', '') - if mandatory or (key in kwargs and kwargs[key]): - controlfile += MakeDebianControlField(fieldname, kwargs[key], multiline) - # Create the control.tar file - tar = io.BytesIO() - with gzip.GzipFile('control.tar.gz', mode='w', fileobj=tar, mtime=0) as gz: - with tarfile.open('control.tar.gz', mode='w', fileobj=gz, - format=tarfile.GNU_FORMAT) as f: - tarinfo = tarfile.TarInfo('./control') - control_file_data = controlfile.encode('utf-8') - tarinfo.size = len(control_file_data) - f.addfile(tarinfo, fileobj=io.BytesIO(control_file_data)) - if extrafiles: - for name, (data, mode) in extrafiles.items(): - tarinfo = tarfile.TarInfo('./' + name) - data_encoded = data.encode('utf-8') - tarinfo.size = len(data_encoded) - tarinfo.mode = mode - f.addfile(tarinfo, fileobj=io.BytesIO(data_encoded)) - control = tar.getvalue() - tar.close() - return control - - -def CreateDeb(output, - data, - preinst=None, - postinst=None, - prerm=None, - postrm=None, - config=None, - templates=None, - triggers=None, - conffiles=None, - changelog=None, - **kwargs): - """Create a full debian package.""" - extrafiles = OrderedDict() - if preinst: - extrafiles['preinst'] = (preinst, 0o755) - if postinst: - extrafiles['postinst'] = (postinst, 0o755) - if prerm: - extrafiles['prerm'] = (prerm, 0o755) - if postrm: - extrafiles['postrm'] = (postrm, 0o755) - if config: - extrafiles['config'] = (config, 0o755) - if templates: - extrafiles['templates'] = (templates, 0o644) - if triggers: - extrafiles['triggers'] = (triggers, 0o644) - if conffiles: - extrafiles['conffiles'] = ('\n'.join(conffiles) + '\n', 0o644) - if changelog: - extrafiles['changelog'] = (changelog, 0o644) - control = CreateDebControl(extrafiles=extrafiles, **kwargs) - - # Write the final AR archive (the deb package) - with open(output, 'wb') as f: - f.write(b'!<arch>\n') # Magic AR header - AddArFileEntry(f, 'debian-binary', b'2.0\n') - AddArFileEntry(f, 'control.tar.gz', control) - # Tries to preserve the extension name - ext = os.path.basename(data).split('.')[-2:] - if len(ext) < 2: - ext = 'tar' - elif ext[1] == 'tgz': - ext = 'tar.gz' - elif ext[1] == 'tar.bzip2': - ext = 'tar.bz2' - else: - ext = '.'.join(ext) - if ext not in ['tar.bz2', 'tar.gz', 'tar.xz', 'tar.lzma', 'tar.zst']: - ext = 'tar' - data_size = os.stat(data).st_size - with open(data, 'rb') as datafile: - AddArFileEntry(f, 'data.' + ext, datafile, content_len=data_size) - - -def GetChecksumsFromFile(filename, hash_fns=None): - """Computes MD5 and/or other checksums of a file. - - Args: - filename: Name of the file. - hash_fns: Mapping of hash functions. - Default is {'md5': hashlib.md5} - - Returns: - Mapping of hash names to hexdigest strings. - { <hashname>: <hexdigest>, ... } - """ - hash_fns = hash_fns or {'md5': hashlib.md5} - checksums = {k: fn() for (k, fn) in hash_fns.items()} - - with open(filename, 'rb') as file_handle: - while True: - buf = file_handle.read(1048576) # 1 MiB - if not buf: - break - for hashfn in checksums.values(): - hashfn.update(buf) - - return {k: fn.hexdigest() for (k, fn) in checksums.items()} - - -def CreateChanges(output, - deb_file, - architecture, - description, - maintainer, - package, - version, - section, - priority, - distribution, - urgency, - timestamp=0): - """Create the changes file.""" - checksums = GetChecksumsFromFile(deb_file, {'md5': hashlib.md5, - 'sha1': hashlib.sha1, - 'sha256': hashlib.sha256}) - debsize = str(os.path.getsize(deb_file)) - deb_basename = os.path.basename(deb_file) - - changesdata = u''.join([ - MakeDebianControlField('Format', '1.8'), - MakeDebianControlField('Date', time.asctime(time.gmtime(timestamp))), - MakeDebianControlField('Source', package), - MakeDebianControlField('Binary', package), - MakeDebianControlField('Architecture', architecture), - MakeDebianControlField('Version', version), - MakeDebianControlField('Distribution', distribution), - MakeDebianControlField('Urgency', urgency), - MakeDebianControlField('Maintainer', maintainer), - MakeDebianControlField('Changed-By', maintainer), - # The description in the changes file is strange - MakeDebianControlField('Description', ( - '%s - %s\n') % ( - package, description.split('\n')[0]), - multiline=Multiline.YES_ADD_NEWLINE), - MakeDebianControlField('Changes', ( - '%s (%s) %s; urgency=%s' - '\n Changes are tracked in revision control.') % ( - package, version, distribution, urgency), - multiline=Multiline.YES_ADD_NEWLINE), - MakeDebianControlField( - 'Files', ' '.join( - [checksums['md5'], debsize, section, priority, deb_basename]), - multiline=Multiline.YES_ADD_NEWLINE), - MakeDebianControlField( - 'Checksums-Sha1', - ' '.join([checksums['sha1'], debsize, deb_basename]), - multiline=Multiline.YES_ADD_NEWLINE), - MakeDebianControlField( - 'Checksums-Sha256', - ' '.join([checksums['sha256'], debsize, deb_basename]), - multiline=Multiline.YES_ADD_NEWLINE) - ]) - with open(output, 'wb') as changes_fh: - changes_fh.write(changesdata.encode('utf-8')) - - -def GetFlagValues(flagvalues): - if flagvalues: - return [helpers.GetFlagValue(f, False) for f in flagvalues] - else: - return None - - -def main(): - parser = argparse.ArgumentParser( - description='Helper for building deb packages') - - parser.add_argument('--output', required=True, - help='The output file, mandatory') - parser.add_argument('--changes', required=True, - help='The changes output file, mandatory.') - parser.add_argument('--data', required=True, - help='Path to the data tarball, mandatory') - parser.add_argument( - '--preinst', - help='The preinst script (prefix with @ to provide a path).') - parser.add_argument( - '--postinst', - help='The postinst script (prefix with @ to provide a path).') - parser.add_argument( - '--prerm', - help='The prerm script (prefix with @ to provide a path).') - parser.add_argument( - '--postrm', - help='The postrm script (prefix with @ to provide a path).') - parser.add_argument( - '--config', - help='The config script (prefix with @ to provide a path).') - parser.add_argument( - '--templates', - help='The templates file (prefix with @ to provide a path).') - parser.add_argument( - '--triggers', - help='The triggers file (prefix with @ to provide a path).') - # see - # https://www.debian.org/doc/manuals/debian-faq/ch-pkg_basics.en.html#s-conffile - parser.add_argument( - '--conffile', action='append', - help='List of conffiles (prefix item with @ to provide a path)') - parser.add_argument( - '--changelog', - help='The changelog file (prefix item with @ to provide a path).') - AddControlFlags(parser) - options = parser.parse_args() - - CreateDeb( - options.output, - options.data, - preinst=helpers.GetFlagValue(options.preinst, False), - postinst=helpers.GetFlagValue(options.postinst, False), - prerm=helpers.GetFlagValue(options.prerm, False), - postrm=helpers.GetFlagValue(options.postrm, False), - config=helpers.GetFlagValue(options.config, False), - templates=helpers.GetFlagValue(options.templates, False), - triggers=helpers.GetFlagValue(options.triggers, False), - conffiles=GetFlagValues(options.conffile), - changelog=helpers.GetFlagValue(options.changelog, False), - package=options.package, - version=helpers.GetFlagValue(options.version), - description=helpers.GetFlagValue(options.description), - maintainer=helpers.GetFlagValue(options.maintainer), - section=options.section, - architecture=helpers.GetFlagValue(options.architecture), - depends=GetFlagValues(options.depends), - suggests=options.suggests, - enhances=options.enhances, - preDepends=options.pre_depends, - recommends=options.recommends, - replaces=options.replaces, - provides=options.provides, - homepage=helpers.GetFlagValue(options.homepage), - license=helpers.GetFlagValue(options.license), - builtUsing=helpers.GetFlagValue(options.built_using), - priority=options.priority, - conflicts=options.conflicts, - breaks=options.breaks, - installedSize=helpers.GetFlagValue(options.installed_size)) - CreateChanges( - output=options.changes, - deb_file=options.output, - architecture=options.architecture, - description=helpers.GetFlagValue(options.description), - maintainer=helpers.GetFlagValue(options.maintainer), package=options.package, - version=helpers.GetFlagValue(options.version), section=options.section, - priority=options.priority, distribution=options.distribution, - urgency=options.urgency) - -if __name__ == '__main__': - main() diff --git a/pkg/private/helpers.py b/pkg/private/helpers.py deleted file mode 100644 index 5147cc2..0000000 --- a/pkg/private/helpers.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2019 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import sys - -def SplitNameValuePairAtSeparator(arg, sep): - """Split a string at the first unquoted occurrence of a character. - - Split the string arg at the first unquoted occurrence of the character c. - Here, in the first part of arg, the backslash is considered the - quoting character indicating that the next character is to be - added literally to the first part, even if it is the split character. - - Args: - arg: the string to be split - sep: the character at which to split - - Returns: - The unquoted string before the separator and the string after the - separator. - """ - head = '' - i = 0 - while i < len(arg): - if arg[i] == sep: - return (head, arg[i + 1:]) - elif arg[i] == '\\': - i += 1 - if i == len(arg): - # dangling quotation symbol - return (head, '') - else: - head += arg[i] - else: - head += arg[i] - i += 1 - # if we leave the loop, the character sep was not found unquoted - return (head, '') - -def GetFlagValue(flagvalue, strip=True): - """Converts a raw flag string to a useable value. - - 1. Expand @filename style flags to the content of filename. - 2. Cope with Python3 strangeness of sys.argv. - sys.argv is not actually proper str types on Unix with Python3 - The bytes of the arg are each directly transcribed to the characters of - the str. It is actually more complex than that, as described in the docs. - https://docs.python.org/3/library/sys.html#sys.argv - https://docs.python.org/3/library/os.html#os.fsencode - https://www.python.org/dev/peps/pep-0383/ - - Args: - flagvalue: (str) raw flag value - strip: (bool) Strip white space. - - Returns: - Python2: unicode - Python3: str - """ - if flagvalue: - if sys.version_info[0] < 3: - # python2 gives us raw bytes in argv. - flagvalue = flagvalue.decode('utf-8') - # assertion: py2: flagvalue is unicode - # assertion: py3: flagvalue is str, but in weird format - if flagvalue[0] == '@': - # Subtle: We do not want to re-encode the value here, because it - # is encoded in the right format for file open operations. - with open(flagvalue[1:], 'rb') as f: - flagvalue = f.read().decode('utf-8') - else: - # convert fs specific encoding back to proper unicode. - if sys.version_info[0] > 2: - flagvalue = os.fsencode(flagvalue).decode('utf-8') - - if strip: - return flagvalue.strip() - return flagvalue diff --git a/pkg/private/install.py.tpl b/pkg/private/install.py.tpl deleted file mode 100644 index b0517f4..0000000 --- a/pkg/private/install.py.tpl +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This template is completed by `pkg_install` to create installation scripts, -# and will not function on its own. See pkg/install.bzl for more details. - -import argparse -import logging -import os -import shutil -import sys - -from pkg.private import manifest - -# Globals used for runfile path manipulation. -# -# These are necessary because runfiles are different when used as a part of -# `bazel build` and `bazel run`. # See also -# https://docs.bazel.build/versions/4.1.0/skylark/rules.html#tools-with-runfiles - -# Bazel's documentation claims these are set when calling `bazel run`, but not other -# modes, like in `build` or `test`. We'll see. -CALLED_FROM_BAZEL_RUN = bool(os.getenv("BUILD_WORKSPACE_DIRECTORY") and - os.getenv("BUILD_WORKING_DIRECTORY")) - -WORKSPACE_NAME = "{WORKSPACE_NAME}" -# This seems to be set when running in `bazel build` or `bazel test` -# TODO(#382): This may not be the case in Windows. -RUNFILE_PREFIX = os.path.join(os.getenv("RUNFILES_DIR"), WORKSPACE_NAME) if os.getenv("RUNFILES_DIR") else None - - -# This is named "NativeInstaller" because it makes use of "native" python -# functionality for installing files that should be cross-platform. -# -# A variant on this might be an installer at least partially based on coreutils. -# Most notably, some filesystems on Linux (and maybe others) support -# copy-on-write functionality that are known to tools like cp(1) and install(1) -# but may not be in the available python runtime. -# -# See also https://bugs.python.org/issue37157. -class NativeInstaller(object): - def __init__(self, default_user=None, default_group=None, destdir=None): - self.default_user = default_user - self.default_group = default_group - self.destdir = destdir - self.entries = [] - - # Logger helper method, may not be necessary or desired - def _subst_destdir(path, self): - return path.replace(self.destdir, "$DESTDIR") - - def _chown_chmod(self, dest, mode, user, group): - if mode: - logging.info("CHMOD %s %s", mode, dest) - os.chmod(dest, int(mode, 8)) - if user or group: - # Ownership can only be changed by sufficiently - # privileged users. - # TODO(nacl): This does not support windows - if hasattr(os, "getuid") and os.getuid() == 0: - logging.info("CHOWN %s:%s %s", user, group, dest) - shutil.chown(dest, user, group) - - def _do_file_copy(self, src, dest, mode, user, group): - logging.info("COPY %s <- %s", dest, src) - shutil.copyfile(src, dest) - - def _do_mkdir(self, dirname, mode, user, group): - logging.info("MKDIR %s %s", mode, dirname) - os.makedirs(dirname, int(mode, 8), exist_ok=True) - - def _do_symlink(self, target, link_name, mode, user, group): - raise NotImplementedError("symlinking not yet supported") - - def _maybe_make_unowned_dir(self, path): - logging.info("MKDIR (unowned) %s", path) - # TODO(nacl): consider default permissions here - # TODO(nacl): consider default ownership here - os.makedirs(path, 0o755, exist_ok=True) - - def _install_file(self, entry): - self._maybe_make_unowned_dir(os.path.dirname(entry.dest)) - self._do_file_copy(entry.src, entry.dest, entry.mode, entry.user, entry.group) - self._chown_chmod(entry.dest, entry.mode, entry.user, entry.group) - - def _install_directory(self, entry): - self._maybe_make_unowned_dir(os.path.dirname(entry.dest)) - self._do_mkdir(entry.dest, entry.mode, entry.user, entry.group) - self._chown_chmod(entry.dest, entry.mode, entry.user, entry.group) - - def _install_treeartifact(self, entry): - logging.info("COPYTREE %s <- %s/**", entry.dest, entry.src) - raise NotImplementedError("treeartifact installation not yet supported") - for root, dirs, files in os.walk(entry.src): - relative_installdir = os.path.join(entry.dest, root) - for d in dirs: - self._maybe_make_unowned_dir(os.path.join(relative_installdir, d)) - - logging.info("COPY_FROM_TREE %s <- %s", entry.dest, entry.src) - logging.info("CHMOD %s %s", entry.mode, entry.dest) - logging.info("CHOWN %s:%s %s", entry.user, entry.group, entry.dest) - - def _install_symlink(self, entry): - raise NotImplementedError("symlinking not yet supported") - logging.info("SYMLINK %s <- %s", entry.dest, entry.link_to) - logging.info("CHMOD %s %s", entry.dest, entry.mode) - logging.info("CHOWN %s.%s %s", entry.dest, entry.user, entry.group) - - def include_manifest_path(self, path): - with open(path, 'r') as fh: - self.include_manifest(fh) - - def include_manifest(self, manifest_fh): - manifest_entries = manifest.read_entries_from(manifest_fh) - - for entry in manifest_entries: - # Swap out the source with the actual "runfile" location if we're - # called as a part of the build rather than "bazel run" - if not CALLED_FROM_BAZEL_RUN and entry.src is not None: - entry.src = os.path.join(RUNFILE_PREFIX, entry.src) - # Prepend the destdir path to all installation paths, if one is - # specified. - if self.destdir is not None: - entry.dest = os.path.join(self.destdir, entry.dest) - self.entries.append(entry) - - def do_the_thing(self): - for entry in self.entries: - if entry.type == manifest.ENTRY_IS_FILE: - self._install_file(entry) - elif entry.type == manifest.ENTRY_IS_LINK: - self._install_symlink(entry) - elif entry.type == manifest.ENTRY_IS_DIR: - self._install_directory(entry) - elif entry.type == manifest.ENTRY_IS_TREE: - self._install_treeartifact(entry) - else: - raise ValueError("Unrecognized entry type '{}'".format(entry.type)) - - -def main(args): - parser = argparse.ArgumentParser( - prog="bazel run -- {TARGET_LABEL}", - description='Installer for bazel target {TARGET_LABEL}', - fromfile_prefix_chars='@', - ) - parser.add_argument('-v', '--verbose', action='count', default=0, - help="Be verbose. Specify multiple times to increase verbosity further") - parser.add_argument('-q', '--quiet', action='store_true', default=False, - help="Be silent, except for errors") - # TODO(nacl): consider supporting DESTDIR=/whatever syntax, like "make - # install". - # - # TODO(nacl): consider removing absolute path restriction, perhaps using - # BUILD_WORKING_DIRECTORY. - parser.add_argument('--destdir', action='store', default=os.getenv("DESTDIR"), - help="Installation root directory (defaults to DESTDIR " - "environment variable). Must be an absolute path.") - - args = parser.parse_args() - - loudness = args.verbose - args.quiet - - if args.quiet: - logging.getLogger().setLevel(logging.ERROR) - elif loudness == 0: - logging.getLogger().setLevel(logging.WARNING) - elif loudness == 1: - logging.getLogger().setLevel(logging.INFO) - else: # loudness >= 2: - logging.getLogger().setLevel(logging.DEBUG) - - installer = NativeInstaller(destdir=args.destdir) - - if not CALLED_FROM_BAZEL_RUN and RUNFILE_PREFIX is None: - logging.critical("RUNFILES_DIR must be set in your environment when this is run as a bazel build tool.") - logging.critical("This is most likely an issue on Windows. See https://github.com/bazelbuild/rules_pkg/issues/387.") - return 1 - - for f in ["{MANIFEST_INCLUSION}"]: - if CALLED_FROM_BAZEL_RUN: - installer.include_manifest_path(f) - else: - installer.include_manifest_path(os.path.join(RUNFILE_PREFIX, f)) - - installer.do_the_thing() - - -if __name__ == "__main__": - exit(main(sys.argv)) diff --git a/pkg/private/make_starlark_library.bzl b/pkg/private/make_starlark_library.bzl deleted file mode 100644 index c6b0dee..0000000 --- a/pkg/private/make_starlark_library.bzl +++ /dev/null @@ -1,35 +0,0 @@ -"""Turn a label_list of mixed sources and bzl_library's into a bzl_library. - -The sources can be anything. Only the ones that end in ".bzl" will be added. -""" - -load("@bazel_skylib//:bzl_library.bzl", "StarlarkLibraryInfo") - -def _make_starlark_library(ctx): - direct = [] - transitive = [] - for src in ctx.attr.srcs: - if StarlarkLibraryInfo in src: - transitive.append(src[StarlarkLibraryInfo]) - else: - for file in src[DefaultInfo].files.to_list(): - if file.path.endswith(".bzl"): - # print(file.path) - direct.append(file) - all_files = depset(direct, transitive = transitive) - return [ - DefaultInfo(files = all_files, runfiles = ctx.runfiles(transitive_files = all_files)), - StarlarkLibraryInfo(srcs = direct, transitive_srcs = all_files), - ] - -starlark_library = rule( - implementation = _make_starlark_library, - attrs = { - "srcs": attr.label_list( - doc = "Any mix of source files. Only .bzl files will be used.", - allow_files = True, - cfg = "exec", - mandatory = True, - ), - }, -) diff --git a/pkg/private/manifest.py b/pkg/private/manifest.py deleted file mode 100644 index 6bfd387..0000000 --- a/pkg/private/manifest.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Common package builder manifest helpers -""" - -import json - - -# These must be kept in sync with the declarations in private/pkg_files.bzl -ENTRY_IS_FILE = "file" # Entry is a file: take content from <src> -ENTRY_IS_LINK = "symlink" # Entry is a symlink: dest -> <src> -ENTRY_IS_DIR = "dir" # Entry is an empty dir -ENTRY_IS_TREE = "tree" # Entry is a tree artifact: take tree from <src> -ENTRY_IS_EMPTY_FILE = "empty-file" # Entry is a an empty file - -class ManifestEntry(object): - """Structured wrapper around rules_pkg-produced manifest entries""" - type: str - dest: str - src: str - mode: str - user: str - group: str - uid: int - gid: int - origin: str = None - - def __init__(self, type, dest, src, mode, user, group, uid = None, gid = None, origin = None): - self.type = type - self.dest = dest - self.src = src - self.mode = mode - self.user = user - self.group = group - self.uid = uid - self.gid = gid - self.origin = origin - - def __repr__(self): - return "ManifestEntry<{}>".format(vars(self)) - -def read_entries_from(fh): - """Return a list of ManifestEntry's from `fh`""" - # Subtle: decode the content with read() rather than in json.load() because - # the load in older python releases (< 3.7?) does not know how to decode. - raw_entries = json.loads(fh.read()) - return [ManifestEntry(**entry) for entry in raw_entries] - -def read_entries_from_file(manifest_path): - """Return a list of ManifestEntry's from the manifest file at `path`""" - with open(manifest_path, 'r', encoding='utf-8') as fh: - return read_entries_from(fh) - -def entry_type_to_string(et): - """Entry type stringifier""" - if et == ENTRY_IS_FILE: - return "file" - elif et == ENTRY_IS_LINK: - return "symlink", - elif et == ENTRY_IS_DIR: - return "directory" - elif et == ENTRY_IS_TREE: - return "tree" - elif et == ENTRY_IS_EMPTY_FILE: - return "empty_file" - else: - raise ValueError("Invalid entry id {}".format(et)) diff --git a/pkg/private/pkg_files.bzl b/pkg/private/pkg_files.bzl deleted file mode 100644 index 86a438b..0000000 --- a/pkg/private/pkg_files.bzl +++ /dev/null @@ -1,610 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Internal functions for processing pkg_file* instances. - -Concepts and terms: - - DestFile: A provider holding the source path, attributes and other - information about a file that should appear in the package. - When attributes are empty in DestFile, we let the package - tool decide their values. - - content map: The map of destination paths to DestFile instances. Note that - several distinct destinations make share the same source path. - Attempting to insert a duplicate entry in the content map is - an error, because it means you are collapsing files together. - We may want to relax this in the future if their DestFiles - are equal. - - manifest: The file which represents the content map. This is generated - by rule implementations and passed to the build_*.py helpers. -""" - -load("//pkg:path.bzl", "compute_data_path", "dest_path") -load( - "//pkg:providers.bzl", - "PackageDirsInfo", - "PackageFilegroupInfo", - "PackageFilesInfo", - "PackageSymlinkInfo", -) - -ENTRY_IS_FILE = "file" # Entry is a file: take content from <src> -ENTRY_IS_LINK = "symlink" # Entry is a symlink: dest -> <src> -ENTRY_IS_DIR = "dir" # Entry is an empty dir -ENTRY_IS_TREE = "tree" # Entry is a tree artifact: take tree from <src> -ENTRY_IS_EMPTY_FILE = "empty-file" # Entry is a an empty file - -_DestFile = provider( - doc = """Information about each destination in the final package.""", - fields = { - "src": "source file", - "mode": "mode, or empty", - "user": "user, or empty", - "group": "group, or empty", - "link_to": "path to link to. src must not be set", - "entry_type": "string. See ENTRY_IS_* values above.", - "origin": "target which added this", - "uid": "uid, or empty", - "gid": "gid, or empty", - }, -) - -def _check_dest(ctx, content_map, dest, src, origin): - old_entry = content_map.get(dest) - if not old_entry: - return - if old_entry.src == src or old_entry.origin == origin: - return - - # TODO(#385): This is insufficient but good enough for now. We should - # compare over all the attributes too. That will detect problems where - # people specify the owner in one place, but another overly broad glob - # brings in the file with a different owner. - if old_entry.src.path != src.path: - msg = "Duplicate output path: <%s>, declared in %s and %s\n SRC: %s" % ( - dest, - origin, - content_map[dest].origin, - src, - ) - if ctx.attr.allow_duplicates_with_different_content: - # buildifier: disable=print - print("WARNING:", msg) - else: - # When we default to this behaviour, we should consider telling - # users the attribute to set to deal with this. - # For now though, let's not, since they've explicitly opted in. - fail(msg) - -def _merge_attributes(info, mode, user, group, uid, gid): - if hasattr(info, "attributes"): - attrs = info.attributes - mode = attrs.get("mode") or mode - user = attrs.get("user") or user - group = attrs.get("group") or group - - new_uid = attrs.get("uid") - if new_uid != None: - uid = new_uid - new_gid = attrs.get("gid") - if new_gid != None: - gid = new_gid - return (mode, user, group, uid, gid) - -def _process_pkg_dirs(ctx, content_map, pkg_dirs_info, origin, default_mode, default_user, default_group, default_uid, default_gid): - attrs = _merge_attributes(pkg_dirs_info, default_mode, default_user, default_group, default_uid, default_gid) - for dir in pkg_dirs_info.dirs: - dest = dir.strip("/") - _check_dest(ctx, content_map, dest, None, origin) - content_map[dest] = _DestFile( - src = None, - entry_type = ENTRY_IS_DIR, - mode = attrs[0], - user = attrs[1], - group = attrs[2], - uid = attrs[3], - gid = attrs[4], - origin = origin, - ) - -def _process_pkg_files(ctx, content_map, pkg_files_info, origin, default_mode, default_user, default_group, default_uid, default_gid): - attrs = _merge_attributes(pkg_files_info, default_mode, default_user, default_group, default_uid, default_gid) - for filename, src in pkg_files_info.dest_src_map.items(): - dest = filename.strip("/") - _check_dest(ctx, content_map, dest, src, origin) - content_map[dest] = _DestFile( - src = src, - entry_type = ENTRY_IS_TREE if src.is_directory else ENTRY_IS_FILE, - mode = attrs[0], - user = attrs[1], - group = attrs[2], - uid = attrs[3], - gid = attrs[4], - origin = origin, - ) - -def _process_pkg_symlink(ctx, content_map, pkg_symlink_info, origin, default_mode, default_user, default_group, default_uid, default_gid): - dest = pkg_symlink_info.destination.strip("/") - attrs = _merge_attributes(pkg_symlink_info, default_mode, default_user, default_group, default_uid, default_gid) - _check_dest(ctx, content_map, dest, None, origin) - content_map[dest] = _DestFile( - src = None, - entry_type = ENTRY_IS_LINK, - mode = attrs[0], - user = attrs[1], - group = attrs[2], - uid = attrs[3], - gid = attrs[4], - origin = origin, - link_to = pkg_symlink_info.target, - ) - -def _process_pkg_filegroup(ctx, content_map, pkg_filegroup_info, origin, default_mode, default_user, default_group, default_uid, default_gid): - if hasattr(pkg_filegroup_info, "pkg_dirs"): - for d in pkg_filegroup_info.pkg_dirs: - _process_pkg_dirs(ctx, content_map, d[0], d[1], default_mode, default_user, default_group, default_uid, default_gid) - if hasattr(pkg_filegroup_info, "pkg_files"): - for pf in pkg_filegroup_info.pkg_files: - _process_pkg_files(ctx, content_map, pf[0], pf[1], default_mode, default_user, default_group, default_uid, default_gid) - if hasattr(pkg_filegroup_info, "pkg_symlinks"): - for psl in pkg_filegroup_info.pkg_symlinks: - _process_pkg_symlink(ctx, content_map, psl[0], psl[1], default_mode, default_user, default_group, default_uid, default_gid) - -def process_src( - ctx, - content_map, - files, - src, - origin, - default_mode, - default_user, - default_group, - default_uid = None, - default_gid = None): - """Add an entry to the content map. - - Args: - content_map: in/out The content map - files: in/out list of file Depsets represented in the map - src: Source Package*Info object - origin: The rule instance adding this entry - default_mode: fallback mode to use for Package*Info elements without mode - default_user: fallback user to use for Package*Info elements without user - default_group: fallback mode to use for Package*Info elements without group - default_uid: fallback uid to use for Package*Info elements without uid - default_gid: fallback gid to use for Package*Info elements without gid - - Returns: - True if src was a Package*Info and added to content_map. - """ - - # Gather the files for every srcs entry here, even if it is not from - # a pkg_* rule. - if DefaultInfo in src: - files.append(src[DefaultInfo].files) - found_info = False - if PackageFilesInfo in src: - _process_pkg_files( - ctx, - content_map, - src[PackageFilesInfo], - origin, - default_mode = default_mode, - default_user = default_user, - default_group = default_group, - default_uid = default_uid, - default_gid = default_gid, - ) - found_info = True - if PackageFilegroupInfo in src: - _process_pkg_filegroup( - ctx, - content_map, - src[PackageFilegroupInfo], - origin, - default_mode = default_mode, - default_user = default_user, - default_group = default_group, - default_uid = default_uid, - default_gid = default_gid, - ) - found_info = True - if PackageSymlinkInfo in src: - _process_pkg_symlink( - ctx, - content_map, - src[PackageSymlinkInfo], - origin, - default_mode = default_mode, - default_user = default_user, - default_group = default_group, - default_uid = default_uid, - default_gid = default_gid, - ) - found_info = True - if PackageDirsInfo in src: - _process_pkg_dirs( - ctx, - content_map, - src[PackageDirsInfo], - origin, - default_mode = "0555", - default_user = default_user, - default_group = default_group, - default_uid = default_uid, - default_gid = default_gid, - ) - found_info = True - return found_info - -def add_directory(content_map, dir_path, origin, mode = None, user = None, group = None, uid = None, gid = None): - """Add an empty directory to the content map. - - Args: - content_map: The content map - dir_path: Where to place the file in the package. - origin: The rule instance adding this entry - mode: fallback mode to use for Package*Info elements without mode - user: fallback user to use for Package*Info elements without user - group: fallback mode to use for Package*Info elements without group - """ - content_map[dir_path.strip("/")] = _DestFile( - src = None, - entry_type = ENTRY_IS_DIR, - origin = origin, - mode = mode, - user = user, - group = group, - uid = uid, - gid = gid, - ) - -def add_empty_file(ctx, content_map, dest_path, origin, mode = None, user = None, group = None, uid = None, gid = None): - """Add a single file to the content map. - - Args: - ctx: rule context. - content_map: The content map - dest_path: Where to place the file in the package. - origin: The rule instance adding this entry - mode: fallback mode to use for Package*Info elements without mode - user: fallback user to use for Package*Info elements without user - group: fallback mode to use for Package*Info elements without group - """ - dest = dest_path.strip("/") - _check_dest(ctx, content_map, dest, None, origin) - content_map[dest] = _DestFile( - src = None, - entry_type = ENTRY_IS_EMPTY_FILE, - origin = origin, - mode = mode, - user = user, - group = group, - uid = uid, - gid = gid, - ) - -def add_label_list( - ctx, - content_map, - file_deps, - srcs, - default_mode = None, - default_user = None, - default_group = None, - default_uid = None, - default_gid = None): - """Helper method to add a list of labels (typically 'srcs') to a content_map. - - Args: - ctx: rule context. - content_map: (r/w) The content map to update. - file_deps: (r/w) The list of file Depsets that srcs depend on. - srcs: List of source objects. - default_mode: fallback mode to use for Package*Info elements without mode - default_user: fallback user to use for Package*Info elements without user - default_group: fallback mode to use for Package*Info elements without group - default_uid: fallback uid to use for Package*Info elements without uid - default_gid: fallback gid to use for Package*Info elements without guid - """ - - if hasattr(ctx.attr, "include_runfiles"): - include_runfiles = ctx.attr.include_runfiles - else: - include_runfiles = False - - # Compute the relative path - data_path = compute_data_path( - ctx, - ctx.attr.strip_prefix if hasattr(ctx.attr, "strip_prefix") else "", - ) - data_path_without_prefix = compute_data_path(ctx, ".") - - for src in srcs: - if not process_src( - ctx, - content_map, - file_deps, - src = src, - origin = src.label, - default_mode = default_mode, - default_user = default_user, - default_group = default_group, - default_uid = default_uid, - default_gid = default_gid, - ): - # Add in the files of srcs which are not pkg_* types - add_from_default_info( - ctx, - content_map, - file_deps, - src, - data_path, - data_path_without_prefix, - default_mode = default_mode, - default_user = default_user, - default_group = default_group, - default_uid = default_uid, - default_gid = default_gid, - include_runfiles = include_runfiles, - ) - -def add_from_default_info( - ctx, - content_map, - file_deps, - src, - data_path, - data_path_without_prefix, - default_mode = None, - default_user = None, - default_group = None, - default_uid = None, - default_gid = None, - include_runfiles = False): - """Helper method to add the DefaultInfo of a target to a content_map. - - Args: - ctx: rule context. - content_map: (r/w) The content map to update. - file_deps: (r/w) The list of file Depsets that srcs depend on. - src: A source object. - data_path: path to package - data_path_without_prefix: path to the package after prefix stripping - default_mode: fallback mode to use for Package*Info elements without mode - default_user: fallback user to use for Package*Info elements without user - default_group: fallback mode to use for Package*Info elements without group - include_runfiles: Include runfiles - """ - if not DefaultInfo in src: - return - - # Auto-detect the executable so we can set its mode. - the_executable = get_my_executable(src) - all_files = src[DefaultInfo].files.to_list() - for f in all_files: - d_path = dest_path(f, data_path, data_path_without_prefix) - if f.is_directory: - add_tree_artifact( - content_map, - d_path, - f, - origin = src.label, - mode = default_mode, - user = default_user, - group = default_group, - ) - else: - fmode = "0755" if f == the_executable else default_mode - add_single_file( - ctx, - content_map, - dest_path = d_path, - src = f, - origin = src.label, - mode = fmode, - user = default_user, - group = default_group, - ) - if include_runfiles: - runfiles = src[DefaultInfo].default_runfiles - if runfiles: - base_path = d_path + ".runfiles" - for rf in runfiles.files.to_list(): - d_path = base_path + "/" + rf.short_path - fmode = "0755" if rf == the_executable else default_mode - _check_dest(ctx, content_map, d_path, rf, src.label) - content_map[d_path] = _DestFile( - src = rf, - entry_type = ENTRY_IS_FILE, - origin = src.label, - mode = fmode, - user = default_user, - group = default_group, - uid = default_uid, - gid = default_gid, - ) - -def get_my_executable(src): - """If a target represents an executable, return its file handle. - - The roundabout hackery here is because there is no good way to see if - DefaultInfo was created with an executable in it. - See: https://github.com/bazelbuild/bazel/issues/14811 - - Args: - src: A label. - Returns: - File or None. - """ - if not DefaultInfo in src: - return None - di = src[DefaultInfo] - if not hasattr(di, "files_to_run"): - return None - ftr = di.files_to_run - - # The docs lead you to believe that you could look at - # files_to_run.executable, but that is filled out even for source - # files. - if not hasattr(ftr, "runfiles_manifest"): - return None - if ftr.runfiles_manifest: - # DEBUG print("Got an manifest executable", ftr.executable) - return ftr.executable - return None - - -def add_single_file(ctx, content_map, dest_path, src, origin, mode = None, user = None, group = None, uid = None, gid = None): - """Add an single file to the content map. - - Args: - ctx: rule context. - content_map: The content map - dest_path: Where to place the file in the package. - src: Source object. Must have len(src[DefaultInfo].files) == 1 - origin: The rule instance adding this entry - mode: fallback mode to use for Package*Info elements without mode - user: fallback user to use for Package*Info elements without user - group: fallback mode to use for Package*Info elements without group - """ - dest = dest_path.strip("/") - _check_dest(ctx, content_map, dest, src, origin) - content_map[dest] = _DestFile( - src = src, - entry_type = ENTRY_IS_FILE, - origin = origin, - mode = mode, - user = user, - group = group, - uid = uid, - gid = gid, - ) - -def add_symlink(ctx, content_map, dest_path, src, origin, mode = None, user = None, group = None, uid = None, gid = None): - - """Add a symlink to the content map. - - Args: - ctx: rule context. - content_map: The content map - dest_path: Where to place the file in the package. - src: Path to link to. - origin: The rule instance adding this entry - mode: fallback mode to use for Package*Info elements without mode - user: fallback user to use for Package*Info elements without user - group: fallback mode to use for Package*Info elements without group - """ - dest = dest_path.strip("/") - _check_dest(ctx, content_map, dest, None, origin) - content_map[dest] = _DestFile( - src = None, - link_to = src, - entry_type = ENTRY_IS_LINK, - origin = origin, - mode = mode, - user = user, - group = group, - uid = uid, - gid = gid, - ) - -def add_tree_artifact(content_map, dest_path, src, origin, mode = None, user = None, group = None, uid = None, gid = None): - """Add an tree artifact (directory output) to the content map. - - Args: - content_map: The content map - dest_path: Where to place the file in the package. - src: Source object. Must have len(src[DefaultInfo].files) == 1 - origin: The rule instance adding this entry - mode: fallback mode to use for Package*Info elements without mode - user: fallback user to use for Package*Info elements without user - group: fallback mode to use for Package*Info elements without group - """ - content_map[dest_path] = _DestFile( - src = src, - origin = origin, - entry_type = ENTRY_IS_TREE, - mode = mode, - user = user, - group = group, - uid = uid, - gid = gid, - ) - -def write_manifest(ctx, manifest_file, content_map, use_short_path=False, pretty_print=False): - """Write a content map to a manifest file. - - The format of this file is currently undocumented, as it is a private - contract between the rule implementation and the package writers. It will - become a published interface in a future release. - - For reproducibility, the manifest file must be ordered consistently. - - Args: - ctx: rule context - manifest_file: File object used as the output destination - content_map: content_map (see concepts at top of file) - use_short_path: write out the manifest file destinations in terms of "short" paths, suitable for `bazel run`. - """ - ctx.actions.write( - manifest_file, - "[\n" + ",\n".join( - [ - _encode_manifest_entry(dst, content_map[dst], use_short_path, pretty_print) - for dst in sorted(content_map.keys()) - ] - ) + "\n]\n" - ) - -def _encode_manifest_entry(dest, df, use_short_path, pretty_print=False): - entry_type = df.entry_type if hasattr(df, "entry_type") else ENTRY_IS_FILE - if df.src: - src = df.src.short_path if use_short_path else df.src.path - # entry_type is left as-is - - elif hasattr(df, "link_to"): - src = df.link_to - entry_type = ENTRY_IS_LINK - else: - src = None - - # Bazel 6 has a new flag "--incompatible_unambiguous_label_stringification" - # (https://github.com/bazelbuild/bazel/issues/15916) that causes labels in - # the repository in which Bazel was run to be stringified with a preceding - # "@". In older versions, this flag did not exist. - # - # Since this causes all sorts of chaos with our tests, be consistent across - # all Bazel versions. - origin_str = str(df.origin) - if not origin_str.startswith('@'): - origin_str = '@' + origin_str - - data = { - "type": df.entry_type, - "src": src, - "dest": dest.strip("/"), - "mode": df.mode or "", - "user": df.user or None, - "group": df.group or None, - "uid": df.uid, - "gid": df.gid, - "origin": origin_str, - } - - if pretty_print: - return json.encode_indent(data) - else: - return json.encode(data) diff --git a/pkg/private/tar/BUILD b/pkg/private/tar/BUILD deleted file mode 100644 index 6bfeb1e..0000000 --- a/pkg/private/tar/BUILD +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""rules_pkg internal code. - -All interfaces are subject to change at any time. -""" - -load("@rules_python//python:defs.bzl", "py_binary", "py_library") - -licenses(["notice"]) - -filegroup( - name = "standard_package", - srcs = [ - "BUILD", - ] + glob([ - "*.bzl", - "*.py", - ]), - visibility = [ - "//distro:__pkg__", - "//pkg:__pkg__", - ], -) - -exports_files( - glob(["*.bzl"]), - visibility = [ - "//distro:__pkg__", - "//doc_build:__pkg__", - ], -) - -py_binary( - name = "build_tar", - srcs = ["build_tar.py"], - imports = ["../../.."], - python_version = "PY3", - srcs_version = "PY3", - visibility = ["//visibility:public"], - deps = [ - ":tar_writer", - "//pkg/private:archive", - "//pkg/private:build_info", - "//pkg/private:helpers", - "//pkg/private:manifest", - ], -) - -py_library( - name = "tar_writer", - srcs = [ - "tar_writer.py", - ], - imports = ["../../.."], - srcs_version = "PY3", - visibility = [ - "//tests:__subpackages__", - ], -) diff --git a/pkg/private/tar/build_tar.py b/pkg/private/tar/build_tar.py deleted file mode 100644 index 08a3a06..0000000 --- a/pkg/private/tar/build_tar.py +++ /dev/null @@ -1,459 +0,0 @@ -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This tool build tar files from a list of inputs.""" - -import argparse -import os -import tarfile -import tempfile - -from pkg.private import archive -from pkg.private import helpers -from pkg.private import build_info -from pkg.private import manifest -from pkg.private.tar import tar_writer - - -def normpath(path): - """Normalize a path to the format we need it. - - os.path.normpath changes / to \ on windows, but tarfile needs / style paths. - - Args: - path: (str) path to normalize. - """ - return os.path.normpath(path).replace(os.path.sep, '/') - - -class TarFile(object): - """A class to generates a TAR file.""" - - class DebError(Exception): - pass - - def __init__(self, output, directory, compression, compressor, default_mtime): - # Directory prefix on all output paths - d = directory.strip('/') - self.directory = (d + '/') if d else None - self.output = output - self.compression = compression - self.compressor = compressor - self.default_mtime = default_mtime - - def __enter__(self): - self.tarfile = tar_writer.TarFileWriter( - self.output, - self.compression, - self.compressor, - default_mtime=self.default_mtime) - return self - - def __exit__(self, t, v, traceback): - self.tarfile.close() - - def normalize_path(self, path: str) -> str: - dest = normpath(path) - # paths should not have a leading ./ - if dest.startswith('./'): - dest = dest[2:] - # No path should ever come in with slashs on either end, but protect - # against that anyway. - dest = dest.strip('/') - # This prevents a potential problem for users with both a prefix_dir and - # symlinks that also repeat the prefix_dir. The old behavior was that we - # would get just the symlink path. Now we are prefixing with the prefix, - # so you get the file in the wrong place. - # We silently de-dup that. If people come up with a real use case for - # the /a/b/a/b/rest... output we can start an issue and come up with a - # solution at that time. - if self.directory and not dest.startswith(self.directory): - dest = self.directory + dest - return dest - - def add_file(self, f, destfile, mode=None, ids=None, names=None): - """Add a file to the tar file. - - Args: - f: the file to add to the layer - destfile: the name of the file in the layer - mode: (int) force to set the specified mode, by default the value from the - source is taken. - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. `f` will be - copied to `self.directory/destfile` in the layer. - """ - dest = self.normalize_path(destfile) - # If mode is unspecified, derive the mode from the file's mode. - if mode is None: - mode = 0o755 if os.access(f, os.X_OK) else 0o644 - if ids is None: - ids = (0, 0) - if names is None: - names = ('', '') - self.tarfile.add_file( - dest, - file_content=f, - mode=mode, - uid=ids[0], - gid=ids[1], - uname=names[0], - gname=names[1]) - - def add_empty_file(self, - destfile, - mode=None, - ids=None, - names=None, - kind=tarfile.REGTYPE): - """Add a file to the tar file. - - Args: - destfile: the name of the file in the layer - mode: force to set the specified mode, defaults to 644 - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. - kind: type of the file. tarfile.DIRTYPE for directory. An empty file - will be created as `destfile` in the layer. - """ - dest = destfile.lstrip('/') # Remove leading slashes - # If mode is unspecified, assume read only - if mode is None: - mode = 0o644 - if ids is None: - ids = (0, 0) - if names is None: - names = ('', '') - dest = normpath(dest) - self.tarfile.add_file( - dest, - content='' if kind == tarfile.REGTYPE else None, - kind=kind, - mode=mode, - uid=ids[0], - gid=ids[1], - uname=names[0], - gname=names[1]) - - def add_empty_dir(self, destpath, mode=None, ids=None, names=None): - """Add a directory to the tar file. - - Args: - destpath: the name of the directory in the layer - mode: force to set the specified mode, defaults to 644 - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. An empty - file will be created as `destfile` in the layer. - """ - self.add_empty_file( - destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE) - - def add_tar(self, tar): - """Merge a tar file into the destination tar file. - - All files presents in that tar will be added to the output file - under self.directory/path. No user name nor group name will be - added to the output. - - Args: - tar: the tar file to add - """ - self.tarfile.add_tar(tar, numeric=True, prefix=self.directory) - - def add_link(self, symlink, destination, mode=None, ids=None, names=None): - """Add a symbolic link pointing to `destination`. - - Args: - symlink: the name of the symbolic link to add. - destination: where the symbolic link point to. - mode: (int) force to set the specified posix mode (e.g. 0o755). The - default is derived from the source - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. An empty - file will be created as `destfile` in the layer. - """ - dest = self.normalize_path(symlink) - self.tarfile.add_file( - dest, - tarfile.SYMTYPE, - link=destination, - mode = mode, - uid=ids[0], - gid=ids[1], - uname=names[0], - gname=names[1]) - - def add_deb(self, deb): - """Extract a debian package in the output tar. - - All files presents in that debian package will be added to the - output tar under the same paths. No user name nor group names will - be added to the output. - - Args: - deb: the tar file to add - - Raises: - DebError: if the format of the deb archive is incorrect. - """ - with archive.SimpleArReader(deb) as arfile: - current = next(arfile) - while current and not current.filename.startswith('data.'): - current = next(arfile) - if not current: - raise self.DebError(deb + ' does not contains a data file!') - tmpfile = tempfile.mkstemp(suffix=os.path.splitext(current.filename)[-1]) - with open(tmpfile[1], 'wb') as f: - f.write(current.data) - self.add_tar(tmpfile[1]) - os.remove(tmpfile[1]) - - def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None): - """Add a tree artifact to the tar file. - - Args: - tree_top: the top of the tree to add - destpath: the path under which to place the files - mode: (int) force to set the specified posix mode (e.g. 0o755). The - default is derived from the source - ids: (uid, gid) for the file to set ownership - names: (username, groupname) for the file to set ownership. `f` will be - copied to `self.directory/destfile` in the layer. - """ - # We expect /-style paths. - tree_top = normpath(tree_top) - - dest = destpath.strip('/') # redundant, dests should never have / here - if self.directory and self.directory != '/': - dest = self.directory.lstrip('/') + '/' + dest - - # Again, we expect /-style paths. - dest = normpath(dest) - # normpath may be ".", and dest paths should not start with "./" - dest = '' if dest == '.' else dest + '/' - - if ids is None: - ids = (0, 0) - if names is None: - names = ('', '') - - to_write = {} - for root, dirs, files in os.walk(tree_top): - # While `tree_top` uses '/' as a path separator, results returned by - # `os.walk` and `os.path.join` on Windows may not. - root = normpath(root) - - dirs = sorted(dirs) - rel_path_from_top = root[len(tree_top):].lstrip('/') - if rel_path_from_top: - dest_dir = dest + rel_path_from_top + '/' - else: - dest_dir = dest - for dir in dirs: - to_write[dest_dir + dir] = None - for file in sorted(files): - content_path = os.path.abspath(os.path.join(root, file)) - if os.name == "nt": - # "To specify an extended-length path, use the `\\?\` prefix. For - # example, `\\?\D:\very long path`."[1] - # - # [1]: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation - to_write[dest_dir + file] = "\\\\?\\" + content_path - else: - to_write[dest_dir + file] = content_path - - for path in sorted(to_write.keys()): - content_path = to_write[path] - if not content_path: - # This is an intermediate directory. Bazel has no API to specify modes - # for this, so the least surprising thing we can do is make it the - # canonical rwxr-xr-x - self.add_empty_file( - path, - mode=0o755, - ids=ids, - names=names, - kind=tarfile.DIRTYPE) - else: - # If mode is unspecified, derive the mode from the file's mode. - if mode is None: - f_mode = 0o755 if os.access(content_path, os.X_OK) else 0o644 - else: - f_mode = mode - self.tarfile.add_file( - path, - file_content=content_path, - mode=f_mode, - uid=ids[0], - gid=ids[1], - uname=names[0], - gname=names[1]) - - def add_manifest_entry(self, entry, file_attributes): - # Use the pkg_tar mode/owner remapping as a fallback - non_abs_path = entry.dest.strip('/') - if file_attributes: - attrs = file_attributes(non_abs_path) - else: - attrs = {} - # But any attributes from the manifest have higher precedence - if entry.mode is not None and entry.mode != '': - attrs['mode'] = int(entry.mode, 8) - if entry.user: - if entry.group: - attrs['names'] = (entry.user, entry.group) - else: - # Use group that legacy tar process would assign - attrs['names'] = (entry.user, attrs.get('names')[1]) - if entry.uid is not None: - if entry.gid is not None: - attrs['ids'] = (entry.uid, entry.gid) - else: - attrs['ids'] = (entry.uid, entry.uid) - if entry.type == manifest.ENTRY_IS_LINK: - self.add_link(entry.dest, entry.src, **attrs) - elif entry.type == manifest.ENTRY_IS_DIR: - self.add_empty_dir(self.normalize_path(entry.dest), **attrs) - elif entry.type == manifest.ENTRY_IS_TREE: - self.add_tree(entry.src, entry.dest, **attrs) - elif entry.type == manifest.ENTRY_IS_EMPTY_FILE: - self.add_empty_file(self.normalize_path(entry.dest), **attrs) - else: - self.add_file(entry.src, entry.dest, **attrs) - - -def main(): - parser = argparse.ArgumentParser( - description='Helper for building tar packages', - fromfile_prefix_chars='@') - parser.add_argument('--output', required=True, - help='The output file, mandatory.') - parser.add_argument('--manifest', - help='manifest of contents to add to the layer.') - parser.add_argument('--mode', - help='Force the mode on the added files (in octal).') - parser.add_argument( - '--mtime', - help='Set mtime on tar file entries. May be an integer or the' - ' value "portable", to get the value 2000-01-01, which is' - ' is usable with non *nix OSes.') - parser.add_argument('--tar', action='append', - help='A tar file to add to the layer') - parser.add_argument('--deb', action='append', - help='A debian package to add to the layer') - parser.add_argument( - '--directory', - help='Directory in which to store the file inside the layer') - - compression = parser.add_mutually_exclusive_group() - compression.add_argument('--compression', - help='Compression (`gz` or `bz2`), default is none.') - compression.add_argument('--compressor', - help='Compressor program and arguments, ' - 'e.g. `pigz -p 4`') - - parser.add_argument( - '--modes', action='append', - help='Specific mode to apply to specific file (from the file argument),' - ' e.g., path/to/file=0455.') - parser.add_argument( - '--owners', action='append', - help='Specify the numeric owners of individual files, ' - 'e.g. path/to/file=0.0.') - parser.add_argument( - '--owner', default='0.0', - help='Specify the numeric default owner of all files,' - ' e.g., 0.0') - parser.add_argument( - '--owner_name', - help='Specify the owner name of all files, e.g. root.root.') - parser.add_argument( - '--owner_names', action='append', - help='Specify the owner names of individual files, e.g. ' - 'path/to/file=root.root.') - parser.add_argument('--stamp_from', default='', - help='File to find BUILD_STAMP in') - options = parser.parse_args() - - # Parse modes arguments - default_mode = None - if options.mode: - # Convert from octal - default_mode = int(options.mode, 8) - - mode_map = {} - if options.modes: - for filemode in options.modes: - (f, mode) = helpers.SplitNameValuePairAtSeparator(filemode, '=') - if f[0] == '/': - f = f[1:] - mode_map[f] = int(mode, 8) - - default_ownername = ('', '') - if options.owner_name: - default_ownername = options.owner_name.split('.', 1) - names_map = {} - if options.owner_names: - for file_owner in options.owner_names: - (f, owner) = helpers.SplitNameValuePairAtSeparator(file_owner, '=') - (user, group) = owner.split('.', 1) - if f[0] == '/': - f = f[1:] - names_map[f] = (user, group) - - default_ids = options.owner.split('.', 1) - default_ids = (int(default_ids[0]), int(default_ids[1])) - ids_map = {} - if options.owners: - for file_owner in options.owners: - (f, owner) = helpers.SplitNameValuePairAtSeparator(file_owner, '=') - (user, group) = owner.split('.', 1) - if f[0] == '/': - f = f[1:] - ids_map[f] = (int(user), int(group)) - - default_mtime = options.mtime - if options.stamp_from: - default_mtime = build_info.get_timestamp(options.stamp_from) - - # Add objects to the tar file - with TarFile( - options.output, - directory = helpers.GetFlagValue(options.directory), - compression = options.compression, - compressor = options.compressor, - default_mtime=default_mtime) as output: - - def file_attributes(filename): - if filename.startswith('/'): - filename = filename[1:] - return { - 'mode': mode_map.get(filename, default_mode), - 'ids': ids_map.get(filename, default_ids), - 'names': names_map.get(filename, default_ownername), - } - - if options.manifest: - with open(options.manifest, 'r') as manifest_fp: - manifest_entries = manifest.read_entries_from(manifest_fp) - for entry in manifest_entries: - output.add_manifest_entry(entry, file_attributes) - - for tar in options.tar or []: - output.add_tar(tar) - for deb in options.deb or []: - output.add_deb(deb) - - -if __name__ == '__main__': - main() diff --git a/pkg/private/tar/tar.bzl b/pkg/private/tar/tar.bzl deleted file mode 100644 index 3a4d1a6..0000000 --- a/pkg/private/tar/tar.bzl +++ /dev/null @@ -1,372 +0,0 @@ -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Rules for making .tar files.""" - -load("//pkg:path.bzl", "compute_data_path", "dest_path") -load("//pkg:providers.bzl", "PackageVariablesInfo") -load( - "//pkg/private:pkg_files.bzl", - "add_directory", - "add_empty_file", - "add_label_list", - "add_single_file", - "add_symlink", - "add_tree_artifact", - "process_src", - "write_manifest", -) -load("//pkg/private:util.bzl", "setup_output_files", "substitute_package_variables") - -# TODO(aiuto): Figure out how to get this from the python toolchain. -# See check for lzma in archive.py for a hint at a method. -HAS_XZ_SUPPORT = True - -# Filetype to restrict inputs -tar_filetype = ( - [".tar", ".tar.gz", ".tgz", ".tar.bz2", "tar.xz", ".txz"] if HAS_XZ_SUPPORT else [".tar", ".tar.gz", ".tgz", ".tar.bz2"] -) -SUPPORTED_TAR_COMPRESSIONS = ( - ["", "gz", "bz2", "xz"] if HAS_XZ_SUPPORT else ["", "gz", "bz2"] -) -_DEFAULT_MTIME = -1 -_stamp_condition = Label("//pkg/private:private_stamp_detect") - -def _remap(remap_paths, path): - """If path starts with a key in remap_paths, rewrite it.""" - for prefix, replacement in remap_paths.items(): - if path.startswith(prefix): - return replacement + path[len(prefix):] - return path - -def _quote(filename, protect = "="): - """Quote the filename, by escaping = by \\= and \\ by \\\\""" - return filename.replace("\\", "\\\\").replace(protect, "\\" + protect) - -def _pkg_tar_impl(ctx): - """Implementation of the pkg_tar rule.""" - - # Files needed by rule implementation at runtime - files = [] - - outputs, output_file, output_name = setup_output_files(ctx) - - # Compute the relative path - data_path = compute_data_path(ctx, ctx.attr.strip_prefix) - data_path_without_prefix = compute_data_path(ctx, ".") - - # Find a list of path remappings to apply. - remap_paths = ctx.attr.remap_paths - - # Start building the arguments. - args = ctx.actions.args() - args.add("--output", output_file.path) - args.add("--mode", ctx.attr.mode) - args.add("--owner", ctx.attr.owner) - args.add("--owner_name", ctx.attr.ownername) - - # Package dir can be specified by a file or inlined. - if ctx.attr.package_dir_file: - if ctx.attr.package_dir: - fail("Both package_dir and package_dir_file attributes were specified") - args.add("--directory", "@" + ctx.file.package_dir_file.path) - files.append(ctx.file.package_dir_file) - else: - package_dir_expanded = substitute_package_variables(ctx, ctx.attr.package_dir) - args.add("--directory", package_dir_expanded or "/") - - if ctx.executable.compressor: - args.add("--compressor", "%s %s" % (ctx.executable.compressor.path, ctx.attr.compressor_args)) - else: - extension = ctx.attr.extension - if extension and extension != "tar": - compression = None - dot_pos = ctx.attr.extension.rfind(".") - if dot_pos >= 0: - compression = ctx.attr.extension[dot_pos + 1:] - else: - compression = ctx.attr.extension - if compression == "tgz": - compression = "gz" - if compression == "txz": - compression = "xz" - if compression: - if compression in SUPPORTED_TAR_COMPRESSIONS: - args.add("--compression", compression) - else: - fail("Unsupported compression: '%s'" % compression) - - if ctx.attr.mtime != _DEFAULT_MTIME: - if ctx.attr.portable_mtime: - fail("You may not set both mtime and portable_mtime") - args.add("--mtime", "%d" % ctx.attr.mtime) - if ctx.attr.portable_mtime: - args.add("--mtime", "portable") - - # Now we begin processing the files. - file_deps = [] # inputs we depend on - content_map = {} # content handled in the manifest - - # Start with all the pkg_* inputs - for src in ctx.attr.srcs: - if not process_src( - ctx, - content_map, - file_deps, - src = src, - origin = src.label, - default_mode = None, - default_user = None, - default_group = None, - default_uid = None, - default_gid = None, - ): - src_files = src[DefaultInfo].files.to_list() - if ctx.attr.include_runfiles: - runfiles = src[DefaultInfo].default_runfiles - if runfiles: - file_deps.append(runfiles.files) - src_files.extend(runfiles.files.to_list()) - - # Add in the files of srcs which are not pkg_* types - for f in src_files: - d_path = dest_path(f, data_path, data_path_without_prefix) - - # Note: This extra remap is the bottleneck preventing this - # large block from being a utility method as shown below. - # Should we disallow mixing pkg_files in srcs with remap? - # I am fine with that if it makes the code more readable. - dest = _remap(remap_paths, d_path) - if f.is_directory: - add_tree_artifact(content_map, dest, f, src.label) - else: - # Note: This extra remap is the bottleneck preventing this - # large block from being a utility method as shown below. - # Should we disallow mixing pkg_files in srcs with remap? - # I am fine with that if it makes the code more readable. - dest = _remap(remap_paths, d_path) - add_single_file(ctx, content_map, dest, f, src.label) - - # TODO(aiuto): I want the code to look like this, but we don't have lambdas. - # transform_path = lambda f: _remap( - # remap_paths, dest_path(f, data_path, data_path_without_prefix)) - # add_label_list(ctx, content_map, file_deps, ctx.attr.srcs, transform_path) - - # The files attribute is a map of labels to destinations. We can add them - # directly to the content map. - for target, f_dest_path in ctx.attr.files.items(): - target_files = target.files.to_list() - if len(target_files) != 1: - fail("Each input must describe exactly one file.", attr = "files") - file_deps.append(depset([target_files[0]])) - add_single_file( - ctx, - content_map, - f_dest_path, - target_files[0], - target.label, - ) - - if ctx.attr.modes: - for key in ctx.attr.modes: - args.add("--modes", "%s=%s" % (_quote(key), ctx.attr.modes[key])) - if ctx.attr.owners: - for key in ctx.attr.owners: - args.add("--owners", "%s=%s" % (_quote(key), ctx.attr.owners[key])) - if ctx.attr.ownernames: - for key in ctx.attr.ownernames: - args.add( - "--owner_names", - "%s=%s" % (_quote(key), ctx.attr.ownernames[key]), - ) - for empty_file in ctx.attr.empty_files: - add_empty_file(ctx, content_map, empty_file, ctx.label) - for empty_dir in ctx.attr.empty_dirs or []: - add_directory(content_map, empty_dir, ctx.label) - for f in ctx.files.deps: - args.add("--tar", f.path) - for link in ctx.attr.symlinks: - add_symlink( - ctx, - content_map, - link, - ctx.attr.symlinks[link], - ctx.label, - ) - if ctx.attr.stamp == 1 or (ctx.attr.stamp == -1 and - ctx.attr.private_stamp_detect): - args.add("--stamp_from", ctx.version_file.path) - files.append(ctx.version_file) - - manifest_file = ctx.actions.declare_file(ctx.label.name + ".manifest") - files.append(manifest_file) - write_manifest(ctx, manifest_file, content_map) - args.add("--manifest", manifest_file.path) - - args.set_param_file_format("flag_per_line") - args.use_param_file("@%s", use_always = False) - - inputs = depset(direct = ctx.files.deps + files, transitive = file_deps) - - ctx.actions.run( - mnemonic = "PackageTar", - progress_message = "Writing: %s" % output_file.path, - inputs = inputs, - tools = [ctx.executable.compressor] if ctx.executable.compressor else [], - executable = ctx.executable._build_tar, - arguments = [args], - outputs = [output_file], - env = { - "LANG": "en_US.UTF-8", - "LC_CTYPE": "UTF-8", - "PYTHONIOENCODING": "UTF-8", - "PYTHONUTF8": "1", - }, - use_default_shell_env = True, - ) - return [ - DefaultInfo( - files = depset([output_file]), - runfiles = ctx.runfiles(files = outputs), - ), - # NB: this is not a committed public API. - # The format of this file is subject to change without notice, - # or this OutputGroup might be totally removed. - # Depend on it at your own risk! - OutputGroupInfo( - manifest = [manifest_file], - ), - ] - -# A rule for creating a tar file, see README.md -pkg_tar_impl = rule( - implementation = _pkg_tar_impl, - attrs = { - "strip_prefix": attr.string( - doc = """(note: Use strip_prefix = "." to strip path to the package but preserve relative paths of sub directories beneath the package.)""", - ), - "package_dir": attr.string( - doc = """Prefix to be prepend to all paths written. - - This is applied as a final step, while writing to the archive. - Any other attributes (e.g. symlinks) which specify a path, must do so relative to package_dir. - The value may contain variables. See [package_file_name](#package_file_name) for examples. - """, - ), - "package_dir_file": attr.label(allow_single_file = True), - "deps": attr.label_list( - doc = """tar files which will be unpacked and repacked into the archive.""", - allow_files = tar_filetype, - ), - "srcs": attr.label_list( - doc = """Inputs which will become part of the tar archive.""", - allow_files = True, - ), - "files": attr.label_keyed_string_dict( - doc = """Obsolete. Do not use.""", - allow_files = True, - ), - "mode": attr.string(default = "0555"), - "modes": attr.string_dict(), - "mtime": attr.int(default = _DEFAULT_MTIME), - "portable_mtime": attr.bool(default = True), - "owner": attr.string( - doc = """Default numeric owner.group to apply to files when not set via pkg_attribures.""", - default = "0.0", - ), - "ownername": attr.string(default = "."), - "owners": attr.string_dict(), - "ownernames": attr.string_dict(), - "extension": attr.string(default = "tar"), - "symlinks": attr.string_dict(), - "empty_files": attr.string_list(), - "include_runfiles": attr.bool(), - "empty_dirs": attr.string_list(), - "remap_paths": attr.string_dict(), - "compressor": attr.label( - doc = """External tool which can compress the archive.""", - executable = True, - cfg = "exec", - ), - "compressor_args": attr.string( - doc = """Arg list for `compressor`.""", - ), - - # Common attributes - "out": attr.output(mandatory = True), - "package_file_name": attr.string(doc = "See [Common Attributes](#package_file_name)"), - "package_variables": attr.label( - doc = "See [Common Attributes](#package_variables)", - providers = [PackageVariablesInfo], - ), - "allow_duplicates_with_different_content": attr.bool( - default=True, - doc="""If true, will allow you to reference multiple pkg_* which conflict -(writing different content or metadata to the same destination). -Such behaviour is always incorrect, but we provide a flag to support it in case old -builds were accidentally doing it. Never explicitly set this to true for new code. -""" - ), - "stamp": attr.int( - doc = """Enable file time stamping. Possible values: -<li>stamp = 1: Use the time of the build as the modification time of each file in the archive. -<li>stamp = 0: Use an "epoch" time for the modification time of each file. This gives good build result caching. -<li>stamp = -1: Control the chosen modification time using the --[no]stamp flag. -@since(0.5.0) -""", - default = 0, - ), - # Is --stamp set on the command line? - # TODO(https://github.com/bazelbuild/rules_pkg/issues/340): Remove this. - "private_stamp_detect": attr.bool(default = False), - - # Implicit dependencies. - "_build_tar": attr.label( - default = Label("//pkg/private/tar:build_tar"), - cfg = "exec", - executable = True, - allow_files = True, - ), - }, -) - -def pkg_tar(name, **kwargs): - """Creates a .tar file. See pkg_tar_impl. - - @wraps(pkg_tar_impl) - """ - - # Compatibility with older versions of pkg_tar that define files as - # a flat list of labels. - if "srcs" not in kwargs: - if "files" in kwargs: - if not hasattr(kwargs["files"], "items"): - label = "%s//%s:%s" % (native.repository_name(), native.package_name(), name) - - # buildifier: disable=print - print("%s: you provided a non dictionary to the pkg_tar `files` attribute. " % (label,) + - "This attribute was renamed to `srcs`. " + - "Consider renaming it in your BUILD file.") - kwargs["srcs"] = kwargs.pop("files") - extension = kwargs.get("extension") or "tar" - if extension[0] == ".": - extension = extension[1:] - pkg_tar_impl( - name = name, - out = kwargs.pop("out", None) or (name + "." + extension), - private_stamp_detect = select({ - _stamp_condition: True, - "//conditions:default": False, - }), - **kwargs - ) diff --git a/pkg/private/tar/tar_writer.py b/pkg/private/tar/tar_writer.py deleted file mode 100644 index 325db76..0000000 --- a/pkg/private/tar/tar_writer.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright 2022 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tar writing helper.""" - -import gzip -import io -import os -import subprocess -import tarfile - -try: - import lzma # pylint: disable=g-import-not-at-top, unused-import - HAS_LZMA = True -except ImportError: - HAS_LZMA = False - -# This is slightly a lie. We do support xz fallback through the xz tool, but -# that is fragile. Users should stick to the expectations provided here. -COMPRESSIONS = ('', 'gz', 'bz2', 'xz') if HAS_LZMA else ('', 'gz', 'bz2') - -# Use a deterministic mtime that doesn't confuse other programs. -# See: https://github.com/bazelbuild/bazel/issues/1299 -PORTABLE_MTIME = 946684800 # 2000-01-01 00:00:00.000 UTC - -_DEBUG_VERBOSITY = 0 - - -class TarFileWriter(object): - """A wrapper to write tar files.""" - - class Error(Exception): - pass - - def __init__(self, - name, - compression='', - compressor='', - default_mtime=None, - preserve_tar_mtimes=True): - """TarFileWriter wraps tarfile.open(). - - Args: - name: the tar file name. - compression: compression type: bzip2, bz2, gz, tgz, xz, lzma. - compressor: custom command to do the compression. - default_mtime: default mtime to use for elements in the archive. - May be an integer or the value 'portable' to use the date - 2000-01-01, which is compatible with non *nix OSes'. - preserve_tar_mtimes: if true, keep file mtimes from input tar file. - """ - self.preserve_mtime = preserve_tar_mtimes - if default_mtime is None: - self.default_mtime = 0 - elif default_mtime == 'portable': - self.default_mtime = PORTABLE_MTIME - else: - self.default_mtime = int(default_mtime) - - self.fileobj = None - self.compressor_cmd = (compressor or '').strip() - if self.compressor_cmd: - # Some custom command has been specified: no need for further - # configuration, we're just going to use it. - pass - # Support xz compression through xz... until we can use Py3 - elif compression in ['xz', 'lzma']: - if HAS_LZMA: - mode = 'w:xz' - else: - self.compressor_cmd = 'xz -F {} -'.format(compression) - elif compression in ['bzip2', 'bz2']: - mode = 'w:bz2' - else: - mode = 'w:' - if compression in ['tgz', 'gz']: - # The Tarfile class doesn't allow us to specify gzip's mtime attribute. - # Instead, we manually reimplement gzopen from tarfile.py and set mtime. - self.fileobj = gzip.GzipFile( - filename=name, mode='w', compresslevel=6, mtime=self.default_mtime) - self.compressor_proc = None - if self.compressor_cmd: - mode = 'w|' - self.compressor_proc = subprocess.Popen(self.compressor_cmd.split(), - stdin=subprocess.PIPE, - stdout=open(name, 'wb')) - self.fileobj = self.compressor_proc.stdin - self.name = name - - self.tar = tarfile.open(name=name, mode=mode, fileobj=self.fileobj, - format=tarfile.GNU_FORMAT) - self.members = set() - self.directories = set() - # Preseed the added directory list with things we should not add. If we - # some day need to allow '.' or '/' as an explicit member of the archive, - # we can adjust that here based on the setting of root_dirctory. - self.directories.add('/') - self.directories.add('./') - - def __enter__(self): - return self - - def __exit__(self, t, v, traceback): - self.close() - - def _have_added(self, path): - """Have we added this file before.""" - return (path in self.members) or (path in self.directories) - - def _addfile(self, info, fileobj=None): - """Add a file in the tar file if there is no conflict.""" - if info.type == tarfile.DIRTYPE: - # Enforce the ending / for directories so we correctly deduplicate. - if not info.name.endswith('/'): - info.name += '/' - if not self._have_added(info.name): - self.tar.addfile(info, fileobj) - self.members.add(info.name) - if info.type == tarfile.DIRTYPE: - self.directories.add(info.name) - elif info.type != tarfile.DIRTYPE: - print('Duplicate file in archive: %s, ' - 'picking first occurrence' % info.name) - - def add_directory_path(self, - path, - uid=0, - gid=0, - uname='', - gname='', - mtime=None, - mode=0o755): - """Add a directory to the current tar. - - Args: - path: the ('/' delimited) path of the file to add. - uid: owner user identifier. - gid: owner group identifier. - uname: owner user names. - gname: owner group names. - mtime: modification time to put in the archive. - mode: unix permission mode of the file, default: 0755. - """ - assert path[-1] == '/' - if not path or self._have_added(path): - return - if _DEBUG_VERBOSITY > 1: - print('DEBUG: adding directory', path) - tarinfo = tarfile.TarInfo(path) - tarinfo.type = tarfile.DIRTYPE - tarinfo.mtime = mtime - tarinfo.mode = mode - tarinfo.uid = uid - tarinfo.gid = gid - tarinfo.uname = uname - tarinfo.gname = gname - self._addfile(tarinfo) - - def add_parents(self, path, uid=0, gid=0, uname='', gname='', mtime=0, mode=0o755): - dirs = path.split('/') - parent_path = '' - for next_level in dirs[0:-1]: - parent_path = parent_path + next_level + '/' - self.add_directory_path( - parent_path, - uid=uid, - gid=gid, - uname=uname, - gname=gname, - mtime=mtime, - mode=0o755) - - def add_file(self, - name, - kind=tarfile.REGTYPE, - content=None, - link=None, - file_content=None, - uid=0, - gid=0, - uname='', - gname='', - mtime=None, - mode=None): - """Add a file to the current tar. - - Args: - name: the ('/' delimited) path of the file to add. - kind: the type of the file to add, see tarfile.*TYPE. - content: the content to put in the file. - link: if the file is a link, the destination of the link. - file_content: file to read the content from. Provide either this - one or `content` to specifies a content for the file. - uid: owner user identifier. - gid: owner group identifier. - uname: owner user names. - gname: owner group names. - mtime: modification time to put in the archive. - mode: unix permission mode of the file, default 0644 (0755). - """ - if not name: - return - if name == '.': - return - if name in self.members: - return - - if mtime is None: - mtime = self.default_mtime - - # Make directories up the file - self.add_parents(name, mtime=mtime, mode=0o755, uid=uid, gid=gid, uname=uname, gname=gname) - - tarinfo = tarfile.TarInfo(name) - tarinfo.mtime = mtime - tarinfo.uid = uid - tarinfo.gid = gid - tarinfo.uname = uname - tarinfo.gname = gname - tarinfo.type = kind - if mode is None: - tarinfo.mode = 0o644 if kind == tarfile.REGTYPE else 0o755 - else: - tarinfo.mode = mode - if link: - tarinfo.linkname = link - if content: - content_bytes = content.encode('utf-8') - tarinfo.size = len(content_bytes) - self._addfile(tarinfo, io.BytesIO(content_bytes)) - elif file_content: - with open(file_content, 'rb') as f: - tarinfo.size = os.fstat(f.fileno()).st_size - self._addfile(tarinfo, f) - else: - self._addfile(tarinfo) - - def add_tar(self, - tar, - rootuid=None, - rootgid=None, - numeric=False, - name_filter=None, - prefix=None): - """Merge a tar content into the current tar, stripping timestamp. - - Args: - tar: the name of tar to extract and put content into the current tar. - rootuid: user id that we will pretend is root (replaced by uid 0). - rootgid: group id that we will pretend is root (replaced by gid 0). - numeric: set to true to strip out name of owners (and just use the - numeric values). - name_filter: filter out file by names. If not none, this method will be - called for each file to add, given the name and should return true if - the file is to be added to the final tar and false otherwise. - prefix: prefix to add to all file paths. - - Raises: - TarFileWriter.Error: if an error happens when uncompressing the tar file. - """ - if prefix: - prefix = prefix.strip('/') + '/' - if _DEBUG_VERBOSITY > 1: - print('========================== prefix is', prefix) - intar = tarfile.open(name=tar, mode='r:*') - for tarinfo in intar: - if name_filter is None or name_filter(tarinfo.name): - if not self.preserve_mtime: - tarinfo.mtime = self.default_mtime - if rootuid is not None and tarinfo.uid == rootuid: - tarinfo.uid = 0 - tarinfo.uname = 'root' - if rootgid is not None and tarinfo.gid == rootgid: - tarinfo.gid = 0 - tarinfo.gname = 'root' - if numeric: - tarinfo.uname = '' - tarinfo.gname = '' - - in_name = tarinfo.name - if prefix: - in_name = os.path.normpath(prefix + in_name).replace(os.path.sep, '/') - tarinfo.name = in_name - self.add_parents( - path=tarinfo.name, - mtime=tarinfo.mtime, - mode=0o755, - uid=tarinfo.uid, - gid=tarinfo.gid, - uname=tarinfo.uname, - gname=tarinfo.gname) - - if prefix is not None: - # Relocate internal hardlinks as well to avoid breaking them. - link = tarinfo.linkname - if link.startswith('.') and tarinfo.type == tarfile.LNKTYPE: - tarinfo.linkname = '.' + prefix + link.lstrip('.') - - # Remove path pax header to ensure that the proposed name is going - # to be used. Without this, files with long names will not be - # properly written to its new path. - if 'path' in tarinfo.pax_headers: - del tarinfo.pax_headers['path'] - - if tarinfo.isfile(): - # use extractfile(tarinfo) instead of tarinfo.name to preserve - # seek position in intar - self._addfile(tarinfo, intar.extractfile(tarinfo)) - else: - self._addfile(tarinfo) - intar.close() - - def close(self): - """Close the output tar file. - - This class should not be used anymore after calling that method. - - Raises: - TarFileWriter.Error: if an error happens when compressing the output file. - """ - self.tar.close() - # Close the file object if necessary. - if self.fileobj: - self.fileobj.close() - if self.compressor_proc and self.compressor_proc.wait() != 0: - raise self.Error('Custom compression command ' - '"{}" failed'.format(self.compressor_cmd)) - diff --git a/pkg/private/util.bzl b/pkg/private/util.bzl deleted file mode 100644 index 7d36cf8..0000000 --- a/pkg/private/util.bzl +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Internal utilities for rules_pkg.""" - -load("//pkg:providers.bzl", "PackageVariablesInfo") - -def setup_output_files(ctx, package_file_name = None, default_output_file = None): - """Provide output file metadata for common packaging rules - - By default, this will instruct rules to write directly to the File specified - by the `default_output_file` argument or `ctx.outputs.out` otherwise. - - If `package_file_name` is given, or is available in `ctx`, we will write to - that name instead, do substitution via `ctx.attr.package_variables`, and the - default output (either by `default_output_file` or `ctx.outputs.out`) will - be a symlink to it. - - Callers should: - - write to `output_file` - - add `outputs` to their returned `DefaultInfo(files)` provider - - Possibly add a distinguishing element to OutputGroups - - Args: - ctx: rule context - package_file_name: computed value for package_file_name - default_output_file: File identifying the rule's default output, otherwise `ctx.outputs.out` will be used instead. - - Returns: - outputs: list(output handles) - output_file: file handle to write to - output_name: name of output file - - """ - default_output = default_output_file or ctx.outputs.out - - outputs = [default_output] - if not package_file_name: - package_file_name = ctx.attr.package_file_name - if package_file_name: - output_name = substitute_package_variables(ctx, package_file_name) - output_file = ctx.actions.declare_file(output_name) - outputs.append(output_file) - ctx.actions.symlink( - output = default_output, - target_file = output_file, - ) - else: - output_file = default_output - output_name = output_file.basename - return outputs, output_file, output_name - -def substitute_package_variables(ctx, attribute_value): - """Substitute package_variables in the attribute with the given name. - - Args: - ctx: context - attribute_value: the name of the attribute to perform package_variables substitution for - - Returns: - expanded_attribute_value: new value of the attribute after package_variables substitution - """ - if not attribute_value: - return attribute_value - - if type(attribute_value) != "string": - fail("attempt to substitute package_variables in the attribute value %s which is not a string" % attribute_value) - vars = dict(ctx.var) - if ctx.attr.package_variables: - package_variables = ctx.attr.package_variables[PackageVariablesInfo] - vars.update(package_variables.values) - - # Map $(var) to {x} and then use format for substitution. - # This is brittle and I hate it. We should have template substitution - # in the Starlark runtime. - return attribute_value.replace("$(", "{").replace(")", "}").format(**vars) diff --git a/pkg/private/zip/BUILD b/pkg/private/zip/BUILD deleted file mode 100644 index 4823e90..0000000 --- a/pkg/private/zip/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""rules_pkg internal code. - -All interfaces are subject to change at any time. -""" - -load("@rules_python//python:defs.bzl", "py_binary") - -package(default_applicable_licenses = ["//:license"]) - -filegroup( - name = "standard_package", - srcs = [ - "BUILD", - ] + glob([ - "*.bzl", - "*.py", - ]), - visibility = [ - "//distro:__pkg__", - "//pkg:__pkg__", - ], -) - -exports_files( - glob([ - "*.bzl", - ]), - visibility = [ - "//distro:__pkg__", - "//doc_build:__pkg__", - "//pkg:__pkg__", - ], -) - -py_binary( - name = "build_zip", - srcs = ["build_zip.py"], - imports = ["../../.."], - python_version = "PY3", - srcs_version = "PY3", - visibility = ["//visibility:public"], - deps = [ - "//pkg/private:build_info", - "//pkg/private:helpers", - "//pkg/private:manifest", - ], -) diff --git a/pkg/private/zip/__init__.py b/pkg/private/zip/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/pkg/private/zip/__init__.py +++ /dev/null diff --git a/pkg/private/zip/build_zip.py b/pkg/private/zip/build_zip.py deleted file mode 100644 index ca48a08..0000000 --- a/pkg/private/zip/build_zip.py +++ /dev/null @@ -1,305 +0,0 @@ -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This tool builds zip files from a list of inputs.""" - -import argparse -import datetime -import logging -import os -import sys -import zipfile - -from pkg.private import build_info -from pkg.private import manifest - -ZIP_EPOCH = 315532800 - -# Unix dir bit and Windows dir bit. Magic from zip spec -UNIX_DIR_BIT = 0o40000 -MSDOS_DIR_BIT = 0x10 -UNIX_SYMLINK_BIT = 0o120000 - -def _create_argument_parser(): - """Creates the command line arg parser.""" - parser = argparse.ArgumentParser(description='create a zip file', - fromfile_prefix_chars='@') - parser.add_argument('-o', '--output', type=str, - help='The output zip file path.') - parser.add_argument( - '-d', '--directory', type=str, default='/', - help='An absolute path to use as a prefix for all files in the zip.') - parser.add_argument( - '-t', '--timestamp', type=int, default=ZIP_EPOCH, - help='The unix time to use for files added into the zip. values prior to' - ' Jan 1, 1980 are ignored.') - parser.add_argument('--stamp_from', default='', - help='File to find BUILD_STAMP in') - parser.add_argument( - '-m', '--mode', - help='The file system mode to use for files added into the zip.') - parser.add_argument( - '-c', '--compression_type', - help='The compression type to use') - parser.add_argument( - '-l', '--compression_level', - help='The compression level to use') - parser.add_argument('--manifest', - help='manifest of contents to add to the layer.', - required=True) - parser.add_argument( - 'files', type=str, nargs='*', - help='Files to be added to the zip, in the form of {srcpath}={dstpath}.') - return parser - - -def _combine_paths(left, right): - result = left.rstrip('/') + '/' + right.lstrip('/') - - # important: remove leading /'s: the zip format spec says paths should never - # have a leading slash, but Python will happily do this. The built-in zip - # tool in Windows will complain that such a zip file is invalid. - return result.lstrip('/') - - -def parse_date(ts): - ts = datetime.datetime.utcfromtimestamp(ts) - return (ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second) - - -class ZipWriter(object): - - def __init__(self, output_path: str, time_stamp: int, default_mode: int, compression_type: str, compression_level: int): - """Create a writer. - - You must close() after use or use in a 'with' statement. - - Args: - output_path: path to write to - time_stamp: time stamp to add to files - default_mode: file mode to use if not specified in the entry. - """ - self.output_path = output_path - self.time_stamp = time_stamp - self.default_mode = default_mode - compressions = { - "deflated": zipfile.ZIP_DEFLATED, - "lzma": zipfile.ZIP_LZMA, - "bzip2": zipfile.ZIP_BZIP2, - "stored": zipfile.ZIP_STORED - } - self.compression_type = compressions[compression_type] - self.compression_level = compression_level - self.zip_file = zipfile.ZipFile(self.output_path, mode='w', compression=self.compression_type) - - def __enter__(self): - return self - - def __exit__(self, t, v, traceback): - self.close() - - def close(self): - self.zip_file.close() - self.zip_file = None - - def writestr(self, entry_info, content: str, compresslevel: int): - if sys.version_info >= (3, 7): - self.zip_file.writestr(entry_info, content, compresslevel=compresslevel) - else: - # Python 3.6 and lower don't support compresslevel - self.zip_file.writestr(entry_info, content) - if compresslevel != 6: - logging.warn("Custom compresslevel is not supported with python < 3.7") - - def make_zipinfo(self, path: str, mode: str): - """Create a Zipinfo. - - Args: - path: file path - mode: file mode - """ - entry_info = zipfile.ZipInfo(filename=path, date_time=self.time_stamp) - # See http://www.pkware.com/documents/casestudies/APPNOTE.TXT - # denotes UTF-8 encoded file name. - entry_info.flag_bits |= 0x800 - - # See: https://trac.edgewall.org/attachment/ticket/8919/ZipDownload.patch - # external_attr is 4 bytes in size. The high order two bytes represent UNIX - # permission and file type bits, while the low order two contain MS-DOS FAT - # file attributes. - if mode: - f_mode = int(mode, 8) - else: - f_mode = self.default_mode - entry_info.external_attr = f_mode << 16 - return entry_info - - def add_manifest_entry(self, entry): - """Add an entry to the zip file. - - Args: - zip_file: ZipFile to write to - entry: manifest entry - """ - - entry_type = entry.type - dest = entry.dest - src = entry.src - mode = entry.mode - user = entry.user - group = entry.group - - # Use the pkg_tar mode/owner remapping as a fallback - dst_path = dest.strip('/') - if entry_type == manifest.ENTRY_IS_DIR and not dst_path.endswith('/'): - dst_path += '/' - entry_info = self.make_zipinfo(path=dst_path, mode=mode) - - if entry_type == manifest.ENTRY_IS_FILE: - entry_info.compress_type = self.compression_type - # Using utf-8 for the file names is for python <3.7 compatibility. - with open(src.encode('utf-8'), 'rb') as src_content: - self.writestr(entry_info, src_content.read(), compresslevel=self.compression_level) - elif entry_type == manifest.ENTRY_IS_DIR: - entry_info.compress_type = zipfile.ZIP_STORED - # Set directory bits - entry_info.external_attr |= (UNIX_DIR_BIT << 16) | MSDOS_DIR_BIT - self.zip_file.writestr(entry_info, '') - elif entry_type == manifest.ENTRY_IS_LINK: - entry_info.compress_type = zipfile.ZIP_STORED - # Set directory bits - entry_info.external_attr |= (UNIX_SYMLINK_BIT << 16) - self.zip_file.writestr(entry_info, src.encode('utf-8')) - elif entry_type == manifest.ENTRY_IS_TREE: - self.add_tree(src, dst_path, mode) - elif entry_type == manifest.ENTRY_IS_EMPTY_FILE: - entry_info.compress_type = zipfile.ZIP_STORED - self.zip_file.writestr(entry_info, '') - else: - raise Exception('Unknown type for manifest entry:', entry) - - def add_tree(self, tree_top: str, destpath: str, mode: int): - """Add a tree artifact to the zip file. - - Args: - tree_top: the top of the tree to add - destpath: the path under which to place the files - mode: if not None, file mode to apply to all files - """ - - # We expect /-style paths. - tree_top = os.path.normpath(tree_top).replace(os.path.sep, '/') - - # Again, we expect /-style paths. - dest = destpath.strip('/') # redundant, dests should never have / here - dest = os.path.normpath(dest).replace(os.path.sep, '/') - # paths should not have a leading ./ - dest = '' if dest == '.' else dest + '/' - - to_write = {} - for root, dirs, files in os.walk(tree_top): - # While `tree_top` uses '/' as a path separator, results returned by - # `os.walk` and `os.path.join` on Windows may not. - root = os.path.normpath(root).replace(os.path.sep, '/') - - rel_path_from_top = root[len(tree_top):].lstrip('/') - if rel_path_from_top: - dest_dir = dest + rel_path_from_top + '/' - else: - dest_dir = dest - to_write[dest_dir] = None - for file in files: - content_path = os.path.abspath(os.path.join(root, file)) - if os.name == "nt": - # "To specify an extended-length path, use the `\\?\` prefix. For - # example, `\\?\D:\very long path`."[1] - # - # [1]: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation - to_write[dest_dir + file] = "\\\\?\\" + content_path - else: - to_write[dest_dir + file] = content_path - - for path in sorted(to_write.keys()): - content_path = to_write[path] - if content_path: - # If mode is unspecified, derive the mode from the file's mode. - if mode is None: - f_mode = "0o755" if os.access(content_path, os.X_OK) else self.default_mode - else: - f_mode = mode - entry_info = self.make_zipinfo(path=path, mode=f_mode) - entry_info.compress_type = self.compression_type - with open(content_path, 'rb') as src: - self.writestr(entry_info, src.read(), compresslevel=self.compression_level) - else: - # Implicitly created directory - dir_path = path - if not dir_path.endswith('/'): - dir_path += '/' - entry_info = self.make_zipinfo(path=dir_path, mode="0o755") - entry_info.compress_type = zipfile.ZIP_STORED - # Set directory bits - entry_info.external_attr |= (UNIX_DIR_BIT << 16) | MSDOS_DIR_BIT - self.zip_file.writestr(entry_info, '') - -def _load_manifest(prefix, manifest_path): - manifest_map = {} - - for entry in manifest.read_entries_from_file(manifest_path): - entry.dest = _combine_paths(prefix, entry.dest) - manifest_map[entry.dest] = entry - - # We modify the dictionary as we're iterating over it, so we need to listify - # the keys here. - manifest_keys = list(manifest_map.keys()) - # Add all parent directories of entries that have not been added explicitly. - for dest in manifest_keys: - parent = dest - # TODO: use pathlib instead of string manipulation? - for _ in range(dest.count("/")): - parent, _, _ = parent.rpartition("/") - if parent and parent not in manifest_map: - manifest_map[parent] = manifest.ManifestEntry( - type = manifest.ENTRY_IS_DIR, - dest = parent, - src = "", - mode = "0o755", - user = None, - group = None, - uid = None, - gid = None, - origin = "parent directory of {}".format(manifest_map[dest].origin), - ) - - return sorted(manifest_map.values(), key = lambda x: x.dest) - -def main(args): - unix_ts = max(ZIP_EPOCH, args.timestamp) - if args.stamp_from: - unix_ts = build_info.get_timestamp(args.stamp_from) - ts = parse_date(unix_ts) - default_mode = None - if args.mode: - default_mode = int(args.mode, 8) - compression_level = int(args.compression_level) - - manifest = _load_manifest(args.directory, args.manifest) - with ZipWriter( - args.output, time_stamp=ts, default_mode=default_mode, compression_type=args.compression_type, compression_level=compression_level) as zip_out: - for entry in manifest: - zip_out.add_manifest_entry(entry) - - -if __name__ == '__main__': - arg_parser = _create_argument_parser() - main(arg_parser.parse_args()) diff --git a/pkg/private/zip/zip.bzl b/pkg/private/zip/zip.bzl deleted file mode 100644 index e038b48..0000000 --- a/pkg/private/zip/zip.bzl +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Zip archive creation rule and associated logic.""" - -load("//pkg:path.bzl", "compute_data_path", "dest_path") -load( - "//pkg:providers.bzl", - "PackageVariablesInfo", -) -load( - "//pkg/private:util.bzl", - "setup_output_files", - "substitute_package_variables", -) -load( - "//pkg/private:pkg_files.bzl", - "add_label_list", - "write_manifest", -) - -_stamp_condition = Label("//pkg/private:private_stamp_detect") - -def _pkg_zip_impl(ctx): - outputs, output_file, output_name = setup_output_files(ctx) - - args = ctx.actions.args() - args.add("-o", output_file.path) - args.add("-d", substitute_package_variables(ctx, ctx.attr.package_dir)) - args.add("-t", ctx.attr.timestamp) - args.add("-m", ctx.attr.mode) - args.add("-c", str(ctx.attr.compression_type)) - args.add("-l", ctx.attr.compression_level) - inputs = [] - if ctx.attr.stamp == 1 or (ctx.attr.stamp == -1 and - ctx.attr.private_stamp_detect): - args.add("--stamp_from", ctx.version_file.path) - inputs.append(ctx.version_file) - - data_path = compute_data_path(ctx, ctx.attr.strip_prefix) - data_path_without_prefix = compute_data_path(ctx, ".") - - content_map = {} # content handled in the manifest - file_deps = [] # list of Depsets needed by srcs - add_label_list(ctx, content_map, file_deps, srcs = ctx.attr.srcs) - - manifest_file = ctx.actions.declare_file(ctx.label.name + ".manifest") - inputs.append(manifest_file) - write_manifest(ctx, manifest_file, content_map) - args.add("--manifest", manifest_file.path) - args.set_param_file_format("multiline") - args.use_param_file("@%s") - - all_inputs = depset(direct = inputs, transitive = file_deps) - - ctx.actions.run( - mnemonic = "PackageZip", - inputs = all_inputs, - executable = ctx.executable._build_zip, - arguments = [args], - outputs = [output_file], - env = { - "LANG": "en_US.UTF-8", - "LC_CTYPE": "UTF-8", - "PYTHONIOENCODING": "UTF-8", - "PYTHONUTF8": "1", - }, - use_default_shell_env = True, - ) - return [ - DefaultInfo( - files = depset([output_file]), - runfiles = ctx.runfiles(files = outputs), - ), - ] - -pkg_zip_impl = rule( - implementation = _pkg_zip_impl, - # @unsorted-dict-items - attrs = { - "srcs": attr.label_list( - doc = """List of files that should be included in the archive.""", - allow_files = True, - ), - "mode": attr.string( - doc = """The default mode for all files in the archive.""", - default = "0555", - ), - "package_dir": attr.string( - doc = """Prefix to be prepend to all paths written. -The name may contain variables, same as [package_file_name](#package_file_name)""", - default = "/", - ), - "strip_prefix": attr.string(), - "timestamp": attr.int( - doc = """Time stamp to place on all files in the archive, expressed -as seconds since the Unix Epoch, as per RFC 3339. The default is January 01, -1980, 00:00 UTC. - -Due to limitations in the format of zip files, values before -Jan 1, 1980 will be rounded up and the precision in the zip file is -limited to a granularity of 2 seconds.""", - default = 315532800, - ), - "compression_level": attr.int( - default = 6, - doc = "The compression level to use, 1 is the fastest, 9 gives the smallest results. 0 skips compression, depending on the method used" - ), - "compression_type": attr.string( - default = "deflated", - doc = """The compression to use. Note that lzma and bzip2 might not be supported by all readers. -The list of compressions is the same as Python's ZipFile: https://docs.python.org/3/library/zipfile.html#zipfile.ZIP_STORED""", - values = ["deflated", "lzma", "bzip2", "stored"] - ), - - # Common attributes - "out": attr.output( - doc = """output file name. Default: name + ".zip".""", - mandatory = True, - ), - "package_file_name": attr.string(doc = "See [Common Attributes](#package_file_name)"), - "package_variables": attr.label( - doc = "See [Common Attributes](#package_variables)", - providers = [PackageVariablesInfo], - ), - "stamp": attr.int( - doc = """Enable file time stamping. Possible values: -<li>stamp = 1: Use the time of the build as the modification time of each file in the archive. -<li>stamp = 0: Use an "epoch" time for the modification time of each file. This gives good build result caching. -<li>stamp = -1: Control the chosen modification time using the --[no]stamp flag. -""", - default = 0, - ), - - "allow_duplicates_with_different_content": attr.bool( - default=True, - doc="""If true, will allow you to reference multiple pkg_* which conflict -(writing different content or metadata to the same destination). -Such behaviour is always incorrect, but we provide a flag to support it in case old -builds were accidentally doing it. Never explicitly set this to true for new code. -""" - ), - # Is --stamp set on the command line? - # TODO(https://github.com/bazelbuild/rules_pkg/issues/340): Remove this. - "private_stamp_detect": attr.bool(default = False), - - # Implicit dependencies. - "_build_zip": attr.label( - default = Label("//pkg/private/zip:build_zip"), - cfg = "exec", - executable = True, - allow_files = True, - ), - }, -) - -def pkg_zip(name, out = None, **kwargs): - """Creates a .zip file. - - @wraps(pkg_zip_impl) - - Args: - out: output file name. Default: name + ".zip". - """ - if not out: - out = name + ".zip" - pkg_zip_impl( - name = name, - out = out, - private_stamp_detect = select({ - _stamp_condition: True, - "//conditions:default": False, - }), - **kwargs - ) |