diff options
Diffstat (limited to 'pkg/private/zip')
| -rw-r--r-- | pkg/private/zip/BUILD | 60 | ||||
| -rw-r--r-- | pkg/private/zip/__init__.py | 0 | ||||
| -rw-r--r-- | pkg/private/zip/build_zip.py | 305 | ||||
| -rw-r--r-- | pkg/private/zip/zip.bzl | 185 |
4 files changed, 0 insertions, 550 deletions
diff --git a/pkg/private/zip/BUILD b/pkg/private/zip/BUILD deleted file mode 100644 index 4823e90..0000000 --- a/pkg/private/zip/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""rules_pkg internal code. - -All interfaces are subject to change at any time. -""" - -load("@rules_python//python:defs.bzl", "py_binary") - -package(default_applicable_licenses = ["//:license"]) - -filegroup( - name = "standard_package", - srcs = [ - "BUILD", - ] + glob([ - "*.bzl", - "*.py", - ]), - visibility = [ - "//distro:__pkg__", - "//pkg:__pkg__", - ], -) - -exports_files( - glob([ - "*.bzl", - ]), - visibility = [ - "//distro:__pkg__", - "//doc_build:__pkg__", - "//pkg:__pkg__", - ], -) - -py_binary( - name = "build_zip", - srcs = ["build_zip.py"], - imports = ["../../.."], - python_version = "PY3", - srcs_version = "PY3", - visibility = ["//visibility:public"], - deps = [ - "//pkg/private:build_info", - "//pkg/private:helpers", - "//pkg/private:manifest", - ], -) diff --git a/pkg/private/zip/__init__.py b/pkg/private/zip/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/pkg/private/zip/__init__.py +++ /dev/null diff --git a/pkg/private/zip/build_zip.py b/pkg/private/zip/build_zip.py deleted file mode 100644 index ca48a08..0000000 --- a/pkg/private/zip/build_zip.py +++ /dev/null @@ -1,305 +0,0 @@ -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This tool builds zip files from a list of inputs.""" - -import argparse -import datetime -import logging -import os -import sys -import zipfile - -from pkg.private import build_info -from pkg.private import manifest - -ZIP_EPOCH = 315532800 - -# Unix dir bit and Windows dir bit. Magic from zip spec -UNIX_DIR_BIT = 0o40000 -MSDOS_DIR_BIT = 0x10 -UNIX_SYMLINK_BIT = 0o120000 - -def _create_argument_parser(): - """Creates the command line arg parser.""" - parser = argparse.ArgumentParser(description='create a zip file', - fromfile_prefix_chars='@') - parser.add_argument('-o', '--output', type=str, - help='The output zip file path.') - parser.add_argument( - '-d', '--directory', type=str, default='/', - help='An absolute path to use as a prefix for all files in the zip.') - parser.add_argument( - '-t', '--timestamp', type=int, default=ZIP_EPOCH, - help='The unix time to use for files added into the zip. values prior to' - ' Jan 1, 1980 are ignored.') - parser.add_argument('--stamp_from', default='', - help='File to find BUILD_STAMP in') - parser.add_argument( - '-m', '--mode', - help='The file system mode to use for files added into the zip.') - parser.add_argument( - '-c', '--compression_type', - help='The compression type to use') - parser.add_argument( - '-l', '--compression_level', - help='The compression level to use') - parser.add_argument('--manifest', - help='manifest of contents to add to the layer.', - required=True) - parser.add_argument( - 'files', type=str, nargs='*', - help='Files to be added to the zip, in the form of {srcpath}={dstpath}.') - return parser - - -def _combine_paths(left, right): - result = left.rstrip('/') + '/' + right.lstrip('/') - - # important: remove leading /'s: the zip format spec says paths should never - # have a leading slash, but Python will happily do this. The built-in zip - # tool in Windows will complain that such a zip file is invalid. - return result.lstrip('/') - - -def parse_date(ts): - ts = datetime.datetime.utcfromtimestamp(ts) - return (ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second) - - -class ZipWriter(object): - - def __init__(self, output_path: str, time_stamp: int, default_mode: int, compression_type: str, compression_level: int): - """Create a writer. - - You must close() after use or use in a 'with' statement. - - Args: - output_path: path to write to - time_stamp: time stamp to add to files - default_mode: file mode to use if not specified in the entry. - """ - self.output_path = output_path - self.time_stamp = time_stamp - self.default_mode = default_mode - compressions = { - "deflated": zipfile.ZIP_DEFLATED, - "lzma": zipfile.ZIP_LZMA, - "bzip2": zipfile.ZIP_BZIP2, - "stored": zipfile.ZIP_STORED - } - self.compression_type = compressions[compression_type] - self.compression_level = compression_level - self.zip_file = zipfile.ZipFile(self.output_path, mode='w', compression=self.compression_type) - - def __enter__(self): - return self - - def __exit__(self, t, v, traceback): - self.close() - - def close(self): - self.zip_file.close() - self.zip_file = None - - def writestr(self, entry_info, content: str, compresslevel: int): - if sys.version_info >= (3, 7): - self.zip_file.writestr(entry_info, content, compresslevel=compresslevel) - else: - # Python 3.6 and lower don't support compresslevel - self.zip_file.writestr(entry_info, content) - if compresslevel != 6: - logging.warn("Custom compresslevel is not supported with python < 3.7") - - def make_zipinfo(self, path: str, mode: str): - """Create a Zipinfo. - - Args: - path: file path - mode: file mode - """ - entry_info = zipfile.ZipInfo(filename=path, date_time=self.time_stamp) - # See http://www.pkware.com/documents/casestudies/APPNOTE.TXT - # denotes UTF-8 encoded file name. - entry_info.flag_bits |= 0x800 - - # See: https://trac.edgewall.org/attachment/ticket/8919/ZipDownload.patch - # external_attr is 4 bytes in size. The high order two bytes represent UNIX - # permission and file type bits, while the low order two contain MS-DOS FAT - # file attributes. - if mode: - f_mode = int(mode, 8) - else: - f_mode = self.default_mode - entry_info.external_attr = f_mode << 16 - return entry_info - - def add_manifest_entry(self, entry): - """Add an entry to the zip file. - - Args: - zip_file: ZipFile to write to - entry: manifest entry - """ - - entry_type = entry.type - dest = entry.dest - src = entry.src - mode = entry.mode - user = entry.user - group = entry.group - - # Use the pkg_tar mode/owner remapping as a fallback - dst_path = dest.strip('/') - if entry_type == manifest.ENTRY_IS_DIR and not dst_path.endswith('/'): - dst_path += '/' - entry_info = self.make_zipinfo(path=dst_path, mode=mode) - - if entry_type == manifest.ENTRY_IS_FILE: - entry_info.compress_type = self.compression_type - # Using utf-8 for the file names is for python <3.7 compatibility. - with open(src.encode('utf-8'), 'rb') as src_content: - self.writestr(entry_info, src_content.read(), compresslevel=self.compression_level) - elif entry_type == manifest.ENTRY_IS_DIR: - entry_info.compress_type = zipfile.ZIP_STORED - # Set directory bits - entry_info.external_attr |= (UNIX_DIR_BIT << 16) | MSDOS_DIR_BIT - self.zip_file.writestr(entry_info, '') - elif entry_type == manifest.ENTRY_IS_LINK: - entry_info.compress_type = zipfile.ZIP_STORED - # Set directory bits - entry_info.external_attr |= (UNIX_SYMLINK_BIT << 16) - self.zip_file.writestr(entry_info, src.encode('utf-8')) - elif entry_type == manifest.ENTRY_IS_TREE: - self.add_tree(src, dst_path, mode) - elif entry_type == manifest.ENTRY_IS_EMPTY_FILE: - entry_info.compress_type = zipfile.ZIP_STORED - self.zip_file.writestr(entry_info, '') - else: - raise Exception('Unknown type for manifest entry:', entry) - - def add_tree(self, tree_top: str, destpath: str, mode: int): - """Add a tree artifact to the zip file. - - Args: - tree_top: the top of the tree to add - destpath: the path under which to place the files - mode: if not None, file mode to apply to all files - """ - - # We expect /-style paths. - tree_top = os.path.normpath(tree_top).replace(os.path.sep, '/') - - # Again, we expect /-style paths. - dest = destpath.strip('/') # redundant, dests should never have / here - dest = os.path.normpath(dest).replace(os.path.sep, '/') - # paths should not have a leading ./ - dest = '' if dest == '.' else dest + '/' - - to_write = {} - for root, dirs, files in os.walk(tree_top): - # While `tree_top` uses '/' as a path separator, results returned by - # `os.walk` and `os.path.join` on Windows may not. - root = os.path.normpath(root).replace(os.path.sep, '/') - - rel_path_from_top = root[len(tree_top):].lstrip('/') - if rel_path_from_top: - dest_dir = dest + rel_path_from_top + '/' - else: - dest_dir = dest - to_write[dest_dir] = None - for file in files: - content_path = os.path.abspath(os.path.join(root, file)) - if os.name == "nt": - # "To specify an extended-length path, use the `\\?\` prefix. For - # example, `\\?\D:\very long path`."[1] - # - # [1]: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation - to_write[dest_dir + file] = "\\\\?\\" + content_path - else: - to_write[dest_dir + file] = content_path - - for path in sorted(to_write.keys()): - content_path = to_write[path] - if content_path: - # If mode is unspecified, derive the mode from the file's mode. - if mode is None: - f_mode = "0o755" if os.access(content_path, os.X_OK) else self.default_mode - else: - f_mode = mode - entry_info = self.make_zipinfo(path=path, mode=f_mode) - entry_info.compress_type = self.compression_type - with open(content_path, 'rb') as src: - self.writestr(entry_info, src.read(), compresslevel=self.compression_level) - else: - # Implicitly created directory - dir_path = path - if not dir_path.endswith('/'): - dir_path += '/' - entry_info = self.make_zipinfo(path=dir_path, mode="0o755") - entry_info.compress_type = zipfile.ZIP_STORED - # Set directory bits - entry_info.external_attr |= (UNIX_DIR_BIT << 16) | MSDOS_DIR_BIT - self.zip_file.writestr(entry_info, '') - -def _load_manifest(prefix, manifest_path): - manifest_map = {} - - for entry in manifest.read_entries_from_file(manifest_path): - entry.dest = _combine_paths(prefix, entry.dest) - manifest_map[entry.dest] = entry - - # We modify the dictionary as we're iterating over it, so we need to listify - # the keys here. - manifest_keys = list(manifest_map.keys()) - # Add all parent directories of entries that have not been added explicitly. - for dest in manifest_keys: - parent = dest - # TODO: use pathlib instead of string manipulation? - for _ in range(dest.count("/")): - parent, _, _ = parent.rpartition("/") - if parent and parent not in manifest_map: - manifest_map[parent] = manifest.ManifestEntry( - type = manifest.ENTRY_IS_DIR, - dest = parent, - src = "", - mode = "0o755", - user = None, - group = None, - uid = None, - gid = None, - origin = "parent directory of {}".format(manifest_map[dest].origin), - ) - - return sorted(manifest_map.values(), key = lambda x: x.dest) - -def main(args): - unix_ts = max(ZIP_EPOCH, args.timestamp) - if args.stamp_from: - unix_ts = build_info.get_timestamp(args.stamp_from) - ts = parse_date(unix_ts) - default_mode = None - if args.mode: - default_mode = int(args.mode, 8) - compression_level = int(args.compression_level) - - manifest = _load_manifest(args.directory, args.manifest) - with ZipWriter( - args.output, time_stamp=ts, default_mode=default_mode, compression_type=args.compression_type, compression_level=compression_level) as zip_out: - for entry in manifest: - zip_out.add_manifest_entry(entry) - - -if __name__ == '__main__': - arg_parser = _create_argument_parser() - main(arg_parser.parse_args()) diff --git a/pkg/private/zip/zip.bzl b/pkg/private/zip/zip.bzl deleted file mode 100644 index e038b48..0000000 --- a/pkg/private/zip/zip.bzl +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright 2021 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Zip archive creation rule and associated logic.""" - -load("//pkg:path.bzl", "compute_data_path", "dest_path") -load( - "//pkg:providers.bzl", - "PackageVariablesInfo", -) -load( - "//pkg/private:util.bzl", - "setup_output_files", - "substitute_package_variables", -) -load( - "//pkg/private:pkg_files.bzl", - "add_label_list", - "write_manifest", -) - -_stamp_condition = Label("//pkg/private:private_stamp_detect") - -def _pkg_zip_impl(ctx): - outputs, output_file, output_name = setup_output_files(ctx) - - args = ctx.actions.args() - args.add("-o", output_file.path) - args.add("-d", substitute_package_variables(ctx, ctx.attr.package_dir)) - args.add("-t", ctx.attr.timestamp) - args.add("-m", ctx.attr.mode) - args.add("-c", str(ctx.attr.compression_type)) - args.add("-l", ctx.attr.compression_level) - inputs = [] - if ctx.attr.stamp == 1 or (ctx.attr.stamp == -1 and - ctx.attr.private_stamp_detect): - args.add("--stamp_from", ctx.version_file.path) - inputs.append(ctx.version_file) - - data_path = compute_data_path(ctx, ctx.attr.strip_prefix) - data_path_without_prefix = compute_data_path(ctx, ".") - - content_map = {} # content handled in the manifest - file_deps = [] # list of Depsets needed by srcs - add_label_list(ctx, content_map, file_deps, srcs = ctx.attr.srcs) - - manifest_file = ctx.actions.declare_file(ctx.label.name + ".manifest") - inputs.append(manifest_file) - write_manifest(ctx, manifest_file, content_map) - args.add("--manifest", manifest_file.path) - args.set_param_file_format("multiline") - args.use_param_file("@%s") - - all_inputs = depset(direct = inputs, transitive = file_deps) - - ctx.actions.run( - mnemonic = "PackageZip", - inputs = all_inputs, - executable = ctx.executable._build_zip, - arguments = [args], - outputs = [output_file], - env = { - "LANG": "en_US.UTF-8", - "LC_CTYPE": "UTF-8", - "PYTHONIOENCODING": "UTF-8", - "PYTHONUTF8": "1", - }, - use_default_shell_env = True, - ) - return [ - DefaultInfo( - files = depset([output_file]), - runfiles = ctx.runfiles(files = outputs), - ), - ] - -pkg_zip_impl = rule( - implementation = _pkg_zip_impl, - # @unsorted-dict-items - attrs = { - "srcs": attr.label_list( - doc = """List of files that should be included in the archive.""", - allow_files = True, - ), - "mode": attr.string( - doc = """The default mode for all files in the archive.""", - default = "0555", - ), - "package_dir": attr.string( - doc = """Prefix to be prepend to all paths written. -The name may contain variables, same as [package_file_name](#package_file_name)""", - default = "/", - ), - "strip_prefix": attr.string(), - "timestamp": attr.int( - doc = """Time stamp to place on all files in the archive, expressed -as seconds since the Unix Epoch, as per RFC 3339. The default is January 01, -1980, 00:00 UTC. - -Due to limitations in the format of zip files, values before -Jan 1, 1980 will be rounded up and the precision in the zip file is -limited to a granularity of 2 seconds.""", - default = 315532800, - ), - "compression_level": attr.int( - default = 6, - doc = "The compression level to use, 1 is the fastest, 9 gives the smallest results. 0 skips compression, depending on the method used" - ), - "compression_type": attr.string( - default = "deflated", - doc = """The compression to use. Note that lzma and bzip2 might not be supported by all readers. -The list of compressions is the same as Python's ZipFile: https://docs.python.org/3/library/zipfile.html#zipfile.ZIP_STORED""", - values = ["deflated", "lzma", "bzip2", "stored"] - ), - - # Common attributes - "out": attr.output( - doc = """output file name. Default: name + ".zip".""", - mandatory = True, - ), - "package_file_name": attr.string(doc = "See [Common Attributes](#package_file_name)"), - "package_variables": attr.label( - doc = "See [Common Attributes](#package_variables)", - providers = [PackageVariablesInfo], - ), - "stamp": attr.int( - doc = """Enable file time stamping. Possible values: -<li>stamp = 1: Use the time of the build as the modification time of each file in the archive. -<li>stamp = 0: Use an "epoch" time for the modification time of each file. This gives good build result caching. -<li>stamp = -1: Control the chosen modification time using the --[no]stamp flag. -""", - default = 0, - ), - - "allow_duplicates_with_different_content": attr.bool( - default=True, - doc="""If true, will allow you to reference multiple pkg_* which conflict -(writing different content or metadata to the same destination). -Such behaviour is always incorrect, but we provide a flag to support it in case old -builds were accidentally doing it. Never explicitly set this to true for new code. -""" - ), - # Is --stamp set on the command line? - # TODO(https://github.com/bazelbuild/rules_pkg/issues/340): Remove this. - "private_stamp_detect": attr.bool(default = False), - - # Implicit dependencies. - "_build_zip": attr.label( - default = Label("//pkg/private/zip:build_zip"), - cfg = "exec", - executable = True, - allow_files = True, - ), - }, -) - -def pkg_zip(name, out = None, **kwargs): - """Creates a .zip file. - - @wraps(pkg_zip_impl) - - Args: - out: output file name. Default: name + ".zip". - """ - if not out: - out = name + ".zip" - pkg_zip_impl( - name = name, - out = out, - private_stamp_detect = select({ - _stamp_condition: True, - "//conditions:default": False, - }), - **kwargs - ) |