From 618ddecd59c3083cac13610c94a429f9f2d7c63d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 11 May 2021 02:26:11 +0000 Subject: [zip] Add fdegros@ and noel@ to OWNERS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:912236 TEST=None Change-Id: I644f48d3b12e7528056e6e6e70993c1a5acf69c6 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2886559 Reviewed-by: Satoru Takabayashi Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#881377} NOKEYCHECK=True GitOrigin-RevId: 3fd8cef93135edc2dc78f73c226fe5d1ec741b12 --- google/OWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/google/OWNERS b/google/OWNERS index 868af3c..411670c 100644 --- a/google/OWNERS +++ b/google/OWNERS @@ -1,3 +1,5 @@ +fdegros@chromium.org +noel@chromium.org satorux@chromium.org # compression_utils* -- cgit v1.2.3 From c3edbeed8ac2232058315f43bde5831b9e4cbb53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 12 May 2021 03:29:59 +0000 Subject: [zip] Minor code refactoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Converted some functions to private ZipWriter methods. No functional changes here. This is just to make upcoming changes easier to review. BUG=chromium:912236 TEST=autoninja -C out/release zlib_unittests && out/release/zlib_unittests Change-Id: Ic69c52d461e84790fb698a430ff753d8953cdcb1 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2886560 Commit-Queue: François Degros Reviewed-by: Noel Gordon Cr-Commit-Position: refs/heads/master@{#881886} NOKEYCHECK=True GitOrigin-RevId: cbb403b9277948416400ad683b69feac6c77af56 --- google/zip_writer.cc | 55 +++++++++++++++++++++------------------------------- google/zip_writer.h | 20 ++++++++++++++++++- 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/google/zip_writer.cc b/google/zip_writer.cc index 6f38d42..5a9082d 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -12,21 +12,15 @@ namespace zip { namespace internal { -namespace { - -// Numbers of pending entries that trigger writting them to the ZIP file. -constexpr size_t kMaxPendingEntriesCount = 50; - -bool AddFileContentToZip(zipFile zip_file, - base::File file, - const base::FilePath& file_path) { +bool ZipWriter::AddFileContent(const base::FilePath& file_path, + base::File file) { int num_bytes; char buf[zip::internal::kZipBufSize]; do { num_bytes = file.ReadAtCurrentPos(buf, zip::internal::kZipBufSize); if (num_bytes > 0) { - if (zipWriteInFileInZip(zip_file, buf, num_bytes) != ZIP_OK) { + if (zipWriteInFileInZip(zip_file_, buf, num_bytes) != ZIP_OK) { DLOG(ERROR) << "Could not write data to zip for path " << file_path.value(); return false; @@ -37,10 +31,9 @@ bool AddFileContentToZip(zipFile zip_file, return true; } -bool OpenNewFileEntry(zipFile zip_file, - const base::FilePath& path, - bool is_directory, - base::Time last_modified) { +bool ZipWriter::OpenNewFileEntry(const base::FilePath& path, + bool is_directory, + base::Time last_modified) { std::string str_path = path.AsUTF8Unsafe(); #if defined(OS_WIN) base::ReplaceSubstringsAfterOffset(&str_path, 0u, "\\", "/"); @@ -48,41 +41,34 @@ bool OpenNewFileEntry(zipFile zip_file, if (is_directory) str_path += "/"; - return zip::internal::ZipOpenNewFileInZip(zip_file, str_path, last_modified); + return zip::internal::ZipOpenNewFileInZip(zip_file_, str_path, last_modified); } -bool CloseNewFileEntry(zipFile zip_file) { - return zipCloseFileInZip(zip_file) == ZIP_OK; +bool ZipWriter::CloseNewFileEntry() { + return zipCloseFileInZip(zip_file_) == ZIP_OK; } -bool AddFileEntryToZip(zipFile zip_file, - const base::FilePath& path, - base::File file) { +bool ZipWriter::AddFileEntry(const base::FilePath& path, base::File file) { base::File::Info file_info; if (!file.GetInfo(&file_info)) return false; - if (!OpenNewFileEntry(zip_file, path, /*is_directory=*/false, - file_info.last_modified)) + if (!OpenNewFileEntry(path, /*is_directory=*/false, file_info.last_modified)) return false; - bool success = AddFileContentToZip(zip_file, std::move(file), path); - if (!CloseNewFileEntry(zip_file)) + bool success = AddFileContent(path, std::move(file)); + if (!CloseNewFileEntry()) return false; return success; } -bool AddDirectoryEntryToZip(zipFile zip_file, - const base::FilePath& path, - base::Time last_modified) { - return OpenNewFileEntry(zip_file, path, /*is_directory=*/true, - last_modified) && - CloseNewFileEntry(zip_file); +bool ZipWriter::AddDirectoryEntry(const base::FilePath& path, + base::Time last_modified) { + return OpenNewFileEntry(path, /*is_directory=*/true, last_modified) && + CloseNewFileEntry(); } -} // namespace - #if defined(OS_POSIX) // static std::unique_ptr ZipWriter::CreateWithFd( @@ -144,6 +130,9 @@ bool ZipWriter::Close() { } bool ZipWriter::FlushEntriesIfNeeded(bool force) { + // Numbers of pending entries that triggers writing them to the ZIP file. + const size_t kMaxPendingEntriesCount = 50; + if (pending_entries_.size() < kMaxPendingEntriesCount && !force) return true; @@ -174,7 +163,7 @@ bool ZipWriter::FlushEntriesIfNeeded(bool force) { const base::FilePath& absolute_path = absolute_paths[i]; base::File file = std::move(files[i]); if (file.IsValid()) { - if (!AddFileEntryToZip(zip_file_, relative_path, std::move(file))) { + if (!AddFileEntry(relative_path, std::move(file))) { LOG(ERROR) << "Failed to write file " << relative_path.value() << " to ZIP file."; return false; @@ -189,7 +178,7 @@ bool ZipWriter::FlushEntriesIfNeeded(bool force) { return false; } DCHECK(file_accessor_->DirectoryExists(absolute_path)); - if (!AddDirectoryEntryToZip(zip_file_, relative_path, last_modified)) { + if (!AddDirectoryEntry(relative_path, last_modified)) { LOG(ERROR) << "Failed to write directory " << relative_path.value() << " to ZIP file."; return false; diff --git a/google/zip_writer.h b/google/zip_writer.h index bd2a727..53e47fb 100644 --- a/google/zip_writer.h +++ b/google/zip_writer.h @@ -9,6 +9,7 @@ #include #include "base/files/file_path.h" +#include "base/time/time.h" #include "build/build_config.h" #include "third_party/zlib/google/zip.h" @@ -65,6 +66,23 @@ class ZipWriter { // to |root_dir| specified in the Create method. bool AddEntries(const std::vector& paths); + // Adds file content to currently open file entry. + bool AddFileContent(const base::FilePath& path, base::File file); + + // Adds a file entry (including file contents). + bool AddFileEntry(const base::FilePath& path, base::File file); + + // Adds a directory entry. + bool AddDirectoryEntry(const base::FilePath& path, base::Time last_modified); + + // Opens a file or directory entry. + bool OpenNewFileEntry(const base::FilePath& path, + bool is_directory, + base::Time last_modified); + + // Closes the currently open entry. + bool CloseNewFileEntry(); + // Closes the ZIP file. // Returns true if successful, false otherwise (typically if an entry failed // to be written). @@ -88,4 +106,4 @@ class ZipWriter { } // namespace internal } // namespace zip -#endif // THIRD_PARTY_ZLIB_GOOGLE_ZIP_WRITER_H_ \ No newline at end of file +#endif // THIRD_PARTY_ZLIB_GOOGLE_ZIP_WRITER_H_ -- cgit v1.2.3 From 4f6af9a0d691f2f735edf78d34ad377d359aa1b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 12 May 2021 07:55:25 +0000 Subject: [zip] Fix error handling in ZipWriter::AddFileContent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't silently ignore read errors. BUG=chromium:1207740, chromium:912236 TEST=autoninja -C out/release zlib_unittests && out/release/zlib_unittests Change-Id: Idb47530204d6c5ce68eec50e29c88c7948fdf2d6 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2886471 Commit-Queue: François Degros Commit-Queue: Noel Gordon Reviewed-by: Noel Gordon Cr-Commit-Position: refs/heads/master@{#881949} NOKEYCHECK=True GitOrigin-RevId: 6ca704d107ff48053f3689be9390564f9f9b8eb6 --- google/zip_writer.cc | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/google/zip_writer.cc b/google/zip_writer.cc index 5a9082d..c5930cf 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -12,23 +12,25 @@ namespace zip { namespace internal { -bool ZipWriter::AddFileContent(const base::FilePath& file_path, - base::File file) { - int num_bytes; +bool ZipWriter::AddFileContent(const base::FilePath& path, base::File file) { char buf[zip::internal::kZipBufSize]; - do { - num_bytes = file.ReadAtCurrentPos(buf, zip::internal::kZipBufSize); - - if (num_bytes > 0) { - if (zipWriteInFileInZip(zip_file_, buf, num_bytes) != ZIP_OK) { - DLOG(ERROR) << "Could not write data to zip for path " - << file_path.value(); - return false; - } + while (true) { + const int num_bytes = + file.ReadAtCurrentPos(buf, zip::internal::kZipBufSize); + + if (num_bytes < 0) { + DPLOG(ERROR) << "Cannot read file '" << path << "'"; + return false; } - } while (num_bytes > 0); - return true; + if (num_bytes == 0) + return true; + + if (zipWriteInFileInZip(zip_file_, buf, num_bytes) != ZIP_OK) { + DLOG(ERROR) << "Cannot write data from file '" << path << "' to ZIP"; + return false; + } + } } bool ZipWriter::OpenNewFileEntry(const base::FilePath& path, -- cgit v1.2.3 From c830e4eaa2b25afa65d2dde0f1eb625a1f6c1a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 12 May 2021 11:39:24 +0000 Subject: [zip] Optimize ZipWrite::WriteEntries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed FlushEntriesIfNeeded. There is no need to copy vectors of FilePaths over and over again. This leads to quadratic behavior. Fixed this. Passed FilePaths as a base::span without any actual copy. BUG=chromium:1207746, chromium:912236 TEST=autoninja -C out/release zlib_unittests && out/release/zlib_unittests Change-Id: Id663f23147ae6cd2e02c71108a60eded45cd156f Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2890489 Commit-Queue: François Degros Reviewed-by: Noel Gordon Cr-Commit-Position: refs/heads/master@{#881993} NOKEYCHECK=True GitOrigin-RevId: 921dee185c6ec89e18b413b02dd32873aab91d2b --- google/zip_writer.cc | 82 +++++++++++++++++++++++----------------------------- google/zip_writer.h | 23 +++++---------- 2 files changed, 44 insertions(+), 61 deletions(-) diff --git a/google/zip_writer.cc b/google/zip_writer.cc index c5930cf..1cea685 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -110,84 +110,74 @@ ZipWriter::ZipWriter(zipFile zip_file, FileAccessor* file_accessor) : zip_file_(zip_file), root_dir_(root_dir), file_accessor_(file_accessor) {} -ZipWriter::~ZipWriter() { - DCHECK(pending_entries_.empty()); -} +ZipWriter::~ZipWriter() {} -bool ZipWriter::WriteEntries(const std::vector& paths) { +bool ZipWriter::WriteEntries(Paths paths) { return AddEntries(paths) && Close(); } -bool ZipWriter::AddEntries(const std::vector& paths) { - DCHECK(zip_file_); - pending_entries_.insert(pending_entries_.end(), paths.begin(), paths.end()); - return FlushEntriesIfNeeded(/*force=*/false); -} - bool ZipWriter::Close() { - bool success = FlushEntriesIfNeeded(/*force=*/true) && - zipClose(zip_file_, nullptr) == ZIP_OK; + const bool success = zipClose(zip_file_, nullptr) == ZIP_OK; zip_file_ = nullptr; return success; } -bool ZipWriter::FlushEntriesIfNeeded(bool force) { - // Numbers of pending entries that triggers writing them to the ZIP file. - const size_t kMaxPendingEntriesCount = 50; - - if (pending_entries_.size() < kMaxPendingEntriesCount && !force) - return true; - - while (pending_entries_.size() >= kMaxPendingEntriesCount || - (force && !pending_entries_.empty())) { - size_t entry_count = - std::min(pending_entries_.size(), kMaxPendingEntriesCount); - std::vector relative_paths; - std::vector absolute_paths; - relative_paths.insert(relative_paths.begin(), pending_entries_.begin(), - pending_entries_.begin() + entry_count); - for (auto iter = pending_entries_.begin(); - iter != pending_entries_.begin() + entry_count; ++iter) { - // The FileAccessor requires absolute paths. - absolute_paths.push_back(root_dir_.Append(*iter)); +bool ZipWriter::AddEntries(Paths paths) { + // Constructed outside the loop in order to reuse its internal buffer. + std::vector absolute_paths; + + while (!paths.empty()) { + // Work with chunks of 50 paths at most. + const size_t n = std::min(paths.size(), 50); + const Paths relative_paths = paths.subspan(0, n); + paths = paths.subspan(n, paths.size() - n); + + // FileAccessor requires absolute paths. + absolute_paths.clear(); + absolute_paths.reserve(n); + for (const base::FilePath& relative_path : relative_paths) { + absolute_paths.push_back(root_dir_.Append(relative_path)); } - pending_entries_.erase(pending_entries_.begin(), - pending_entries_.begin() + entry_count); + + DCHECK_EQ(relative_paths.size(), n); + DCHECK_EQ(absolute_paths.size(), n); // We don't know which paths are files and which ones are directories, and - // we want to avoid making a call to file_accessor_ for each entry. Open the - // files instead, invalid files are returned for directories. + // we want to avoid making a call to file_accessor_ for each entry. Try to + // open all of the paths as files. We'll get invalid file descriptors for + // directories. std::vector files = file_accessor_->OpenFilesForReading(absolute_paths); - DCHECK_EQ(files.size(), relative_paths.size()); - for (size_t i = 0; i < files.size(); i++) { + DCHECK_EQ(files.size(), n); + + for (size_t i = 0; i < n; i++) { const base::FilePath& relative_path = relative_paths[i]; const base::FilePath& absolute_path = absolute_paths[i]; - base::File file = std::move(files[i]); + base::File& file = files[i]; + if (file.IsValid()) { if (!AddFileEntry(relative_path, std::move(file))) { - LOG(ERROR) << "Failed to write file " << relative_path.value() - << " to ZIP file."; + LOG(ERROR) << "Cannot add file '" << relative_path << "' to ZIP"; return false; } } else { - // Missing file or directory case. - base::Time last_modified = + // Either directory or missing file. + const base::Time last_modified = file_accessor_->GetLastModifiedTime(absolute_path); if (last_modified.is_null()) { - LOG(ERROR) << "Failed to write entry " << relative_path.value() - << " to ZIP file."; + LOG(ERROR) << "Missing file or directory '" << relative_path << "'"; return false; } + DCHECK(file_accessor_->DirectoryExists(absolute_path)); if (!AddDirectoryEntry(relative_path, last_modified)) { - LOG(ERROR) << "Failed to write directory " << relative_path.value() - << " to ZIP file."; + LOG(ERROR) << "Cannot add directory '" << relative_path << "' to ZIP"; return false; } } } } + return true; } diff --git a/google/zip_writer.h b/google/zip_writer.h index 53e47fb..e4c7270 100644 --- a/google/zip_writer.h +++ b/google/zip_writer.h @@ -8,6 +8,7 @@ #include #include +#include "base/containers/span.h" #include "base/files/file_path.h" #include "base/time/time.h" #include "build/build_config.h" @@ -32,7 +33,7 @@ namespace internal { class ZipWriter { public: // Creates a writer that will write a ZIP file to |zip_file_fd|/|zip_file| -// and which entries (specifies with AddEntries) are relative to |root_dir|. +// and which entries (specified with WriteEntries) are relative to |root_dir|. // All file reads are performed using |file_accessor|. #if defined(OS_POSIX) static std::unique_ptr CreateWithFd(int zip_file_fd, @@ -44,27 +45,22 @@ class ZipWriter { FileAccessor* file_accessor); ~ZipWriter(); - // Writes the files at |paths| to the ZIP file and closes this Zip file. - // Note that the the FilePaths must be relative to |root_dir| specified in the + using Paths = base::span; + + // Writes the files at |paths| to the ZIP file and closes this ZIP file. + // The file paths must be relative to |root_dir| specified in the // Create method. // Returns true if all entries were written successfuly. - bool WriteEntries(const std::vector& paths); + bool WriteEntries(Paths paths); private: ZipWriter(zipFile zip_file, const base::FilePath& root_dir, FileAccessor* file_accessor); - // Writes the pending entries to the ZIP file if there are at least - // |kMaxPendingEntriesCount| of them. If |force| is true, all pending entries - // are written regardless of how many there are. - // Returns false if writing an entry fails, true if no entry was written or - // there was no error writing entries. - bool FlushEntriesIfNeeded(bool force); - // Adds the files at |paths| to the ZIP file. These FilePaths must be relative // to |root_dir| specified in the Create method. - bool AddEntries(const std::vector& paths); + bool AddEntries(Paths paths); // Adds file content to currently open file entry. bool AddFileContent(const base::FilePath& path, base::File file); @@ -88,9 +84,6 @@ class ZipWriter { // to be written). bool Close(); - // The entries that have been added but not yet written to the ZIP file. - std::vector pending_entries_; - // The actual zip file. zipFile zip_file_; -- cgit v1.2.3 From e3c17da190132133a159a1247dc0f59fb7120c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Thu, 13 May 2021 06:40:24 +0000 Subject: [zip] Fix potential resource leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ZipWriter closes the ZIP file even if AddEntries() failed. BUG=chromium:1208254, chromium:912236 TEST=autoninja -C out/release zlib_unittests && out/release/zlib_unittests Change-Id: I614192f7537939d3e6cf272da10d47574a61b135 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2890319 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#882474} NOKEYCHECK=True GitOrigin-RevId: c37e14941bdb09cb02d906834bab1953b172c213 --- google/zip_writer.cc | 13 ++++++++++--- google/zip_writer.h | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/google/zip_writer.cc b/google/zip_writer.cc index 1cea685..c151cc8 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -80,10 +80,12 @@ std::unique_ptr ZipWriter::CreateWithFd( DCHECK(zip_file_fd != base::kInvalidPlatformFile); zipFile zip_file = internal::OpenFdForZipping(zip_file_fd, APPEND_STATUS_CREATE); + if (!zip_file) { - DLOG(ERROR) << "Couldn't create ZIP file for FD " << zip_file_fd; + DLOG(ERROR) << "Cannot create ZIP file for FD " << zip_file_fd; return nullptr; } + return std::unique_ptr( new ZipWriter(zip_file, root_dir, file_accessor)); } @@ -97,10 +99,12 @@ std::unique_ptr ZipWriter::Create( DCHECK(!zip_file_path.empty()); zipFile zip_file = internal::OpenForZipping(zip_file_path.AsUTF8Unsafe(), APPEND_STATUS_CREATE); + if (!zip_file) { - DLOG(ERROR) << "Couldn't create ZIP file at path " << zip_file_path; + DLOG(ERROR) << "Cannot create ZIP file '" << zip_file_path << "'"; return nullptr; } + return std::unique_ptr( new ZipWriter(zip_file, root_dir, file_accessor)); } @@ -110,7 +114,10 @@ ZipWriter::ZipWriter(zipFile zip_file, FileAccessor* file_accessor) : zip_file_(zip_file), root_dir_(root_dir), file_accessor_(file_accessor) {} -ZipWriter::~ZipWriter() {} +ZipWriter::~ZipWriter() { + if (zip_file_) + zipClose(zip_file_, nullptr); +} bool ZipWriter::WriteEntries(Paths paths) { return AddEntries(paths) && Close(); diff --git a/google/zip_writer.h b/google/zip_writer.h index e4c7270..0dc4c6c 100644 --- a/google/zip_writer.h +++ b/google/zip_writer.h @@ -54,6 +54,7 @@ class ZipWriter { bool WriteEntries(Paths paths); private: + // Takes ownership of |zip_file|. ZipWriter(zipFile zip_file, const base::FilePath& root_dir, FileAccessor* file_accessor); -- cgit v1.2.3 From d7ba837b2b671783190f63d6969452fbb57780a6 Mon Sep 17 00:00:00 2001 From: Noel Gordon Date: Fri, 14 May 2021 01:24:47 +0000 Subject: [zlib] Add myself to OWNERS Bug: None Change-Id: Id6919285b4a05b916a2bedb827f8956ea8b15774 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2888448 Reviewed-by: Leon Scroggins Reviewed-by: Adenilson Cavalcanti Commit-Queue: Noel Gordon Cr-Commit-Position: refs/heads/master@{#882796} NOKEYCHECK=True GitOrigin-RevId: fc2977b28707dd6fa5e3e42eb9bb5f688b65e4ab --- OWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS b/OWNERS index 632b3f9..0bfa9fb 100644 --- a/OWNERS +++ b/OWNERS @@ -2,4 +2,5 @@ agl@chromium.org cavalcantii@chromium.org cblume@chromium.org mtklein@google.com +noel@chromium.org scroggo@google.com -- cgit v1.2.3 From 82aff2d773d698da4f108572020e0b4abea13472 Mon Sep 17 00:00:00 2001 From: Noel Gordon Date: Fri, 14 May 2021 02:25:04 +0000 Subject: [zlib] Build minizip zip and unzip tools Minizip is a library provided by //third_party/zlib. Its zip and unzip tools can be built in a developer checkout for testing purposes with: autoninja -C out/Release minizip_bin autoninja -C out/Release miniunz_bin Add GN build rules for these files. Patch minizip and miniunz tools so they compile (all bots). Add a patch file and minizip.md for usage. Bug: 1207895 Change-Id: Ia2e1a920445414fed78e14d9195e484f209cf089 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2887385 Commit-Queue: Noel Gordon Reviewed-by: Adenilson Cavalcanti Cr-Commit-Position: refs/heads/master@{#882824} NOKEYCHECK=True GitOrigin-RevId: 741c3736b5f1a015a6d34bd8393f3074313b4466 --- BUILD.gn | 40 +++++++++++++++ contrib/minizip/miniunz.c | 11 ++--- contrib/minizip/minizip.c | 4 +- contrib/minizip/minizip.md | 9 ++++ patches/0008-minizip-zip-unzip-tools.patch | 79 ++++++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 contrib/minizip/minizip.md create mode 100644 patches/0008-minizip-zip-unzip-tools.patch diff --git a/BUILD.gn b/BUILD.gn index d64cb38..c7dfafa 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -419,6 +419,46 @@ executable("zlib_bench") { configs += [ "//build/config/compiler:no_chromium_code" ] } +executable("minizip_bin") { + include_dirs = [ "." ] + + sources = [ "contrib/minizip/minizip.c" ] + + if (is_clang) { + cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] + } + + if (!is_debug) { + configs -= [ "//build/config/compiler:default_optimization" ] + configs += [ "//build/config/compiler:optimize_speed" ] + } + + deps = [ ":minizip" ] + + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] +} + +executable("miniunz_bin") { + include_dirs = [ "." ] + + sources = [ "contrib/minizip/miniunz.c" ] + + if (is_clang) { + cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] + } + + if (!is_debug) { + configs -= [ "//build/config/compiler:default_optimization" ] + configs += [ "//build/config/compiler:optimize_speed" ] + } + + deps = [ ":minizip" ] + + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] +} + if (build_with_chromium) { test("zlib_unittests") { testonly = true diff --git a/contrib/minizip/miniunz.c b/contrib/minizip/miniunz.c index 3d65401..74d1a92 100644 --- a/contrib/minizip/miniunz.c +++ b/contrib/minizip/miniunz.c @@ -27,7 +27,7 @@ #endif #endif -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(__Fuchsia__) // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions #define FOPEN_FUNC(filename, mode) fopen(filename, mode) #define FTELLO_FUNC(stream) ftello(stream) @@ -45,6 +45,7 @@ #include #include #include +#include #ifdef _WIN32 # include @@ -97,7 +98,7 @@ void change_file_date(filename,dosdate,tmu_date) SetFileTime(hFile,&ftm,&ftLastAcc,&ftm); CloseHandle(hFile); #else -#ifdef unix || __APPLE__ +#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) struct utimbuf ut; struct tm newdate; newdate.tm_sec = tmu_date.tm_sec; @@ -125,11 +126,9 @@ int mymkdir(dirname) const char* dirname; { int ret=0; -#ifdef _WIN32 +#if defined(_WIN32) ret = _mkdir(dirname); -#elif unix - ret = mkdir (dirname,0775); -#elif __APPLE__ +#elif defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) ret = mkdir (dirname,0775); #endif return ret; diff --git a/contrib/minizip/minizip.c b/contrib/minizip/minizip.c index 4288962..1624bd4 100644 --- a/contrib/minizip/minizip.c +++ b/contrib/minizip/minizip.c @@ -28,7 +28,7 @@ #endif #endif -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(__Fuchsia__) // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions #define FOPEN_FUNC(filename, mode) fopen(filename, mode) #define FTELLO_FUNC(stream) ftello(stream) @@ -94,7 +94,7 @@ uLong filetime(f, tmzip, dt) return ret; } #else -#ifdef unix || __APPLE__ +#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) uLong filetime(f, tmzip, dt) char *f; /* name of file to get info on */ tm_zip *tmzip; /* return value: access, modific. and creation times */ diff --git a/contrib/minizip/minizip.md b/contrib/minizip/minizip.md new file mode 100644 index 0000000..9f15dd2 --- /dev/null +++ b/contrib/minizip/minizip.md @@ -0,0 +1,9 @@ +Minizip is a library provided by //third_party/zlib [1]. Its zip and unzip +tools can be built in a developer checkout for testing purposes with: + +```shell + autoninja -C out/Release minizip_bin + autoninja -C out/Release miniunz_bin +``` + +[1] Upstream is https://github.com/madler/zlib/tree/master/contrib/minizip diff --git a/patches/0008-minizip-zip-unzip-tools.patch b/patches/0008-minizip-zip-unzip-tools.patch new file mode 100644 index 0000000..8908907 --- /dev/null +++ b/patches/0008-minizip-zip-unzip-tools.patch @@ -0,0 +1,79 @@ +From ef2b67c03bc6f1395021f3583a3aaddf74976039 Mon Sep 17 00:00:00 2001 +From: Noel Gordon +Date: Wed, 12 May 2021 21:55:34 +1000 +Subject: [PATCH] Build minizip zip and unzip tools + +--- + third_party/zlib/contrib/minizip/miniunz.c | 11 +++++------ + third_party/zlib/contrib/minizip/minizip.c | 4 ++-- + 2 files changed, 7 insertions(+), 8 deletions(-) + +diff --git a/third_party/zlib/contrib/minizip/miniunz.c b/third_party/zlib/contrib/minizip/miniunz.c +index 3d65401be5cd..74d1a92739c0 100644 +--- a/third_party/zlib/contrib/minizip/miniunz.c ++++ b/third_party/zlib/contrib/minizip/miniunz.c +@@ -27,7 +27,7 @@ + #endif + #endif + +-#ifdef __APPLE__ ++#if defined(__APPLE__) || defined(__Fuchsia__) + // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions + #define FOPEN_FUNC(filename, mode) fopen(filename, mode) + #define FTELLO_FUNC(stream) ftello(stream) +@@ -45,6 +45,7 @@ + #include + #include + #include ++#include + + #ifdef _WIN32 + # include +@@ -97,7 +98,7 @@ void change_file_date(filename,dosdate,tmu_date) + SetFileTime(hFile,&ftm,&ftLastAcc,&ftm); + CloseHandle(hFile); + #else +-#ifdef unix || __APPLE__ ++#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) + struct utimbuf ut; + struct tm newdate; + newdate.tm_sec = tmu_date.tm_sec; +@@ -125,11 +126,9 @@ int mymkdir(dirname) + const char* dirname; + { + int ret=0; +-#ifdef _WIN32 ++#if defined(_WIN32) + ret = _mkdir(dirname); +-#elif unix +- ret = mkdir (dirname,0775); +-#elif __APPLE__ ++#elif defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) + ret = mkdir (dirname,0775); + #endif + return ret; +diff --git a/third_party/zlib/contrib/minizip/minizip.c b/third_party/zlib/contrib/minizip/minizip.c +index 4288962ecef0..1624bd4b2955 100644 +--- a/third_party/zlib/contrib/minizip/minizip.c ++++ b/third_party/zlib/contrib/minizip/minizip.c +@@ -28,7 +28,7 @@ + #endif + #endif + +-#ifdef __APPLE__ ++#if defined(__APPLE__) || defined(__Fuchsia__) + // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions + #define FOPEN_FUNC(filename, mode) fopen(filename, mode) + #define FTELLO_FUNC(stream) ftello(stream) +@@ -94,7 +94,7 @@ uLong filetime(f, tmzip, dt) + return ret; + } + #else +-#ifdef unix || __APPLE__ ++#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) + uLong filetime(f, tmzip, dt) + char *f; /* name of file to get info on */ + tm_zip *tmzip; /* return value: access, modific. and creation times */ +-- +2.31.1.607.g51e8a6a459-goog + -- cgit v1.2.3 From 3e6d3450728da422e14750860a479756948b7e3a Mon Sep 17 00:00:00 2001 From: Maggie Cai Date: Fri, 14 May 2021 02:53:39 +0000 Subject: Revert "[zlib] Build minizip zip and unzip tools" This reverts commit 741c3736b5f1a015a6d34bd8393f3074313b4466. Reason for revert: This CL is the likely cause for the tree closure with build compile failure in https://ci.chromium.org/ui/p/chromium/builders/ci/android-archive-rel/20566/overview. Error message: https://logs.chromium.org/logs/chromium/buildbucket/cr-buildbucket.appspot.com/8847293948719164656/+/steps/compile/0/stdout Original change's description: > [zlib] Build minizip zip and unzip tools > > Minizip is a library provided by //third_party/zlib. Its zip and unzip > tools can be built in a developer checkout for testing purposes with: > > autoninja -C out/Release minizip_bin > autoninja -C out/Release miniunz_bin > > Add GN build rules for these files. Patch minizip and miniunz tools so > they compile (all bots). Add a patch file and minizip.md for usage. > > Bug: 1207895 > Change-Id: Ia2e1a920445414fed78e14d9195e484f209cf089 > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2887385 > Commit-Queue: Noel Gordon > Reviewed-by: Adenilson Cavalcanti > Cr-Commit-Position: refs/heads/master@{#882824} Bug: 1207895 Change-Id: Ib6f568de77cc89e537806ad289b985b43fb6d0f2 No-Presubmit: true No-Tree-Checks: true No-Try: true Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2895825 Auto-Submit: Maggie Cai Bot-Commit: Rubber Stamper Commit-Queue: Maggie Cai Owners-Override: Maggie Cai Cr-Commit-Position: refs/heads/master@{#882835} NOKEYCHECK=True GitOrigin-RevId: 5bd6329bd2b2e935283a9ed9d61126b5e2c3774a --- BUILD.gn | 40 --------------- contrib/minizip/miniunz.c | 11 +++-- contrib/minizip/minizip.c | 4 +- contrib/minizip/minizip.md | 9 ---- patches/0008-minizip-zip-unzip-tools.patch | 79 ------------------------------ 5 files changed, 8 insertions(+), 135 deletions(-) delete mode 100644 contrib/minizip/minizip.md delete mode 100644 patches/0008-minizip-zip-unzip-tools.patch diff --git a/BUILD.gn b/BUILD.gn index c7dfafa..d64cb38 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -419,46 +419,6 @@ executable("zlib_bench") { configs += [ "//build/config/compiler:no_chromium_code" ] } -executable("minizip_bin") { - include_dirs = [ "." ] - - sources = [ "contrib/minizip/minizip.c" ] - - if (is_clang) { - cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] - } - - if (!is_debug) { - configs -= [ "//build/config/compiler:default_optimization" ] - configs += [ "//build/config/compiler:optimize_speed" ] - } - - deps = [ ":minizip" ] - - configs -= [ "//build/config/compiler:chromium_code" ] - configs += [ "//build/config/compiler:no_chromium_code" ] -} - -executable("miniunz_bin") { - include_dirs = [ "." ] - - sources = [ "contrib/minizip/miniunz.c" ] - - if (is_clang) { - cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] - } - - if (!is_debug) { - configs -= [ "//build/config/compiler:default_optimization" ] - configs += [ "//build/config/compiler:optimize_speed" ] - } - - deps = [ ":minizip" ] - - configs -= [ "//build/config/compiler:chromium_code" ] - configs += [ "//build/config/compiler:no_chromium_code" ] -} - if (build_with_chromium) { test("zlib_unittests") { testonly = true diff --git a/contrib/minizip/miniunz.c b/contrib/minizip/miniunz.c index 74d1a92..3d65401 100644 --- a/contrib/minizip/miniunz.c +++ b/contrib/minizip/miniunz.c @@ -27,7 +27,7 @@ #endif #endif -#if defined(__APPLE__) || defined(__Fuchsia__) +#ifdef __APPLE__ // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions #define FOPEN_FUNC(filename, mode) fopen(filename, mode) #define FTELLO_FUNC(stream) ftello(stream) @@ -45,7 +45,6 @@ #include #include #include -#include #ifdef _WIN32 # include @@ -98,7 +97,7 @@ void change_file_date(filename,dosdate,tmu_date) SetFileTime(hFile,&ftm,&ftLastAcc,&ftm); CloseHandle(hFile); #else -#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) +#ifdef unix || __APPLE__ struct utimbuf ut; struct tm newdate; newdate.tm_sec = tmu_date.tm_sec; @@ -126,9 +125,11 @@ int mymkdir(dirname) const char* dirname; { int ret=0; -#if defined(_WIN32) +#ifdef _WIN32 ret = _mkdir(dirname); -#elif defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) +#elif unix + ret = mkdir (dirname,0775); +#elif __APPLE__ ret = mkdir (dirname,0775); #endif return ret; diff --git a/contrib/minizip/minizip.c b/contrib/minizip/minizip.c index 1624bd4..4288962 100644 --- a/contrib/minizip/minizip.c +++ b/contrib/minizip/minizip.c @@ -28,7 +28,7 @@ #endif #endif -#if defined(__APPLE__) || defined(__Fuchsia__) +#ifdef __APPLE__ // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions #define FOPEN_FUNC(filename, mode) fopen(filename, mode) #define FTELLO_FUNC(stream) ftello(stream) @@ -94,7 +94,7 @@ uLong filetime(f, tmzip, dt) return ret; } #else -#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) +#ifdef unix || __APPLE__ uLong filetime(f, tmzip, dt) char *f; /* name of file to get info on */ tm_zip *tmzip; /* return value: access, modific. and creation times */ diff --git a/contrib/minizip/minizip.md b/contrib/minizip/minizip.md deleted file mode 100644 index 9f15dd2..0000000 --- a/contrib/minizip/minizip.md +++ /dev/null @@ -1,9 +0,0 @@ -Minizip is a library provided by //third_party/zlib [1]. Its zip and unzip -tools can be built in a developer checkout for testing purposes with: - -```shell - autoninja -C out/Release minizip_bin - autoninja -C out/Release miniunz_bin -``` - -[1] Upstream is https://github.com/madler/zlib/tree/master/contrib/minizip diff --git a/patches/0008-minizip-zip-unzip-tools.patch b/patches/0008-minizip-zip-unzip-tools.patch deleted file mode 100644 index 8908907..0000000 --- a/patches/0008-minizip-zip-unzip-tools.patch +++ /dev/null @@ -1,79 +0,0 @@ -From ef2b67c03bc6f1395021f3583a3aaddf74976039 Mon Sep 17 00:00:00 2001 -From: Noel Gordon -Date: Wed, 12 May 2021 21:55:34 +1000 -Subject: [PATCH] Build minizip zip and unzip tools - ---- - third_party/zlib/contrib/minizip/miniunz.c | 11 +++++------ - third_party/zlib/contrib/minizip/minizip.c | 4 ++-- - 2 files changed, 7 insertions(+), 8 deletions(-) - -diff --git a/third_party/zlib/contrib/minizip/miniunz.c b/third_party/zlib/contrib/minizip/miniunz.c -index 3d65401be5cd..74d1a92739c0 100644 ---- a/third_party/zlib/contrib/minizip/miniunz.c -+++ b/third_party/zlib/contrib/minizip/miniunz.c -@@ -27,7 +27,7 @@ - #endif - #endif - --#ifdef __APPLE__ -+#if defined(__APPLE__) || defined(__Fuchsia__) - // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions - #define FOPEN_FUNC(filename, mode) fopen(filename, mode) - #define FTELLO_FUNC(stream) ftello(stream) -@@ -45,6 +45,7 @@ - #include - #include - #include -+#include - - #ifdef _WIN32 - # include -@@ -97,7 +98,7 @@ void change_file_date(filename,dosdate,tmu_date) - SetFileTime(hFile,&ftm,&ftLastAcc,&ftm); - CloseHandle(hFile); - #else --#ifdef unix || __APPLE__ -+#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) - struct utimbuf ut; - struct tm newdate; - newdate.tm_sec = tmu_date.tm_sec; -@@ -125,11 +126,9 @@ int mymkdir(dirname) - const char* dirname; - { - int ret=0; --#ifdef _WIN32 -+#if defined(_WIN32) - ret = _mkdir(dirname); --#elif unix -- ret = mkdir (dirname,0775); --#elif __APPLE__ -+#elif defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) - ret = mkdir (dirname,0775); - #endif - return ret; -diff --git a/third_party/zlib/contrib/minizip/minizip.c b/third_party/zlib/contrib/minizip/minizip.c -index 4288962ecef0..1624bd4b2955 100644 ---- a/third_party/zlib/contrib/minizip/minizip.c -+++ b/third_party/zlib/contrib/minizip/minizip.c -@@ -28,7 +28,7 @@ - #endif - #endif - --#ifdef __APPLE__ -+#if defined(__APPLE__) || defined(__Fuchsia__) - // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions - #define FOPEN_FUNC(filename, mode) fopen(filename, mode) - #define FTELLO_FUNC(stream) ftello(stream) -@@ -94,7 +94,7 @@ uLong filetime(f, tmzip, dt) - return ret; - } - #else --#ifdef unix || __APPLE__ -+#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) - uLong filetime(f, tmzip, dt) - char *f; /* name of file to get info on */ - tm_zip *tmzip; /* return value: access, modific. and creation times */ --- -2.31.1.607.g51e8a6a459-goog - -- cgit v1.2.3 From 16cd46cac22da1679979032316963d01526b9192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Fri, 14 May 2021 05:07:27 +0000 Subject: [zip] Made ZipParams a plain struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This simplifies code and avoids copying vectors of file paths. BUG=chromium:912236 TEST=autoninja -C out/release zlib_unittests && out/release/zlib_unittests Change-Id: I1107a651df9145fa02183f43b5d09cb44ac4bb8c Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2891171 Reviewed-by: Ken Rockot Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#882862} NOKEYCHECK=True GitOrigin-RevId: 709e2784b7d58d6d0fac6f367b0678db1a74e5ad --- google/zip.cc | 80 ++++++++++++++++++-------------------- google/zip.h | 103 ++++++++++++++++++------------------------------- google/zip_unittest.cc | 7 +++- google/zip_writer.h | 3 -- 4 files changed, 80 insertions(+), 113 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index 907e5da..83ad88a 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -100,39 +100,27 @@ class DirectFileAccessor : public FileAccessor { } // namespace -ZipParams::ZipParams(const base::FilePath& src_dir, - const base::FilePath& dest_file) - : src_dir_(src_dir), - dest_file_(dest_file), - file_accessor_(new DirectFileAccessor(src_dir)) {} +bool Zip(const ZipParams& params) { + DirectFileAccessor default_accessor(params.src_dir); + FileAccessor* const file_accessor = params.file_accessor ?: &default_accessor; -#if defined(OS_POSIX) -// Does not take ownership of |fd|. -ZipParams::ZipParams(const base::FilePath& src_dir, int dest_fd) - : src_dir_(src_dir), - dest_fd_(dest_fd), - file_accessor_(new DirectFileAccessor(src_dir)) {} -#endif + Paths files_to_add = params.src_files; -bool Zip(const ZipParams& params) { - // Using a pointer to avoid copies of a potentially large array. - const std::vector* files_to_add = ¶ms.files_to_zip(); std::vector all_files; - if (files_to_add->empty()) { + if (files_to_add.empty()) { // Include all files from the src_dir (modulo the src_dir itself and // filtered and hidden files). - files_to_add = &all_files; // Using a list so we can call push_back while iterating. std::list entries; entries.push_back(FileAccessor::DirectoryContentEntry( - params.src_dir(), true /* is directory*/)); - const FilterCallback& filter_callback = params.filter_callback(); + params.src_dir, true /* is directory*/)); for (auto iter = entries.begin(); iter != entries.end(); ++iter) { const base::FilePath& entry_path = iter->path; if (iter != entries.begin() && // Don't filter the root dir. - ((!params.include_hidden_files() && IsHiddenFile(entry_path)) || - (filter_callback && !filter_callback.Run(entry_path)))) { + ((!params.include_hidden_files && IsHiddenFile(entry_path)) || + (params.filter_callback && + !params.filter_callback.Run(entry_path)))) { continue; } @@ -140,36 +128,41 @@ bool Zip(const ZipParams& params) { // Make the path relative for AddEntryToZip. base::FilePath relative_path; bool success = - params.src_dir().AppendRelativePath(entry_path, &relative_path); + params.src_dir.AppendRelativePath(entry_path, &relative_path); DCHECK(success); all_files.push_back(relative_path); } if (iter->is_directory) { std::vector subentries = - params.file_accessor()->ListDirectoryContent(entry_path); + file_accessor->ListDirectoryContent(entry_path); entries.insert(entries.end(), subentries.begin(), subentries.end()); } } + + files_to_add = all_files; } std::unique_ptr zip_writer; + #if defined(OS_POSIX) - if (params.dest_fd() != base::kInvalidPlatformFile) { - DCHECK(params.dest_file().empty()); + if (params.dest_fd != base::kInvalidPlatformFile) { + DCHECK(params.dest_file.empty()); zip_writer = internal::ZipWriter::CreateWithFd( - params.dest_fd(), params.src_dir(), params.file_accessor()); + params.dest_fd, params.src_dir, file_accessor); if (!zip_writer) return false; } #endif + if (!zip_writer) { - zip_writer = internal::ZipWriter::Create( - params.dest_file(), params.src_dir(), params.file_accessor()); + zip_writer = internal::ZipWriter::Create(params.dest_file, params.src_dir, + file_accessor); if (!zip_writer) return false; } - return zip_writer->WriteEntries(*files_to_add); + + return zip_writer->WriteEntries(files_to_add); } bool Unzip(const base::FilePath& src_file, const base::FilePath& dest_dir) { @@ -179,7 +172,7 @@ bool Unzip(const base::FilePath& src_file, const base::FilePath& dest_dir) { bool UnzipWithFilterCallback(const base::FilePath& src_file, const base::FilePath& dest_dir, - const FilterCallback& filter_cb, + FilterCallback filter_cb, bool log_skipped_files) { base::File file(src_file, base::File::FLAG_OPEN | base::File::FLAG_READ); if (!file.IsValid()) { @@ -189,14 +182,14 @@ bool UnzipWithFilterCallback(const base::FilePath& src_file, return UnzipWithFilterAndWriters( file.GetPlatformFile(), base::BindRepeating(&CreateFilePathWriterDelegate, dest_dir), - base::BindRepeating(&CreateDirectory, dest_dir), filter_cb, + base::BindRepeating(&CreateDirectory, dest_dir), std::move(filter_cb), log_skipped_files); } bool UnzipWithFilterAndWriters(const base::PlatformFile& src_file, - const WriterFactory& writer_factory, - const DirectoryCreator& directory_creator, - const FilterCallback& filter_cb, + WriterFactory writer_factory, + DirectoryCreator directory_creator, + FilterCallback filter_cb, bool log_skipped_files) { ZipReader reader; if (!reader.OpenFromPlatformFile(src_file)) { @@ -239,14 +232,15 @@ bool UnzipWithFilterAndWriters(const base::PlatformFile& src_file, bool ZipWithFilterCallback(const base::FilePath& src_dir, const base::FilePath& dest_file, - const FilterCallback& filter_cb) { + FilterCallback filter_cb) { DCHECK(base::DirectoryExists(src_dir)); - ZipParams params(src_dir, dest_file); - params.set_filter_callback(filter_cb); - return Zip(params); + return Zip({.src_dir = src_dir, + .dest_file = dest_file, + .filter_callback = std::move(filter_cb)}); } -bool Zip(const base::FilePath& src_dir, const base::FilePath& dest_file, +bool Zip(const base::FilePath& src_dir, + const base::FilePath& dest_file, bool include_hidden_files) { if (include_hidden_files) { return ZipWithFilterCallback(src_dir, dest_file, @@ -259,12 +253,12 @@ bool Zip(const base::FilePath& src_dir, const base::FilePath& dest_file, #if defined(OS_POSIX) bool ZipFiles(const base::FilePath& src_dir, - const std::vector& src_relative_paths, + Paths src_relative_paths, int dest_fd) { DCHECK(base::DirectoryExists(src_dir)); - ZipParams params(src_dir, dest_fd); - params.set_files_to_zip(src_relative_paths); - return Zip(params); + return Zip({.src_dir = src_dir, + .dest_fd = dest_fd, + .src_files = src_relative_paths}); } #endif // defined(OS_POSIX) diff --git a/google/zip.h b/google/zip.h index 4f64a8a..d5ce005 100644 --- a/google/zip.h +++ b/google/zip.h @@ -5,9 +5,11 @@ #ifndef THIRD_PARTY_ZLIB_GOOGLE_ZIP_H_ #define THIRD_PARTY_ZLIB_GOOGLE_ZIP_H_ +#include #include #include "base/callback.h" +#include "base/containers/span.h" #include "base/files/file_path.h" #include "base/files/platform_file.h" #include "base/time/time.h" @@ -48,72 +50,44 @@ class FileAccessor { virtual base::Time GetLastModifiedTime(const base::FilePath& path) = 0; }; -class ZipParams { - public: - ZipParams(const base::FilePath& src_dir, const base::FilePath& dest_file); -#if defined(OS_POSIX) - // Does not take ownership of |dest_fd|. - ZipParams(const base::FilePath& src_dir, int dest_fd); +using FilterCallback = base::RepeatingCallback; - int dest_fd() const { return dest_fd_; } -#endif +using Paths = base::span; + +// ZIP creation parameters and options. +struct ZipParams { + // Source directory. + base::FilePath src_dir; + + // Destination file path. + // Either dest_file or dest_fd should be set, but not both. + base::FilePath dest_file; - const base::FilePath& src_dir() const { return src_dir_; } - - const base::FilePath& dest_file() const { return dest_file_; } - - // Restricts the files actually zipped to the paths listed in - // |src_relative_paths|. They must be relative to the |src_dir| passed in the - // constructor and will be used as the file names in the created zip file. All - // source paths must be under |src_dir| in the file system hierarchy. - void set_files_to_zip(const std::vector& src_relative_paths) { - src_files_ = src_relative_paths; - } - const std::vector& files_to_zip() const { return src_files_; } - - using FilterCallback = base::RepeatingCallback; - void set_filter_callback(FilterCallback filter_callback) { - filter_callback_ = filter_callback; - } - const FilterCallback& filter_callback() const { return filter_callback_; } - - void set_include_hidden_files(bool include_hidden_files) { - include_hidden_files_ = include_hidden_files; - } - bool include_hidden_files() const { return include_hidden_files_; } - - // Sets a custom file accessor for file operations. Default is to directly - // access the files (with fopen and the rest). - // Useful in cases where running in a sandbox process and file access has to - // go through IPC, for example. - void set_file_accessor(std::unique_ptr file_accessor) { - file_accessor_ = std::move(file_accessor); - } - FileAccessor* file_accessor() const { return file_accessor_.get(); } - - private: - base::FilePath src_dir_; - - base::FilePath dest_file_; #if defined(OS_POSIX) - int dest_fd_ = base::kInvalidPlatformFile; + // Destination file passed a file descriptor. + // Either dest_file or dest_fd should be set, but not both. + int dest_fd = base::kInvalidPlatformFile; #endif - // The relative paths to the files that should be included in the zip file. If - // this is empty, all files in |src_dir_| are included. - std::vector src_files_; + // The relative paths to the files that should be included in the ZIP file. If + // this is empty, all files in |src_dir| are included. + // + // These paths must be relative to |src_dir| and will be used as the file + // names in the created zip file. All files must be under |src_dir| in the + // file system hierarchy. + Paths src_files; // Filter used to exclude files from the ZIP file. Only effective when - // |src_files_| is empty. - FilterCallback filter_callback_; + // |src_files| is empty. + FilterCallback filter_callback; // Whether hidden files should be included in the ZIP file. Only effective - // when |src_files_| is empty. - bool include_hidden_files_ = true; + // when |src_files| is empty. + bool include_hidden_files = true; - // Abstraction around file system access used to read files. An implementation - // that accesses files directly is provided by default. - std::unique_ptr file_accessor_; + // Abstraction around file system access used to read files. If left null, an + // implementation that accesses files directly is used. + FileAccessor* file_accessor = nullptr; // Not owned }; // Zip files specified into a ZIP archives. The source files and ZIP destination @@ -125,15 +99,15 @@ bool Zip(const ZipParams& params); // of src_dir will be at the root level of the created zip. For each file in // src_dir, include it only if the callback |filter_cb| returns true. Otherwise // omit it. -using FilterCallback = base::RepeatingCallback; bool ZipWithFilterCallback(const base::FilePath& src_dir, const base::FilePath& dest_file, - const FilterCallback& filter_cb); + FilterCallback filter_cb); // Convenience method for callers who don't need to set up the filter callback. // If |include_hidden_files| is true, files starting with "." are included. // Otherwise they are omitted. -bool Zip(const base::FilePath& src_dir, const base::FilePath& dest_file, +bool Zip(const base::FilePath& src_dir, + const base::FilePath& dest_file, bool include_hidden_files); #if defined(OS_POSIX) @@ -143,7 +117,7 @@ bool Zip(const base::FilePath& src_dir, const base::FilePath& dest_file, // file names in the created zip file. All source paths must be under |src_dir| // in the file system hierarchy. bool ZipFiles(const base::FilePath& src_dir, - const std::vector& src_relative_paths, + Paths src_relative_paths, int dest_fd); #endif // defined(OS_POSIX) @@ -152,10 +126,9 @@ bool ZipFiles(const base::FilePath& src_dir, // returns true. Otherwise omit it. // If |log_skipped_files| is true, files skipped during extraction are printed // to debug log. -using FilterCallback = base::RepeatingCallback; bool UnzipWithFilterCallback(const base::FilePath& zip_file, const base::FilePath& dest_dir, - const FilterCallback& filter_cb, + FilterCallback filter_cb, bool log_skipped_files); // Unzip the contents of zip_file, using the writers provided by writer_factory. @@ -168,9 +141,9 @@ typedef base::RepeatingCallback( WriterFactory; typedef base::RepeatingCallback DirectoryCreator; bool UnzipWithFilterAndWriters(const base::PlatformFile& zip_file, - const WriterFactory& writer_factory, - const DirectoryCreator& directory_creator, - const FilterCallback& filter_cb, + WriterFactory writer_factory, + DirectoryCreator directory_creator, + FilterCallback filter_cb, bool log_skipped_files); // Unzip the contents of zip_file into dest_dir. diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 10f2ef7..8a53ac4 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -535,8 +535,11 @@ TEST_F(ZipTest, UnzipFilesWithIncorrectSize) { TEST_F(ZipTest, ZipWithFileAccessor) { base::FilePath zip_file; ASSERT_TRUE(base::CreateTemporaryFile(&zip_file)); - zip::ZipParams params(base::FilePath(FILE_PATH_LITERAL("/test")), zip_file); - params.set_file_accessor(std::make_unique()); + VirtualFileSystem file_accessor; + const zip::ZipParams params{ + .src_dir = base::FilePath(FILE_PATH_LITERAL("/test")), + .dest_file = zip_file, + .file_accessor = &file_accessor}; ASSERT_TRUE(zip::Zip(params)); base::ScopedTempDir scoped_temp_dir; diff --git a/google/zip_writer.h b/google/zip_writer.h index 0dc4c6c..9b8a9fe 100644 --- a/google/zip_writer.h +++ b/google/zip_writer.h @@ -8,7 +8,6 @@ #include #include -#include "base/containers/span.h" #include "base/files/file_path.h" #include "base/time/time.h" #include "build/build_config.h" @@ -45,8 +44,6 @@ class ZipWriter { FileAccessor* file_accessor); ~ZipWriter(); - using Paths = base::span; - // Writes the files at |paths| to the ZIP file and closes this ZIP file. // The file paths must be relative to |root_dir| specified in the // Create method. -- cgit v1.2.3 From 2b99b378749fa2a1957c1580fdd7cd0cd3bd2e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Fri, 14 May 2021 07:01:47 +0000 Subject: [zip] Add progress reporting to ZipWriter Added the ability to report progress and cancel the operation while creating a ZIP file. BUG=chromium:1207752, chromium:1207749, chromium:912236 TEST=autoninja -C out/release zlib_unittests && out/release/zlib_unittests Change-Id: Id443f8c69f25b552d1fbdd330e86ee20c71b1c25 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2891270 Reviewed-by: Noel Gordon Commit-Queue: Noel Gordon Cr-Commit-Position: refs/heads/master@{#882888} NOKEYCHECK=True GitOrigin-RevId: 45bf68080eb8fd1c271c7ae94b6f0a070579cd88 --- google/zip.cc | 8 ++++++++ google/zip.h | 33 +++++++++++++++++++++++++++++++++ google/zip_writer.cc | 41 ++++++++++++++++++++++++++++++++++------- google/zip_writer.h | 24 ++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 7 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index 83ad88a..4e5c698 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -100,6 +100,11 @@ class DirectFileAccessor : public FileAccessor { } // namespace +std::ostream& operator<<(std::ostream& out, const Progress& progress) { + return out << progress.bytes << " bytes, " << progress.files << " files, " + << progress.directories << " dirs"; +} + bool Zip(const ZipParams& params) { DirectFileAccessor default_accessor(params.src_dir); FileAccessor* const file_accessor = params.file_accessor ?: &default_accessor; @@ -162,6 +167,9 @@ bool Zip(const ZipParams& params) { return false; } + zip_writer->SetProgressCallback(params.progress_callback, + params.progress_period); + return zip_writer->WriteEntries(files_to_add); } diff --git a/google/zip.h b/google/zip.h index d5ce005..2c2ef7a 100644 --- a/google/zip.h +++ b/google/zip.h @@ -5,6 +5,8 @@ #ifndef THIRD_PARTY_ZLIB_GOOGLE_ZIP_H_ #define THIRD_PARTY_ZLIB_GOOGLE_ZIP_H_ +#include +#include #include #include @@ -50,6 +52,30 @@ class FileAccessor { virtual base::Time GetLastModifiedTime(const base::FilePath& path) = 0; }; +// Progress of a ZIP creation operation. +struct Progress { + // Total number of bytes read from files getting zipped so far. + std::int64_t bytes = 0; + + // Number of file entries added to the ZIP so far. + // A file entry is added after its bytes have been processed. + int files = 0; + + // Number of directory entries added to the ZIP so far. + // A directory entry is added before items in it. + int directories = 0; +}; + +// Prints Progress to output stream. +std::ostream& operator<<(std::ostream& out, const Progress& progress); + +// Callback reporting the progress of a ZIP creation operation. +// +// This callback returns a boolean indicating whether the ZIP creation operation +// should continue. If it returns false once, then the ZIP creation operation is +// immediately cancelled and the callback won't be called again. +using ProgressCallback = base::RepeatingCallback; + using FilterCallback = base::RepeatingCallback; using Paths = base::span; @@ -81,6 +107,13 @@ struct ZipParams { // |src_files| is empty. FilterCallback filter_callback; + // Optional progress reporting callback. + ProgressCallback progress_callback; + + // Progress reporting period. The final callback is always called when the ZIP + // creation operation completes. + base::TimeDelta progress_period; + // Whether hidden files should be included in the ZIP file. Only effective // when |src_files| is empty. bool include_hidden_files = true; diff --git a/google/zip_writer.cc b/google/zip_writer.cc index c151cc8..9870147 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -12,9 +12,25 @@ namespace zip { namespace internal { +bool ZipWriter::ShouldContinue() { + if (progress_callback_) { + const base::TimeTicks now = base::TimeTicks::Now(); + if (next_progress_report_time_ <= now) { + next_progress_report_time_ = now + progress_period_; + if (!progress_callback_.Run(progress_)) { + LOG(ERROR) << "Cancelling ZIP creation"; + return false; + } + } + } + + return true; +} + bool ZipWriter::AddFileContent(const base::FilePath& path, base::File file) { char buf[zip::internal::kZipBufSize]; - while (true) { + + while (ShouldContinue()) { const int num_bytes = file.ReadAtCurrentPos(buf, zip::internal::kZipBufSize); @@ -30,7 +46,11 @@ bool ZipWriter::AddFileContent(const base::FilePath& path, base::File file) { DLOG(ERROR) << "Cannot write data from file '" << path << "' to ZIP"; return false; } + + progress_.bytes += num_bytes; } + + return false; } bool ZipWriter::OpenNewFileEntry(const base::FilePath& path, @@ -58,17 +78,16 @@ bool ZipWriter::AddFileEntry(const base::FilePath& path, base::File file) { if (!OpenNewFileEntry(path, /*is_directory=*/false, file_info.last_modified)) return false; - bool success = AddFileContent(path, std::move(file)); - if (!CloseNewFileEntry()) - return false; - - return success; + const bool success = AddFileContent(path, std::move(file)); + progress_.files++; + return CloseNewFileEntry() && success; } bool ZipWriter::AddDirectoryEntry(const base::FilePath& path, base::Time last_modified) { + progress_.directories++; return OpenNewFileEntry(path, /*is_directory=*/true, last_modified) && - CloseNewFileEntry(); + CloseNewFileEntry() && ShouldContinue(); } #if defined(OS_POSIX) @@ -126,6 +145,14 @@ bool ZipWriter::WriteEntries(Paths paths) { bool ZipWriter::Close() { const bool success = zipClose(zip_file_, nullptr) == ZIP_OK; zip_file_ = nullptr; + + // Call the progress callback one last time with the final progress status. + // + // We don't care about the callback return value (cancellation request), since + // we're done anyway. + if (progress_callback_) + progress_callback_.Run(progress_); + return success; } diff --git a/google/zip_writer.h b/google/zip_writer.h index 9b8a9fe..489984f 100644 --- a/google/zip_writer.h +++ b/google/zip_writer.h @@ -44,6 +44,14 @@ class ZipWriter { FileAccessor* file_accessor); ~ZipWriter(); + // Sets the optional progress callback. The callback is called once for each + // time |period|. The final callback is always called when the ZIP operation + // completes. + void SetProgressCallback(ProgressCallback callback, base::TimeDelta period) { + progress_callback_ = std::move(callback); + progress_period_ = std::move(period); + } + // Writes the files at |paths| to the ZIP file and closes this ZIP file. // The file paths must be relative to |root_dir| specified in the // Create method. @@ -56,6 +64,10 @@ class ZipWriter { const base::FilePath& root_dir, FileAccessor* file_accessor); + // Regularly called during processing to check whether zipping should continue + // or should be cancelled. + bool ShouldContinue(); + // Adds the files at |paths| to the ZIP file. These FilePaths must be relative // to |root_dir| specified in the Create method. bool AddEntries(Paths paths); @@ -91,6 +103,18 @@ class ZipWriter { // Abstraction over file access methods used to read files. FileAccessor* file_accessor_; + // Progress stats. + Progress progress_; + + // Optional progress callback. + ProgressCallback progress_callback_; + + // Optional progress reporting period. + base::TimeDelta progress_period_; + + // Next time to report progress. + base::TimeTicks next_progress_report_time_ = base::TimeTicks::Now(); + DISALLOW_COPY_AND_ASSIGN(ZipWriter); }; -- cgit v1.2.3 From 5ce08a7f2e45fd8e38bbfb7fa3d1e83495187b86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Fri, 14 May 2021 08:06:57 +0000 Subject: [zip] Optimize BFS file search is zip::Zip() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Less code. Fewer copies. Same result. BUG=chromium:912236 TEST=autoninja -C out/release zlib_unittests && out/release/zlib_unittests Change-Id: I2f99fa90789822a2dc6ba759957293e176db1d75 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2896684 Commit-Queue: François Degros Reviewed-by: Noel Gordon Cr-Commit-Position: refs/heads/master@{#882901} NOKEYCHECK=True GitOrigin-RevId: 616636dd28b7c07917f9ad59ef4d586f3dba81ba --- google/zip.cc | 46 ++++++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index 4e5c698..c71c738 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -4,7 +4,7 @@ #include "third_party/zlib/google/zip.h" -#include +#include #include #include @@ -113,35 +113,25 @@ bool Zip(const ZipParams& params) { std::vector all_files; if (files_to_add.empty()) { - // Include all files from the src_dir (modulo the src_dir itself and - // filtered and hidden files). - - // Using a list so we can call push_back while iterating. - std::list entries; - entries.push_back(FileAccessor::DirectoryContentEntry( - params.src_dir, true /* is directory*/)); - for (auto iter = entries.begin(); iter != entries.end(); ++iter) { - const base::FilePath& entry_path = iter->path; - if (iter != entries.begin() && // Don't filter the root dir. - ((!params.include_hidden_files && IsHiddenFile(entry_path)) || - (params.filter_callback && - !params.filter_callback.Run(entry_path)))) { - continue; - } - - if (iter != entries.begin()) { // Exclude the root dir from the ZIP file. - // Make the path relative for AddEntryToZip. - base::FilePath relative_path; - bool success = - params.src_dir.AppendRelativePath(entry_path, &relative_path); + // Perform a Breadth First Search (BFS) of the source tree. Note that the + // BFS order might not be optimal when storing files in a ZIP (either for + // the storing side, or for the program that will extract this ZIP). + for (std::queue q({params.src_dir}); !q.empty(); q.pop()) { + for (FileAccessor::DirectoryContentEntry& entry : + file_accessor->ListDirectoryContent(q.front())) { + // Skip hidden and filtered files. + if ((!params.include_hidden_files && IsHiddenFile(entry.path)) || + (params.filter_callback && !params.filter_callback.Run(entry.path))) + continue; + + // Store relative path. + all_files.emplace_back(); + const bool success = + params.src_dir.AppendRelativePath(entry.path, &all_files.back()); DCHECK(success); - all_files.push_back(relative_path); - } - if (iter->is_directory) { - std::vector subentries = - file_accessor->ListDirectoryContent(entry_path); - entries.insert(entries.end(), subentries.begin(), subentries.end()); + if (entry.is_directory) + q.push(std::move(entry.path)); } } -- cgit v1.2.3 From 298d9e6072378fe3fc9d4c5de41ef36a01c7a19d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 18 May 2021 01:38:11 +0000 Subject: [zip] Add unit tests for progress reporting and cancellation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1207752, chromium:1207749, chromium:912236 TEST=autoninja -C out/release zlib_unittests && out/release/zlib_unittests --single-process-tests Change-Id: I190b2198fa8925be1539923714013eaf62229d8a Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2896002 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#883752} NOKEYCHECK=True GitOrigin-RevId: d96f933f2d0700d8d841889703aecb97dde108c0 --- google/zip_unittest.cc | 169 ++++++++++++++++++++++++++++++++++++++++++------- google/zip_writer.cc | 32 +++++----- 2 files changed, 161 insertions(+), 40 deletions(-) diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 8a53ac4..5a82795 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -21,6 +21,7 @@ #include "base/path_service.h" #include "base/strings/string_util.h" #include "base/strings/stringprintf.h" +#include "base/test/bind.h" #include "build/build_config.h" #include "testing/gtest/include/gtest/gtest.h" #include "testing/platform_test.h" @@ -131,10 +132,7 @@ constexpr char VirtualFileSystem::kBar2Content[]; // Make the test a PlatformTest to setup autorelease pools properly on Mac. class ZipTest : public PlatformTest { protected: - enum ValidYearType { - VALID_YEAR, - INVALID_YEAR - }; + enum ValidYearType { VALID_YEAR, INVALID_YEAR }; virtual void SetUp() { PlatformTest::SetUp(); @@ -161,20 +159,17 @@ class ZipTest : public PlatformTest { base::FilePath(FILE_PATH_LITERAL("foo/bar/.hidden"))); } - virtual void TearDown() { - PlatformTest::TearDown(); - } + virtual void TearDown() { PlatformTest::TearDown(); } bool GetTestDataDirectory(base::FilePath* path) { bool success = base::PathService::Get(base::DIR_SOURCE_ROOT, path); EXPECT_TRUE(success); if (!success) return false; - *path = path->AppendASCII("third_party"); - *path = path->AppendASCII("zlib"); - *path = path->AppendASCII("google"); - *path = path->AppendASCII("test"); - *path = path->AppendASCII("data"); + for (const base::StringPiece s : + {"third_party", "zlib", "google", "test", "data"}) { + *path = path->AppendASCII(s); + } return true; } @@ -193,7 +188,8 @@ class ZipTest : public PlatformTest { ASSERT_TRUE(GetTestDataDirectory(&original_dir)); original_dir = original_dir.AppendASCII("test"); - base::FileEnumerator files(test_dir_, true, + base::FileEnumerator files( + test_dir_, true, base::FileEnumerator::FILES | base::FileEnumerator::DIRECTORIES); base::FilePath unzipped_entry_path = files.Next(); size_t count = 0; @@ -211,15 +207,19 @@ class ZipTest : public PlatformTest { bool append_relative_path_success = test_dir_.AppendRelativePath(unzipped_entry_path, &relative_path); if (!append_relative_path_success) { - LOG(ERROR) << "Append relative path failed, params: " - << test_dir_.value() << " and " - << unzipped_entry_path.value(); + LOG(ERROR) << "Append relative path failed, params: " << test_dir_ + << " and " << unzipped_entry_path; } base::FilePath original_path = original_dir.Append(relative_path); - LOG(ERROR) << "Comparing original " << original_path.value() - << " and unzipped file " << unzipped_entry_path.value() - << " result: " - << base::ContentsEqual(original_path, unzipped_entry_path); + const bool equal = + base::ContentsEqual(original_path, unzipped_entry_path); + if (equal) { + LOG(INFO) << "Original and unzipped file '" << relative_path + << "' are equal"; + } else { + LOG(ERROR) << "Original and unzipped file '" << relative_path + << "' are different"; + } // EXPECT_TRUE(base::ContentsEqual(original_path, unzipped_entry_path)) // << "Contents differ between original " << original_path.value() // << " and unzipped file " << unzipped_entry_path.value(); @@ -487,8 +487,8 @@ TEST_F(ZipTest, ZipFiles) { base::File zip_file(zip_name, base::File::FLAG_CREATE | base::File::FLAG_WRITE); ASSERT_TRUE(zip_file.IsValid()); - EXPECT_TRUE(zip::ZipFiles(src_dir, zip_file_list_, - zip_file.GetPlatformFile())); + EXPECT_TRUE( + zip::ZipFiles(src_dir, zip_file_list_, zip_file.GetPlatformFile())); zip_file.Close(); zip::ZipReader reader; @@ -524,8 +524,8 @@ TEST_F(ZipTest, UnzipFilesWithIncorrectSize) { for (int i = 0; i < 8; i++) { SCOPED_TRACE(base::StringPrintf("Processing %d.txt", i)); - base::FilePath file_path = temp_dir.AppendASCII( - base::StringPrintf("%d.txt", i)); + base::FilePath file_path = + temp_dir.AppendASCII(base::StringPrintf("%d.txt", i)); int64_t file_size = -1; EXPECT_TRUE(base::GetFileSize(file_path, &file_size)); EXPECT_EQ(static_cast(i), file_size); @@ -560,4 +560,125 @@ TEST_F(ZipTest, ZipWithFileAccessor) { EXPECT_EQ(VirtualFileSystem::kBar2Content, file_content); } +// Tests progress reporting while zipping files. +TEST_F(ZipTest, ZipProgress) { + base::FilePath src_dir; + ASSERT_TRUE(GetTestDataDirectory(&src_dir)); + src_dir = src_dir.AppendASCII("test"); + + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + base::FilePath zip_file = temp_dir.GetPath().AppendASCII("out.zip"); + + int progress_count = 0; + zip::Progress last_progress; + + zip::ProgressCallback progress_callback = + base::BindLambdaForTesting([&](const zip::Progress& progress) { + progress_count++; + LOG(INFO) << "Progress #" << progress_count << ": " << progress; + + // Progress should only go forwards. + EXPECT_GE(progress.bytes, last_progress.bytes); + EXPECT_GE(progress.files, last_progress.files); + EXPECT_GE(progress.directories, last_progress.directories); + + last_progress = progress; + return true; + }); + + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, + .dest_file = zip_file, + .progress_callback = std::move(progress_callback)})); + + EXPECT_EQ(progress_count, 14); + EXPECT_EQ(last_progress.bytes, 13546); + EXPECT_EQ(last_progress.files, 5); + EXPECT_EQ(last_progress.directories, 2); + + TestUnzipFile(zip_file, true); +} + +// Tests throttling of progress reporting while zipping files. +TEST_F(ZipTest, ZipProgressPeriod) { + base::FilePath src_dir; + ASSERT_TRUE(GetTestDataDirectory(&src_dir)); + src_dir = src_dir.AppendASCII("test"); + + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + base::FilePath zip_file = temp_dir.GetPath().AppendASCII("out.zip"); + + int progress_count = 0; + zip::Progress last_progress; + + zip::ProgressCallback progress_callback = + base::BindLambdaForTesting([&](const zip::Progress& progress) { + progress_count++; + LOG(INFO) << "Progress #" << progress_count << ": " << progress; + + // Progress should only go forwards. + EXPECT_GE(progress.bytes, last_progress.bytes); + EXPECT_GE(progress.files, last_progress.files); + EXPECT_GE(progress.directories, last_progress.directories); + + last_progress = progress; + return true; + }); + + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, + .dest_file = zip_file, + .progress_callback = std::move(progress_callback), + .progress_period = base::TimeDelta::FromHours(1)})); + + // We expect only 2 progress reports: the first one, and the last one. + EXPECT_EQ(progress_count, 2); + EXPECT_EQ(last_progress.bytes, 13546); + EXPECT_EQ(last_progress.files, 5); + EXPECT_EQ(last_progress.directories, 2); + + TestUnzipFile(zip_file, true); +} + +// Tests cancellation while zipping files. +TEST_F(ZipTest, ZipCancel) { + base::FilePath src_dir; + ASSERT_TRUE(GetTestDataDirectory(&src_dir)); + src_dir = src_dir.AppendASCII("test"); + + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + base::FilePath zip_file = temp_dir.GetPath().AppendASCII("out.zip"); + + // First: establish the number of possible interruption points. + int progress_count = 0; + + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, + .dest_file = zip_file, + .progress_callback = base::BindLambdaForTesting( + [&progress_count](const zip::Progress&) { + progress_count++; + return true; + })})); + + EXPECT_EQ(progress_count, 14); + + // Second: exercise each and every interruption point. + for (int i = progress_count; i > 0; i--) { + int j = 0; + EXPECT_FALSE(zip::Zip({.src_dir = src_dir, + .dest_file = zip_file, + .progress_callback = base::BindLambdaForTesting( + [i, &j](const zip::Progress&) { + j++; + // Callback shouldn't be called again after + // having returned false once. + EXPECT_LE(j, i); + return j < i; + })})); + + EXPECT_EQ(j, i); + } +} + } // namespace diff --git a/google/zip_writer.cc b/google/zip_writer.cc index 9870147..de1930e 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -13,18 +13,19 @@ namespace zip { namespace internal { bool ZipWriter::ShouldContinue() { - if (progress_callback_) { - const base::TimeTicks now = base::TimeTicks::Now(); - if (next_progress_report_time_ <= now) { - next_progress_report_time_ = now + progress_period_; - if (!progress_callback_.Run(progress_)) { - LOG(ERROR) << "Cancelling ZIP creation"; - return false; - } - } - } + if (!progress_callback_) + return true; - return true; + const base::TimeTicks now = base::TimeTicks::Now(); + if (next_progress_report_time_ > now) + return true; + + next_progress_report_time_ = now + progress_period_; + if (progress_callback_.Run(progress_)) + return true; + + LOG(ERROR) << "Cancelling ZIP creation"; + return false; } bool ZipWriter::AddFileContent(const base::FilePath& path, base::File file) { @@ -147,11 +148,10 @@ bool ZipWriter::Close() { zip_file_ = nullptr; // Call the progress callback one last time with the final progress status. - // - // We don't care about the callback return value (cancellation request), since - // we're done anyway. - if (progress_callback_) - progress_callback_.Run(progress_); + if (progress_callback_ && !progress_callback_.Run(progress_)) { + LOG(ERROR) << "Cancelling ZIP creation at the end"; + return false; + } return success; } -- cgit v1.2.3 From 5b8d433953beb2a75a755ba321a3076b95f7cdb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 18 May 2021 05:56:20 +0000 Subject: [zip] Optimize DirectoryContentEntry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simple optimizations and code clean-up to avoid copying file paths. BUG=chromium:912236 TEST=autoninja -C out/release zlib_unittests && out/release/zlib_unittests TEST=autoninja -C out/release browser_tests && testing/xvfb.py out/release/browser_tests --gtest_filter="ZipFileCreatorTest* Change-Id: Id58e38d93219c4f18cbdc8c9e9ed6bfc90a4b64b Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2901539 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#883817} NOKEYCHECK=True GitOrigin-RevId: de47c67919e498dbb6a64c6481a3052de608c08f --- google/zip.cc | 20 +++++++++----------- google/zip.h | 2 -- google/zip_unittest.cc | 11 ++++------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index c71c738..20cdd57 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -50,12 +50,12 @@ std::unique_ptr CreateFilePathWriterDelegate( class DirectFileAccessor : public FileAccessor { public: - explicit DirectFileAccessor(base::FilePath src_dir) : src_dir_(src_dir) {} ~DirectFileAccessor() override = default; std::vector OpenFilesForReading( const std::vector& paths) override { std::vector files; + for (const auto& path : paths) { base::File file; if (base::PathExists(path) && !base::DirectoryExists(path)) { @@ -63,6 +63,7 @@ class DirectFileAccessor : public FileAccessor { } files.push_back(std::move(file)); } + return files; } @@ -73,29 +74,26 @@ class DirectFileAccessor : public FileAccessor { std::vector ListDirectoryContent( const base::FilePath& dir) override { std::vector files; + base::FileEnumerator file_enumerator( dir, false /* recursive */, base::FileEnumerator::FILES | base::FileEnumerator::DIRECTORIES); - for (base::FilePath path = file_enumerator.Next(); !path.value().empty(); + for (base::FilePath path = file_enumerator.Next(); !path.empty(); path = file_enumerator.Next()) { - files.push_back(DirectoryContentEntry(path, base::DirectoryExists(path))); + const bool is_directory = base::DirectoryExists(path); + files.push_back({std::move(path), is_directory}); } + return files; } base::Time GetLastModifiedTime(const base::FilePath& path) override { base::File::Info file_info; if (!base::GetFileInfo(path, &file_info)) { - LOG(ERROR) << "Failed to retrieve file modification time for " - << path.value(); + LOG(ERROR) << "Cannot get modification time for '" << path << "'"; } return file_info.last_modified; } - - private: - base::FilePath src_dir_; - - DISALLOW_COPY_AND_ASSIGN(DirectFileAccessor); }; } // namespace @@ -106,7 +104,7 @@ std::ostream& operator<<(std::ostream& out, const Progress& progress) { } bool Zip(const ZipParams& params) { - DirectFileAccessor default_accessor(params.src_dir); + DirectFileAccessor default_accessor; FileAccessor* const file_accessor = params.file_accessor ?: &default_accessor; Paths files_to_add = params.src_files; diff --git a/google/zip.h b/google/zip.h index 2c2ef7a..ecd7ba0 100644 --- a/google/zip.h +++ b/google/zip.h @@ -35,8 +35,6 @@ class FileAccessor { virtual ~FileAccessor() = default; struct DirectoryContentEntry { - DirectoryContentEntry(const base::FilePath& path, bool is_directory) - : path(path), is_directory(is_directory) {} base::FilePath path; bool is_directory = false; }; diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 5a82795..cf914d9 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -79,13 +79,10 @@ class VirtualFileSystem : public zip::FileAccessor { DCHECK(success); files_[bar2_txt_path] = std::move(file); - file_tree_[test_dir] = std::vector{ - DirectoryContentEntry(foo_txt_path, /*is_dir=*/false), - DirectoryContentEntry(bar_dir, /*is_dir=*/true)}; - file_tree_[bar_dir] = std::vector{ - DirectoryContentEntry(bar1_txt_path, /*is_dir=*/false), - DirectoryContentEntry(bar2_txt_path, /*is_dir=*/false)}; + file_tree_[test_dir] = {{foo_txt_path, false}, {bar_dir, true}}; + file_tree_[bar_dir] = {{bar1_txt_path, false}, {bar2_txt_path, false}}; } + ~VirtualFileSystem() override = default; private: @@ -109,7 +106,7 @@ class VirtualFileSystem : public zip::FileAccessor { auto iter = file_tree_.find(dir); if (iter == file_tree_.end()) { NOTREACHED(); - return std::vector(); + return {}; } return iter->second; } -- cgit v1.2.3 From 4d0fedc51b2306cadc4f114a451b68bfdfcc3b89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 25 May 2021 01:58:46 +0000 Subject: [zip] Refactor zip::FileAccessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changed zip::FileAccessor to deal with relative paths instead of absolute paths. This avoids a bunch of back-and-forth conversions between relative and absolute paths when zipping files. Modified zip::FileAccessor methods to always return a bool indicating success. This makes error handling more systematic. BUG=chromium:912236 TEST=out/release/zlib_unittests TEST=out/release/browser_tests --gtest_filter="ZipFileCreatorTest* Change-Id: Ic53c62e899a11ddd2186b1f0d55a4a5ea9916342 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2909446 Commit-Queue: François Degros Reviewed-by: Noel Gordon Cr-Commit-Position: refs/heads/master@{#886144} NOKEYCHECK=True GitOrigin-RevId: 9b0b4469f8605bd11056f22018b98c5474015c5d --- google/zip.cc | 117 ++++++++++++++++++++++++++++++------------------- google/zip.h | 28 +++++++----- google/zip_unittest.cc | 82 ++++++++++++++++++++++++---------- google/zip_writer.cc | 76 +++++++++++++------------------- google/zip_writer.h | 21 ++++----- 5 files changed, 185 insertions(+), 139 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index 20cdd57..e83908c 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -50,50 +50,70 @@ std::unique_ptr CreateFilePathWriterDelegate( class DirectFileAccessor : public FileAccessor { public: + explicit DirectFileAccessor(base::FilePath src_dir) + : src_dir_(std::move(src_dir)) {} + ~DirectFileAccessor() override = default; - std::vector OpenFilesForReading( - const std::vector& paths) override { - std::vector files; + bool Open(const Paths paths, std::vector* const files) override { + DCHECK(files); + files->reserve(files->size() + paths.size()); - for (const auto& path : paths) { - base::File file; - if (base::PathExists(path) && !base::DirectoryExists(path)) { - file = base::File(path, base::File::FLAG_OPEN | base::File::FLAG_READ); + for (const base::FilePath& path : paths) { + DCHECK(!path.IsAbsolute()); + const base::FilePath absolute_path = src_dir_.Append(path); + if (base::DirectoryExists(absolute_path)) { + files->emplace_back(); + LOG(ERROR) << "Cannot open '" << path << "': It is a directory"; + } else { + files->emplace_back(absolute_path, + base::File::FLAG_OPEN | base::File::FLAG_READ); + LOG_IF(ERROR, !files->back().IsValid()) + << "Cannot open '" << path << "'"; } - files.push_back(std::move(file)); } - return files; - } - - bool DirectoryExists(const base::FilePath& file) override { - return base::DirectoryExists(file); + return true; } - std::vector ListDirectoryContent( - const base::FilePath& dir) override { - std::vector files; + bool List(const base::FilePath& path, + std::vector* const files, + std::vector* const subdirs) override { + DCHECK(!path.IsAbsolute()); + DCHECK(files); + DCHECK(subdirs); base::FileEnumerator file_enumerator( - dir, false /* recursive */, + src_dir_.Append(path), false /* recursive */, base::FileEnumerator::FILES | base::FileEnumerator::DIRECTORIES); - for (base::FilePath path = file_enumerator.Next(); !path.empty(); - path = file_enumerator.Next()) { - const bool is_directory = base::DirectoryExists(path); - files.push_back({std::move(path), is_directory}); + + while (!file_enumerator.Next().empty()) { + const base::FileEnumerator::FileInfo info = file_enumerator.GetInfo(); + (info.IsDirectory() ? subdirs : files) + ->push_back(path.Append(info.GetName())); } - return files; + return true; } - base::Time GetLastModifiedTime(const base::FilePath& path) override { + bool GetInfo(const base::FilePath& path, Info* const info) override { + DCHECK(!path.IsAbsolute()); + DCHECK(info); + base::File::Info file_info; - if (!base::GetFileInfo(path, &file_info)) { - LOG(ERROR) << "Cannot get modification time for '" << path << "'"; + if (!base::GetFileInfo(src_dir_.Append(path), &file_info)) { + LOG(ERROR) << "Cannot get info of '" << path << "'"; + return false; } - return file_info.last_modified; + + info->is_directory = file_info.is_directory; + info->last_modified = file_info.last_modified; + + return true; } + + private: + const base::FilePath src_dir_; }; } // namespace @@ -104,32 +124,38 @@ std::ostream& operator<<(std::ostream& out, const Progress& progress) { } bool Zip(const ZipParams& params) { - DirectFileAccessor default_accessor; + DirectFileAccessor default_accessor(params.src_dir); FileAccessor* const file_accessor = params.file_accessor ?: &default_accessor; Paths files_to_add = params.src_files; std::vector all_files; if (files_to_add.empty()) { + const auto exclude = [¶ms](const base::FilePath& path) { + return (!params.include_hidden_files && IsHiddenFile(path)) || + (params.filter_callback && + !params.filter_callback.Run(params.src_dir.Append(path))); + }; + // Perform a Breadth First Search (BFS) of the source tree. Note that the // BFS order might not be optimal when storing files in a ZIP (either for // the storing side, or for the program that will extract this ZIP). - for (std::queue q({params.src_dir}); !q.empty(); q.pop()) { - for (FileAccessor::DirectoryContentEntry& entry : - file_accessor->ListDirectoryContent(q.front())) { + for (std::queue q({{}}); !q.empty(); q.pop()) { + std::vector files, dirs; + file_accessor->List(q.front(), &files, &dirs); + + for (base::FilePath& path : files) { // Skip hidden and filtered files. - if ((!params.include_hidden_files && IsHiddenFile(entry.path)) || - (params.filter_callback && !params.filter_callback.Run(entry.path))) - continue; - - // Store relative path. - all_files.emplace_back(); - const bool success = - params.src_dir.AppendRelativePath(entry.path, &all_files.back()); - DCHECK(success); - - if (entry.is_directory) - q.push(std::move(entry.path)); + if (!exclude(path)) + all_files.push_back(std::move(path)); + } + + for (base::FilePath& path : dirs) { + // Skip hidden and filtered subdirs. + if (!exclude(path)) { + q.push(path); + all_files.push_back(std::move(path)); + } } } @@ -141,16 +167,15 @@ bool Zip(const ZipParams& params) { #if defined(OS_POSIX) if (params.dest_fd != base::kInvalidPlatformFile) { DCHECK(params.dest_file.empty()); - zip_writer = internal::ZipWriter::CreateWithFd( - params.dest_fd, params.src_dir, file_accessor); + zip_writer = + internal::ZipWriter::CreateWithFd(params.dest_fd, file_accessor); if (!zip_writer) return false; } #endif if (!zip_writer) { - zip_writer = internal::ZipWriter::Create(params.dest_file, params.src_dir, - file_accessor); + zip_writer = internal::ZipWriter::Create(params.dest_file, file_accessor); if (!zip_writer) return false; } diff --git a/google/zip.h b/google/zip.h index ecd7ba0..e12f446 100644 --- a/google/zip.h +++ b/google/zip.h @@ -25,29 +25,35 @@ namespace zip { class WriterDelegate; +// Paths passed as span to avoid copying them. +using Paths = base::span; + // Abstraction for file access operation required by Zip(). +// // Can be passed to the ZipParams for providing custom access to the files, // for example over IPC. -// If none is provided, the files are accessed directly. -// All parameters paths are expected to be absolute. +// +// All parameters paths are expected to be relative to the source directory. class FileAccessor { public: virtual ~FileAccessor() = default; - struct DirectoryContentEntry { - base::FilePath path; + struct Info { bool is_directory = false; + base::Time last_modified; }; // Opens files specified in |paths|. // Directories should be mapped to invalid files. - virtual std::vector OpenFilesForReading( - const std::vector& paths) = 0; + virtual bool Open(Paths paths, std::vector* files) = 0; - virtual bool DirectoryExists(const base::FilePath& path) = 0; - virtual std::vector ListDirectoryContent( - const base::FilePath& dir_path) = 0; - virtual base::Time GetLastModifiedTime(const base::FilePath& path) = 0; + // Lists contents of a directory at |path|. + virtual bool List(const base::FilePath& path, + std::vector* files, + std::vector* subdirs) = 0; + + // Gets info about a file or directory. + virtual bool GetInfo(const base::FilePath& path, Info* info) = 0; }; // Progress of a ZIP creation operation. @@ -76,8 +82,6 @@ using ProgressCallback = base::RepeatingCallback; using FilterCallback = base::RepeatingCallback; -using Paths = base::span; - // ZIP creation parameters and options. struct ZipParams { // Source directory. diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index cf914d9..558fedd 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -57,7 +57,7 @@ class VirtualFileSystem : public zip::FileAccessor { static constexpr char kBar2Content[] = "This is bar too."; VirtualFileSystem() { - base::FilePath test_dir(FILE_PATH_LITERAL("/test")); + base::FilePath test_dir; base::FilePath foo_txt_path = test_dir.Append(FILE_PATH_LITERAL("foo.txt")); base::FilePath file_path; @@ -79,43 +79,77 @@ class VirtualFileSystem : public zip::FileAccessor { DCHECK(success); files_[bar2_txt_path] = std::move(file); - file_tree_[test_dir] = {{foo_txt_path, false}, {bar_dir, true}}; - file_tree_[bar_dir] = {{bar1_txt_path, false}, {bar2_txt_path, false}}; + file_tree_[base::FilePath()] = {{foo_txt_path}, {bar_dir}}; + file_tree_[bar_dir] = {{bar1_txt_path, bar2_txt_path}}; + file_tree_[foo_txt_path] = {}; + file_tree_[bar1_txt_path] = {}; + file_tree_[bar2_txt_path] = {}; } ~VirtualFileSystem() override = default; private: - std::vector OpenFilesForReading( - const std::vector& paths) override { - std::vector files; - for (const auto& path : paths) { - auto iter = files_.find(path); - files.push_back(iter == files_.end() ? base::File() - : std::move(iter->second)); + bool Open(const zip::Paths paths, + std::vector* const files) override { + DCHECK(files); + files->reserve(files->size() + paths.size()); + + for (const base::FilePath& path : paths) { + const auto it = files_.find(path); + if (it == files_.end()) { + files->emplace_back(); + } else { + EXPECT_TRUE(it->second.IsValid()); + files->push_back(std::move(it->second)); + } } - return files; - } - bool DirectoryExists(const base::FilePath& file) override { - return file_tree_.count(file) == 1 && files_.count(file) == 0; + return true; } - std::vector ListDirectoryContent( - const base::FilePath& dir) override { - auto iter = file_tree_.find(dir); - if (iter == file_tree_.end()) { - NOTREACHED(); - return {}; + bool List(const base::FilePath& path, + std::vector* const files, + std::vector* const subdirs) override { + DCHECK(!path.IsAbsolute()); + DCHECK(files); + DCHECK(subdirs); + + const auto it = file_tree_.find(path); + if (it == file_tree_.end()) + return false; + + for (const base::FilePath& file : it->second.files) { + DCHECK(!file.empty()); + files->push_back(file); + } + + for (const base::FilePath& subdir : it->second.subdirs) { + DCHECK(!subdir.empty()); + subdirs->push_back(subdir); } - return iter->second; + + return true; } - base::Time GetLastModifiedTime(const base::FilePath& path) override { - return base::Time::FromDoubleT(172097977); // Some random date. + bool GetInfo(const base::FilePath& path, Info* const info) override { + DCHECK(!path.IsAbsolute()); + DCHECK(info); + + if (!file_tree_.count(path)) + return false; + + info->is_directory = !files_.count(path); + info->last_modified = + base::Time::FromDoubleT(172097977); // Some random date. + + return true; } - std::map> file_tree_; + struct DirContents { + std::vector files, subdirs; + }; + + std::map file_tree_; std::map files_; DISALLOW_COPY_AND_ASSIGN(VirtualFileSystem); diff --git a/google/zip_writer.cc b/google/zip_writer.cc index de1930e..26d9c13 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -84,18 +84,30 @@ bool ZipWriter::AddFileEntry(const base::FilePath& path, base::File file) { return CloseNewFileEntry() && success; } -bool ZipWriter::AddDirectoryEntry(const base::FilePath& path, - base::Time last_modified) { +bool ZipWriter::AddDirectoryEntry(const base::FilePath& path) { + FileAccessor::Info info; + if (!file_accessor_->GetInfo(path, &info)) + return false; + + if (!info.is_directory) { + LOG(ERROR) << "Not a directory: '" << path << "'"; + return false; + } + + if (!OpenNewFileEntry(path, /*is_directory=*/true, info.last_modified)) + return false; + + if (!CloseNewFileEntry()) + return false; + progress_.directories++; - return OpenNewFileEntry(path, /*is_directory=*/true, last_modified) && - CloseNewFileEntry() && ShouldContinue(); + return ShouldContinue(); } #if defined(OS_POSIX) // static std::unique_ptr ZipWriter::CreateWithFd( int zip_file_fd, - const base::FilePath& root_dir, FileAccessor* file_accessor) { DCHECK(zip_file_fd != base::kInvalidPlatformFile); zipFile zip_file = @@ -106,15 +118,13 @@ std::unique_ptr ZipWriter::CreateWithFd( return nullptr; } - return std::unique_ptr( - new ZipWriter(zip_file, root_dir, file_accessor)); + return std::unique_ptr(new ZipWriter(zip_file, file_accessor)); } #endif // static std::unique_ptr ZipWriter::Create( const base::FilePath& zip_file_path, - const base::FilePath& root_dir, FileAccessor* file_accessor) { DCHECK(!zip_file_path.empty()); zipFile zip_file = internal::OpenForZipping(zip_file_path.AsUTF8Unsafe(), @@ -125,14 +135,11 @@ std::unique_ptr ZipWriter::Create( return nullptr; } - return std::unique_ptr( - new ZipWriter(zip_file, root_dir, file_accessor)); + return std::unique_ptr(new ZipWriter(zip_file, file_accessor)); } -ZipWriter::ZipWriter(zipFile zip_file, - const base::FilePath& root_dir, - FileAccessor* file_accessor) - : zip_file_(zip_file), root_dir_(root_dir), file_accessor_(file_accessor) {} +ZipWriter::ZipWriter(zipFile zip_file, FileAccessor* file_accessor) + : zip_file_(zip_file), file_accessor_(file_accessor) {} ZipWriter::~ZipWriter() { if (zip_file_) @@ -157,8 +164,8 @@ bool ZipWriter::Close() { } bool ZipWriter::AddEntries(Paths paths) { - // Constructed outside the loop in order to reuse its internal buffer. - std::vector absolute_paths; + // Declared outside loop to reuse internal buffer. + std::vector files; while (!paths.empty()) { // Work with chunks of 50 paths at most. @@ -166,48 +173,27 @@ bool ZipWriter::AddEntries(Paths paths) { const Paths relative_paths = paths.subspan(0, n); paths = paths.subspan(n, paths.size() - n); - // FileAccessor requires absolute paths. - absolute_paths.clear(); - absolute_paths.reserve(n); - for (const base::FilePath& relative_path : relative_paths) { - absolute_paths.push_back(root_dir_.Append(relative_path)); - } - - DCHECK_EQ(relative_paths.size(), n); - DCHECK_EQ(absolute_paths.size(), n); - // We don't know which paths are files and which ones are directories, and // we want to avoid making a call to file_accessor_ for each entry. Try to // open all of the paths as files. We'll get invalid file descriptors for // directories. - std::vector files = - file_accessor_->OpenFilesForReading(absolute_paths); - DCHECK_EQ(files.size(), n); + files.clear(); + if (!file_accessor_->Open(relative_paths, &files) || files.size() != n) + return false; for (size_t i = 0; i < n; i++) { const base::FilePath& relative_path = relative_paths[i]; - const base::FilePath& absolute_path = absolute_paths[i]; + DCHECK(!relative_path.empty()); base::File& file = files[i]; if (file.IsValid()) { - if (!AddFileEntry(relative_path, std::move(file))) { - LOG(ERROR) << "Cannot add file '" << relative_path << "' to ZIP"; + // It's a file. + if (!AddFileEntry(relative_path, std::move(file))) return false; - } } else { - // Either directory or missing file. - const base::Time last_modified = - file_accessor_->GetLastModifiedTime(absolute_path); - if (last_modified.is_null()) { - LOG(ERROR) << "Missing file or directory '" << relative_path << "'"; - return false; - } - - DCHECK(file_accessor_->DirectoryExists(absolute_path)); - if (!AddDirectoryEntry(relative_path, last_modified)) { - LOG(ERROR) << "Cannot add directory '" << relative_path << "' to ZIP"; + // It's probably a directory. + if (!AddDirectoryEntry(relative_path)) return false; - } } } } diff --git a/google/zip_writer.h b/google/zip_writer.h index 489984f..e9736a3 100644 --- a/google/zip_writer.h +++ b/google/zip_writer.h @@ -29,19 +29,21 @@ namespace internal { // performance reasons as these calls may be expensive when IPC based). // This class is so far internal and only used by zip.cc, but could be made // public if needed. +// +// All methods returning a bool return true on success and false on error. class ZipWriter { public: -// Creates a writer that will write a ZIP file to |zip_file_fd|/|zip_file| -// and which entries (specified with WriteEntries) are relative to |root_dir|. +// Creates a writer that will write a ZIP file to |zip_file_fd| or |zip_file| +// and which entries are relative to |file_accessor|'s source directory. // All file reads are performed using |file_accessor|. #if defined(OS_POSIX) static std::unique_ptr CreateWithFd(int zip_file_fd, - const base::FilePath& root_dir, FileAccessor* file_accessor); #endif + static std::unique_ptr Create(const base::FilePath& zip_file, - const base::FilePath& root_dir, FileAccessor* file_accessor); + ~ZipWriter(); // Sets the optional progress callback. The callback is called once for each @@ -60,9 +62,7 @@ class ZipWriter { private: // Takes ownership of |zip_file|. - ZipWriter(zipFile zip_file, - const base::FilePath& root_dir, - FileAccessor* file_accessor); + ZipWriter(zipFile zip_file, FileAccessor* file_accessor); // Regularly called during processing to check whether zipping should continue // or should be cancelled. @@ -79,7 +79,7 @@ class ZipWriter { bool AddFileEntry(const base::FilePath& path, base::File file); // Adds a directory entry. - bool AddDirectoryEntry(const base::FilePath& path, base::Time last_modified); + bool AddDirectoryEntry(const base::FilePath& path); // Opens a file or directory entry. bool OpenNewFileEntry(const base::FilePath& path, @@ -97,11 +97,8 @@ class ZipWriter { // The actual zip file. zipFile zip_file_; - // Path to the directory entry paths are relative to. - base::FilePath root_dir_; - // Abstraction over file access methods used to read files. - FileAccessor* file_accessor_; + FileAccessor* const file_accessor_; // Progress stats. Progress progress_; -- cgit v1.2.3 From 2865d44621369a4d9ce1a7e9cad621ca0594895f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 25 May 2021 03:35:57 +0000 Subject: [zip] Use DFS instead of BFS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changed the zip tree exploration algorithm from Breadth-First Search (BFS) to Depth-First_Search (DFS). This has several benefits. There is no need to create a list of all the items to zip from the start. Entries are added to the ZIP as they are discovered, reducing the need for string copies, memory allocations, etc. The order of items in the resulting ZIP is more natural, and is on par with what other archivers produce. Knowledge of which entries are files or folders is directly used to do the right thing, reducing the number of file system accesses. Previously, this information wasn't recorded in the BFS-produced list, and it had to be rediscovered during zipping at the cost of extra file system calls. Finally, this allows to recursively zip folders presented in the list of items to zip. BUG=chromium:912236 TEST=out/release/zlib_unittests TEST=out/release/browser_tests --gtest_filter="ZipFileCreatorTest* Change-Id: I857d758573bfbb3ad2e5fe922581e5cb71af4691 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2912020 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#886166} NOKEYCHECK=True GitOrigin-RevId: b47162db7fd32bf78037e258e18a3a8c28a2a8ae --- google/zip.cc | 78 ++++++++++++-------------------- google/zip.h | 21 ++++++--- google/zip_writer.cc | 125 +++++++++++++++++++++++++++++++++++++++++++-------- google/zip_writer.h | 52 +++++++++++++++------ 4 files changed, 187 insertions(+), 89 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index e83908c..7c9bd61 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -4,7 +4,6 @@ #include "third_party/zlib/google/zip.h" -#include #include #include @@ -30,10 +29,6 @@ bool ExcludeNoFilesFilter(const base::FilePath& file_path) { return true; } -bool ExcludeHiddenFilesFilter(const base::FilePath& file_path) { - return !IsHiddenFile(file_path); -} - // Creates a directory at |extract_dir|/|entry_path|, including any parents. bool CreateDirectory(const base::FilePath& extract_dir, const base::FilePath& entry_path) { @@ -127,41 +122,6 @@ bool Zip(const ZipParams& params) { DirectFileAccessor default_accessor(params.src_dir); FileAccessor* const file_accessor = params.file_accessor ?: &default_accessor; - Paths files_to_add = params.src_files; - - std::vector all_files; - if (files_to_add.empty()) { - const auto exclude = [¶ms](const base::FilePath& path) { - return (!params.include_hidden_files && IsHiddenFile(path)) || - (params.filter_callback && - !params.filter_callback.Run(params.src_dir.Append(path))); - }; - - // Perform a Breadth First Search (BFS) of the source tree. Note that the - // BFS order might not be optimal when storing files in a ZIP (either for - // the storing side, or for the program that will extract this ZIP). - for (std::queue q({{}}); !q.empty(); q.pop()) { - std::vector files, dirs; - file_accessor->List(q.front(), &files, &dirs); - - for (base::FilePath& path : files) { - // Skip hidden and filtered files. - if (!exclude(path)) - all_files.push_back(std::move(path)); - } - - for (base::FilePath& path : dirs) { - // Skip hidden and filtered subdirs. - if (!exclude(path)) { - q.push(path); - all_files.push_back(std::move(path)); - } - } - } - - files_to_add = all_files; - } - std::unique_ptr zip_writer; #if defined(OS_POSIX) @@ -182,8 +142,29 @@ bool Zip(const ZipParams& params) { zip_writer->SetProgressCallback(params.progress_callback, params.progress_period); + zip_writer->SetRecursive(params.recursive); + + if (!params.include_hidden_files || params.filter_callback) + zip_writer->SetFilterCallback(base::BindRepeating( + [](const ZipParams* const params, const base::FilePath& path) -> bool { + return (params->include_hidden_files || !IsHiddenFile(path)) && + (!params->filter_callback || + params->filter_callback.Run(params->src_dir.Append(path))); + }, + ¶ms)); + + if (params.src_files.empty()) { + // No source items are specified. Zip the entire source directory. + zip_writer->SetRecursive(true); + if (!zip_writer->AddDirectoryContents(base::FilePath())) + return false; + } else { + // Only zip the specified source items. + if (!zip_writer->AddMixedEntries(params.src_files)) + return false; + } - return zip_writer->WriteEntries(files_to_add); + return zip_writer->Close(); } bool Unzip(const base::FilePath& src_file, const base::FilePath& dest_dir) { @@ -197,9 +178,10 @@ bool UnzipWithFilterCallback(const base::FilePath& src_file, bool log_skipped_files) { base::File file(src_file, base::File::FLAG_OPEN | base::File::FLAG_READ); if (!file.IsValid()) { - DLOG(WARNING) << "Failed to open " << src_file.value(); + DLOG(WARNING) << "Cannot open '" << src_file << "'"; return false; } + return UnzipWithFilterAndWriters( file.GetPlatformFile(), base::BindRepeating(&CreateFilePathWriterDelegate, dest_dir), @@ -214,7 +196,7 @@ bool UnzipWithFilterAndWriters(const base::PlatformFile& src_file, bool log_skipped_files) { ZipReader reader; if (!reader.OpenFromPlatformFile(src_file)) { - DLOG(WARNING) << "Failed to open src_file " << src_file; + DLOG(WARNING) << "Cannot open '" << src_file << "'"; return false; } while (reader.HasMore()) { @@ -263,13 +245,9 @@ bool ZipWithFilterCallback(const base::FilePath& src_dir, bool Zip(const base::FilePath& src_dir, const base::FilePath& dest_file, bool include_hidden_files) { - if (include_hidden_files) { - return ZipWithFilterCallback(src_dir, dest_file, - base::BindRepeating(&ExcludeNoFilesFilter)); - } else { - return ZipWithFilterCallback( - src_dir, dest_file, base::BindRepeating(&ExcludeHiddenFilesFilter)); - } + return Zip({.src_dir = src_dir, + .dest_file = dest_file, + .include_hidden_files = include_hidden_files}); } #if defined(OS_POSIX) diff --git a/google/zip.h b/google/zip.h index e12f446..382eba1 100644 --- a/google/zip.h +++ b/google/zip.h @@ -97,16 +97,20 @@ struct ZipParams { int dest_fd = base::kInvalidPlatformFile; #endif - // The relative paths to the files that should be included in the ZIP file. If - // this is empty, all files in |src_dir| are included. + // The relative paths to the files and directories that should be included in + // the ZIP file. If this is empty, the whole contents of |src_dir| are + // included. // // These paths must be relative to |src_dir| and will be used as the file - // names in the created zip file. All files must be under |src_dir| in the + // names in the created ZIP file. All files must be under |src_dir| in the // file system hierarchy. + // + // All the paths in |src_files| are included in the created ZIP file, + // irrespective of |include_hidden_files| and |filter_callback|. Paths src_files; - // Filter used to exclude files from the ZIP file. Only effective when - // |src_files| is empty. + // Filter used to exclude files from the ZIP file. This is only taken in + // account when recursively adding subdirectory contents. FilterCallback filter_callback; // Optional progress reporting callback. @@ -116,10 +120,13 @@ struct ZipParams { // creation operation completes. base::TimeDelta progress_period; - // Whether hidden files should be included in the ZIP file. Only effective - // when |src_files| is empty. + // Should add hidden files? This is only taken in account when recursively + // adding subdirectory contents. bool include_hidden_files = true; + // Should recursively add subdirectory contents? + bool recursive = false; + // Abstraction around file system access used to read files. If left null, an // implementation that accesses files directly is used. FileAccessor* file_accessor = nullptr; // Not owned diff --git a/google/zip_writer.cc b/google/zip_writer.cc index 26d9c13..b6f3365 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -4,6 +4,8 @@ #include "third_party/zlib/google/zip_writer.h" +#include + #include "base/files/file.h" #include "base/logging.h" #include "base/strings/string_util.h" @@ -72,16 +74,18 @@ bool ZipWriter::CloseNewFileEntry() { } bool ZipWriter::AddFileEntry(const base::FilePath& path, base::File file) { - base::File::Info file_info; - if (!file.GetInfo(&file_info)) + base::File::Info info; + if (!file.GetInfo(&info)) + return false; + + if (!OpenNewFileEntry(path, /*is_directory=*/false, info.last_modified)) return false; - if (!OpenNewFileEntry(path, /*is_directory=*/false, file_info.last_modified)) + if (!AddFileContent(path, std::move(file))) return false; - const bool success = AddFileContent(path, std::move(file)); progress_.files++; - return CloseNewFileEntry() && success; + return CloseNewFileEntry(); } bool ZipWriter::AddDirectoryEntry(const base::FilePath& path) { @@ -101,7 +105,13 @@ bool ZipWriter::AddDirectoryEntry(const base::FilePath& path) { return false; progress_.directories++; - return ShouldContinue(); + if (!ShouldContinue()) + return false; + + if (!recursive_) + return true; + + return AddDirectoryContents(path); } #if defined(OS_POSIX) @@ -146,10 +156,6 @@ ZipWriter::~ZipWriter() { zipClose(zip_file_, nullptr); } -bool ZipWriter::WriteEntries(Paths paths) { - return AddEntries(paths) && Close(); -} - bool ZipWriter::Close() { const bool success = zipClose(zip_file_, nullptr) == ZIP_OK; zip_file_ = nullptr; @@ -163,20 +169,24 @@ bool ZipWriter::Close() { return success; } -bool ZipWriter::AddEntries(Paths paths) { +bool ZipWriter::AddMixedEntries(Paths paths) { + // Pointers to directory paths in |paths|. + std::vector directories; + // Declared outside loop to reuse internal buffer. std::vector files; + // First pass. We don't know which paths are files and which ones are + // directories, and we want to avoid making a call to file_accessor_ for each + // path. Try to open all of the paths as files. We'll get invalid file + // descriptors for directories, and we'll process these directories in the + // second pass. while (!paths.empty()) { // Work with chunks of 50 paths at most. const size_t n = std::min(paths.size(), 50); const Paths relative_paths = paths.subspan(0, n); paths = paths.subspan(n, paths.size() - n); - // We don't know which paths are files and which ones are directories, and - // we want to avoid making a call to file_accessor_ for each entry. Try to - // open all of the paths as files. We'll get invalid file descriptors for - // directories. files.clear(); if (!file_accessor_->Open(relative_paths, &files) || files.size() != n) return false; @@ -191,15 +201,94 @@ bool ZipWriter::AddEntries(Paths paths) { if (!AddFileEntry(relative_path, std::move(file))) return false; } else { - // It's probably a directory. - if (!AddDirectoryEntry(relative_path)) - return false; + // It's probably a directory. Remember its path for the second pass. + directories.push_back(&relative_path); } } } + // Second pass for directories discovered during the first pass. + for (const base::FilePath* const path : directories) { + DCHECK(path); + if (!AddDirectoryEntry(*path)) + return false; + } + return true; } +bool ZipWriter::AddFileEntries(Paths paths) { + // Declared outside loop to reuse internal buffer. + std::vector files; + + while (!paths.empty()) { + // Work with chunks of 50 paths at most. + const size_t n = std::min(paths.size(), 50); + const Paths relative_paths = paths.subspan(0, n); + paths = paths.subspan(n, paths.size() - n); + + DCHECK_EQ(relative_paths.size(), n); + + files.clear(); + if (!file_accessor_->Open(relative_paths, &files) || files.size() != n) { + LOG(ERROR) << "Cannot open " << n << " files"; + return false; + } + + for (size_t i = 0; i < n; i++) { + const base::FilePath& relative_path = relative_paths[i]; + DCHECK(!relative_path.empty()); + base::File& file = files[i]; + + if (!file.IsValid()) { + LOG(ERROR) << "Cannot open '" << relative_path << "'"; + return false; + } + + if (!AddFileEntry(relative_path, std::move(file))) + return false; + } + } + + return true; +} + +bool ZipWriter::AddDirectoryEntries(Paths paths) { + for (const base::FilePath& path : paths) { + if (!AddDirectoryEntry(path)) + return false; + } + + return true; +} + +bool ZipWriter::AddDirectoryContents(const base::FilePath& path) { + std::vector files, subdirs; + + if (!file_accessor_->List(path, &files, &subdirs)) + return false; + + Filter(&files); + Filter(&subdirs); + + if (!AddFileEntries(files)) + return false; + + return AddDirectoryEntries(subdirs); +} + +void ZipWriter::Filter(std::vector* const paths) { + DCHECK(paths); + + if (!filter_callback_) + return; + + const auto end = std::remove_if(paths->begin(), paths->end(), + [this](const base::FilePath& path) { + return !filter_callback_.Run(path); + }); + paths->erase(end, paths->end()); +} + } // namespace internal } // namespace zip diff --git a/google/zip_writer.h b/google/zip_writer.h index e9736a3..c67903d 100644 --- a/google/zip_writer.h +++ b/google/zip_writer.h @@ -54,11 +54,26 @@ class ZipWriter { progress_period_ = std::move(period); } - // Writes the files at |paths| to the ZIP file and closes this ZIP file. - // The file paths must be relative to |root_dir| specified in the - // Create method. - // Returns true if all entries were written successfuly. - bool WriteEntries(Paths paths); + // Sets the recursive flag, indicating whether the contents of subdirectories + // should be included. + void SetRecursive(bool b) { recursive_ = b; } + + // Sets the filter callback. + void SetFilterCallback(FilterCallback callback) { + filter_callback_ = std::move(callback); + } + + // Adds the contents of a directory. If the recursive flag is set, the + // contents of subdirectories are also added. + bool AddDirectoryContents(const base::FilePath& path); + + // Adds the entries at |paths| to the ZIP file. These can be a mixed bag of + // files and directories. If the recursive flag is set, the contents of + // subdirectories is also added. + bool AddMixedEntries(Paths paths); + + // Closes the ZIP file. + bool Close(); private: // Takes ownership of |zip_file|. @@ -68,19 +83,24 @@ class ZipWriter { // or should be cancelled. bool ShouldContinue(); - // Adds the files at |paths| to the ZIP file. These FilePaths must be relative - // to |root_dir| specified in the Create method. - bool AddEntries(Paths paths); - // Adds file content to currently open file entry. bool AddFileContent(const base::FilePath& path, base::File file); // Adds a file entry (including file contents). bool AddFileEntry(const base::FilePath& path, base::File file); - // Adds a directory entry. + // Adds file entries. All the paths should be existing files. + bool AddFileEntries(Paths paths); + + // Adds a directory entry. If the recursive flag is set, the contents of this + // directory are also added. bool AddDirectoryEntry(const base::FilePath& path); + // Adds directory entries. All the paths should be existing directories. If + // the recursive flag is set, the contents of these directories are also + // added. + bool AddDirectoryEntries(Paths paths); + // Opens a file or directory entry. bool OpenNewFileEntry(const base::FilePath& path, bool is_directory, @@ -89,10 +109,8 @@ class ZipWriter { // Closes the currently open entry. bool CloseNewFileEntry(); - // Closes the ZIP file. - // Returns true if successful, false otherwise (typically if an entry failed - // to be written). - bool Close(); + // Filters entries. + void Filter(std::vector* paths); // The actual zip file. zipFile zip_file_; @@ -112,6 +130,12 @@ class ZipWriter { // Next time to report progress. base::TimeTicks next_progress_report_time_ = base::TimeTicks::Now(); + // Filter used to exclude files from the ZIP file. + FilterCallback filter_callback_; + + // Should recursively add directories? + bool recursive_ = false; + DISALLOW_COPY_AND_ASSIGN(ZipWriter); }; -- cgit v1.2.3 From e4c7c48174629a2258b373615a4de843675077c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 25 May 2021 10:29:08 +0000 Subject: [ZipFileCreator] Recursively zip folders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ZipFileCreator service now recursively zips folders. Files App doesn't need to enumerate all the files in folders to zip anymore. This is done in a much more efficient way by ZipFileCreator. When zipping a folder containing 100 subfolders with 1000 small files each (for a total of 100,000 small files) on a kevin device, the old system takes 144 seconds (listing files 103 seconds + zipping files 41 seconds), whereas the new system takes only 26 seconds. This is an improvement by a factor of 5.5. BUG=chromium:912236 TEST=out/release/zlib_unittests TEST=out/release/browser_tests --gtest_filter="ZipFileCreatorTest* Change-Id: I049e25fbc365528c4d9ec12ac2afc58b4d036046 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2912233 Reviewed-by: Dominick Ng Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#886233} NOKEYCHECK=True GitOrigin-RevId: 9a8e1c0ce544d4fdb9b02640aed2bc83c3162ebd --- google/zip.h | 10 +++++----- google/zip_unittest.cc | 6 ++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/google/zip.h b/google/zip.h index 382eba1..ae08a24 100644 --- a/google/zip.h +++ b/google/zip.h @@ -84,9 +84,13 @@ using FilterCallback = base::RepeatingCallback; // ZIP creation parameters and options. struct ZipParams { - // Source directory. + // Source directory. Ignored if |file_accessor| is set. base::FilePath src_dir; + // Abstraction around file system access used to read files. + // If left null, an implementation that accesses files directly is used. + FileAccessor* file_accessor = nullptr; // Not owned + // Destination file path. // Either dest_file or dest_fd should be set, but not both. base::FilePath dest_file; @@ -126,10 +130,6 @@ struct ZipParams { // Should recursively add subdirectory contents? bool recursive = false; - - // Abstraction around file system access used to read files. If left null, an - // implementation that accesses files directly is used. - FileAccessor* file_accessor = nullptr; // Not owned }; // Zip files specified into a ZIP archives. The source files and ZIP destination diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 558fedd..2d2e66a 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -567,10 +567,8 @@ TEST_F(ZipTest, ZipWithFileAccessor) { base::FilePath zip_file; ASSERT_TRUE(base::CreateTemporaryFile(&zip_file)); VirtualFileSystem file_accessor; - const zip::ZipParams params{ - .src_dir = base::FilePath(FILE_PATH_LITERAL("/test")), - .dest_file = zip_file, - .file_accessor = &file_accessor}; + const zip::ZipParams params{.file_accessor = &file_accessor, + .dest_file = zip_file}; ASSERT_TRUE(zip::Zip(params)); base::ScopedTempDir scoped_temp_dir; -- cgit v1.2.3 From 35aa87e1aefabcaa049b300f90b50d7a573c838d Mon Sep 17 00:00:00 2001 From: Noel Gordon Date: Wed, 26 May 2021 12:35:42 +0000 Subject: [zlib_bench] Add file size --field width option Some test corpora have large file sizes and make the zlib bench output not line-up, more difficult to read. Fix this: add file size --field width option, to control to width used to display input file and compressed size values. The default is 6 and more space can be requested with --width 8 for example. Minor: use 2 decimal points of precision to show the compression rate. This helps verify that the compression rate is improving (or not), for some corpora files, when comparing results over the zlib --compression levels 0 thru 9. Change options getters to use out parameter &value. Bug: 798943, 1070655 Change-Id: I0142d0740cd7861aa64be737806d7aba2dd05b00 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2916460 Auto-Submit: Noel Gordon Commit-Queue: Noel Gordon Reviewed-by: Adenilson Cavalcanti Cr-Commit-Position: refs/heads/master@{#886691} NOKEYCHECK=True GitOrigin-RevId: b4ef756d3204cc05972cea9b61f61e8927577e94 --- contrib/bench/zlib_bench.cc | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/contrib/bench/zlib_bench.cc b/contrib/bench/zlib_bench.cc index bd06ad3..252560e 100644 --- a/contrib/bench/zlib_bench.cc +++ b/contrib/bench/zlib_bench.cc @@ -193,7 +193,7 @@ void verify_equal(const char* input, size_t size, std::string* output) { exit(3); } -void zlib_file(const char* name, const zlib_wrapper type) { +void zlib_file(const char* name, const zlib_wrapper type, int width) { /* * Read the file data. */ @@ -283,9 +283,9 @@ void zlib_file(const char* name, const zlib_wrapper type) { double inflate_rate_max = length * repeats / mega_byte / utime[0]; // type, block size, compression ratio, etc - printf("%s: [b %dM] bytes %6d -> %6u %4.1f%%", - zlib_wrapper_name(type), block_size / (1 << 20), length, - static_cast(output_length), output_length * 100.0 / length); + printf("%s: [b %dM] bytes %*d -> %*u %4.2f%%", + zlib_wrapper_name(type), block_size / (1 << 20), width, length, width, + unsigned(output_length), output_length * 100.0 / length); // compress / uncompress median (max) rates printf(" comp %5.1f (%5.1f) MB/s uncomp %5.1f (%5.1f) MB/s\n", @@ -300,16 +300,20 @@ char* get_option(int argc, char* argv[], const char* option) { return nullptr; } -bool get_compression(int argc, char* argv[], int* value) { +bool get_compression(int argc, char* argv[], int& value) { if (argn < argc) - *value = isdigit(argv[argn][0]) ? atoi(argv[argn++]) : -1; - return *value >= 0 && *value <= 9; + value = isdigit(argv[argn][0]) ? atoi(argv[argn++]) : -1; + return value >= 0 && value <= 9; } -const char* options = "gzip|zlib|raw [--compression 0:9] [--huffman|--rle]"; +void get_field_width(int argc, char* argv[], int& value) { + value = atoi(argv[argn++]); +} void usage_exit(const char* program) { - printf("usage: %s %s files...", program, options); + static auto* options = + "gzip|zlib|raw [--compression 0:9] [--huffman|--rle] [--field width]"; + printf("usage: %s %s files ...\n", program, options); exit(1); } @@ -324,10 +328,14 @@ int main(int argc, char* argv[]) { else usage_exit(argv[0]); + int file_size_field_width = 0; + while (argn < argc && argv[argn][0] == '-') { if (get_option(argc, argv, "--compression")) { - if (!get_compression(argc, argv, &zlib_compression_level)) + if (!get_compression(argc, argv, zlib_compression_level)) usage_exit(argv[0]); + } else if (get_option(argc, argv, "--field")) { + get_field_width(argc, argv, file_size_field_width); } else if (get_option(argc, argv, "--huffman")) { zlib_strategy = Z_HUFFMAN_ONLY; } else if (get_option(argc, argv, "--rle")) { @@ -339,8 +347,11 @@ int main(int argc, char* argv[]) { if (argn >= argc) usage_exit(argv[0]); + + if (file_size_field_width < 6) + file_size_field_width = 6; while (argn < argc) - zlib_file(argv[argn++], type); + zlib_file(argv[argn++], type, file_size_field_width); return 0; } -- cgit v1.2.3 From 5ef44f037eea3aa532b8470a090f392a4b0830bd Mon Sep 17 00:00:00 2001 From: Noel Gordon Date: Thu, 27 May 2021 01:07:15 +0000 Subject: [zlib] Build minizip zip and unzip tools Minizip is a library provided by //third_party/zlib. Its zip and unzip tools can be built in a developer checkout for testing purposes with: autoninja -C out/Release minizip_bin autoninja -C out/Release miniunz_bin Add GN build rules for these files. Patch minizip and miniunz tools so they compile. Add a patch file and minizip.md for usage. On Android, disable large file support (see bug comment #12). off_t is 32 bits (instead of 64 bits) and should compile on the bots where this patch failed compile before [1] and was reverted (CL:2895825). [1] Added CQ android_asan, android_archive_rel_ng try jobs to pre-test that compile works on these waterfall builders. Bug: 1207895 Change-Id: Idbb3414a2c14fc541e31d6eff97859245296003f Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2919568 Reviewed-by: Adenilson Cavalcanti Commit-Queue: Noel Gordon Cr-Commit-Position: refs/heads/master@{#886990} NOKEYCHECK=True GitOrigin-RevId: a45e197d9695de4214f27e4c218b82e256b10d06 --- BUILD.gn | 40 ++++++++++++ contrib/minizip/miniunz.c | 13 ++-- contrib/minizip/minizip.c | 7 +-- contrib/minizip/minizip.md | 9 +++ patches/0008-minizip-zip-unzip-tools.patch | 98 ++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 11 deletions(-) create mode 100644 contrib/minizip/minizip.md create mode 100644 patches/0008-minizip-zip-unzip-tools.patch diff --git a/BUILD.gn b/BUILD.gn index d64cb38..c7dfafa 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -419,6 +419,46 @@ executable("zlib_bench") { configs += [ "//build/config/compiler:no_chromium_code" ] } +executable("minizip_bin") { + include_dirs = [ "." ] + + sources = [ "contrib/minizip/minizip.c" ] + + if (is_clang) { + cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] + } + + if (!is_debug) { + configs -= [ "//build/config/compiler:default_optimization" ] + configs += [ "//build/config/compiler:optimize_speed" ] + } + + deps = [ ":minizip" ] + + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] +} + +executable("miniunz_bin") { + include_dirs = [ "." ] + + sources = [ "contrib/minizip/miniunz.c" ] + + if (is_clang) { + cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] + } + + if (!is_debug) { + configs -= [ "//build/config/compiler:default_optimization" ] + configs += [ "//build/config/compiler:optimize_speed" ] + } + + deps = [ ":minizip" ] + + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] +} + if (build_with_chromium) { test("zlib_unittests") { testonly = true diff --git a/contrib/minizip/miniunz.c b/contrib/minizip/miniunz.c index 3d65401..08737f6 100644 --- a/contrib/minizip/miniunz.c +++ b/contrib/minizip/miniunz.c @@ -12,7 +12,7 @@ Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) */ -#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) && (!defined(__ANDROID_API__)) #ifndef __USE_FILE_OFFSET64 #define __USE_FILE_OFFSET64 #endif @@ -27,7 +27,7 @@ #endif #endif -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions #define FOPEN_FUNC(filename, mode) fopen(filename, mode) #define FTELLO_FUNC(stream) ftello(stream) @@ -45,6 +45,7 @@ #include #include #include +#include #ifdef _WIN32 # include @@ -97,7 +98,7 @@ void change_file_date(filename,dosdate,tmu_date) SetFileTime(hFile,&ftm,&ftLastAcc,&ftm); CloseHandle(hFile); #else -#ifdef unix || __APPLE__ +#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) struct utimbuf ut; struct tm newdate; newdate.tm_sec = tmu_date.tm_sec; @@ -125,11 +126,9 @@ int mymkdir(dirname) const char* dirname; { int ret=0; -#ifdef _WIN32 +#if defined(_WIN32) ret = _mkdir(dirname); -#elif unix - ret = mkdir (dirname,0775); -#elif __APPLE__ +#elif defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) ret = mkdir (dirname,0775); #endif return ret; diff --git a/contrib/minizip/minizip.c b/contrib/minizip/minizip.c index 4288962..b794953 100644 --- a/contrib/minizip/minizip.c +++ b/contrib/minizip/minizip.c @@ -12,8 +12,7 @@ Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) */ - -#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) && (!defined(__ANDROID_API__)) #ifndef __USE_FILE_OFFSET64 #define __USE_FILE_OFFSET64 #endif @@ -28,7 +27,7 @@ #endif #endif -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions #define FOPEN_FUNC(filename, mode) fopen(filename, mode) #define FTELLO_FUNC(stream) ftello(stream) @@ -94,7 +93,7 @@ uLong filetime(f, tmzip, dt) return ret; } #else -#ifdef unix || __APPLE__ +#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) uLong filetime(f, tmzip, dt) char *f; /* name of file to get info on */ tm_zip *tmzip; /* return value: access, modific. and creation times */ diff --git a/contrib/minizip/minizip.md b/contrib/minizip/minizip.md new file mode 100644 index 0000000..9f15dd2 --- /dev/null +++ b/contrib/minizip/minizip.md @@ -0,0 +1,9 @@ +Minizip is a library provided by //third_party/zlib [1]. Its zip and unzip +tools can be built in a developer checkout for testing purposes with: + +```shell + autoninja -C out/Release minizip_bin + autoninja -C out/Release miniunz_bin +``` + +[1] Upstream is https://github.com/madler/zlib/tree/master/contrib/minizip diff --git a/patches/0008-minizip-zip-unzip-tools.patch b/patches/0008-minizip-zip-unzip-tools.patch new file mode 100644 index 0000000..48ceb02 --- /dev/null +++ b/patches/0008-minizip-zip-unzip-tools.patch @@ -0,0 +1,98 @@ +From 0c7de17000659f4f79de878296892c46be0aff77 Mon Sep 17 00:00:00 2001 +From: Noel Gordon +Date: Wed, 26 May 2021 21:57:43 +1000 +Subject: [PATCH] Build minizip zip and unzip tools + +--- + third_party/zlib/contrib/minizip/miniunz.c | 13 ++++++------- + third_party/zlib/contrib/minizip/minizip.c | 7 +++---- + 2 files changed, 9 insertions(+), 11 deletions(-) + +diff --git a/third_party/zlib/contrib/minizip/miniunz.c b/third_party/zlib/contrib/minizip/miniunz.c +index 3d65401be5cd..08737f689a96 100644 +--- a/third_party/zlib/contrib/minizip/miniunz.c ++++ b/third_party/zlib/contrib/minizip/miniunz.c +@@ -12,7 +12,7 @@ + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + */ + +-#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) ++#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) && (!defined(__ANDROID_API__)) + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif +@@ -27,7 +27,7 @@ + #endif + #endif + +-#ifdef __APPLE__ ++#if defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions + #define FOPEN_FUNC(filename, mode) fopen(filename, mode) + #define FTELLO_FUNC(stream) ftello(stream) +@@ -45,6 +45,7 @@ + #include + #include + #include ++#include + + #ifdef _WIN32 + # include +@@ -97,7 +98,7 @@ void change_file_date(filename,dosdate,tmu_date) + SetFileTime(hFile,&ftm,&ftLastAcc,&ftm); + CloseHandle(hFile); + #else +-#ifdef unix || __APPLE__ ++#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + struct utimbuf ut; + struct tm newdate; + newdate.tm_sec = tmu_date.tm_sec; +@@ -125,11 +126,9 @@ int mymkdir(dirname) + const char* dirname; + { + int ret=0; +-#ifdef _WIN32 ++#if defined(_WIN32) + ret = _mkdir(dirname); +-#elif unix +- ret = mkdir (dirname,0775); +-#elif __APPLE__ ++#elif defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + ret = mkdir (dirname,0775); + #endif + return ret; +diff --git a/third_party/zlib/contrib/minizip/minizip.c b/third_party/zlib/contrib/minizip/minizip.c +index 4288962ecef0..b794953c5c23 100644 +--- a/third_party/zlib/contrib/minizip/minizip.c ++++ b/third_party/zlib/contrib/minizip/minizip.c +@@ -12,8 +12,7 @@ + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + */ + +- +-#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) ++#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) && (!defined(__ANDROID_API__)) + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif +@@ -28,7 +27,7 @@ + #endif + #endif + +-#ifdef __APPLE__ ++#if defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions + #define FOPEN_FUNC(filename, mode) fopen(filename, mode) + #define FTELLO_FUNC(stream) ftello(stream) +@@ -94,7 +93,7 @@ uLong filetime(f, tmzip, dt) + return ret; + } + #else +-#ifdef unix || __APPLE__ ++#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + uLong filetime(f, tmzip, dt) + char *f; /* name of file to get info on */ + tm_zip *tmzip; /* return value: access, modific. and creation times */ +-- +2.31.1.818.g46aad6cb9e-goog + -- cgit v1.2.3 From d0e636edaa95e2e04f56b84014f6b5f799acf0f0 Mon Sep 17 00:00:00 2001 From: Jamie Madill Date: Thu, 27 May 2021 18:15:40 +0000 Subject: [zlib] Do not build zip/unzip tools on WinUWP. These tools call IO APIs not available in UWP. Bug: 1207895 Change-Id: I1fd627a72df29f1872d8086e4b0fe981b60999b0 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2920671 Auto-Submit: Jamie Madill Reviewed-by: Adenilson Cavalcanti Commit-Queue: Adenilson Cavalcanti Commit-Queue: Jamie Madill Cr-Commit-Position: refs/heads/master@{#887243} NOKEYCHECK=True GitOrigin-RevId: f7ffb67fa031795aacbe33954ff2a1c845d4cf2c --- BUILD.gn | 56 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/BUILD.gn b/BUILD.gn index c7dfafa..74ecbd1 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -419,44 +419,46 @@ executable("zlib_bench") { configs += [ "//build/config/compiler:no_chromium_code" ] } -executable("minizip_bin") { - include_dirs = [ "." ] +if (!is_win || target_os != "winuwp") { + executable("minizip_bin") { + include_dirs = [ "." ] - sources = [ "contrib/minizip/minizip.c" ] + sources = [ "contrib/minizip/minizip.c" ] - if (is_clang) { - cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] - } + if (is_clang) { + cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] + } - if (!is_debug) { - configs -= [ "//build/config/compiler:default_optimization" ] - configs += [ "//build/config/compiler:optimize_speed" ] - } + if (!is_debug) { + configs -= [ "//build/config/compiler:default_optimization" ] + configs += [ "//build/config/compiler:optimize_speed" ] + } - deps = [ ":minizip" ] + deps = [ ":minizip" ] - configs -= [ "//build/config/compiler:chromium_code" ] - configs += [ "//build/config/compiler:no_chromium_code" ] -} + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] + } -executable("miniunz_bin") { - include_dirs = [ "." ] + executable("miniunz_bin") { + include_dirs = [ "." ] - sources = [ "contrib/minizip/miniunz.c" ] + sources = [ "contrib/minizip/miniunz.c" ] - if (is_clang) { - cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] - } + if (is_clang) { + cflags = [ "-Wno-incompatible-pointer-types-discards-qualifiers" ] + } - if (!is_debug) { - configs -= [ "//build/config/compiler:default_optimization" ] - configs += [ "//build/config/compiler:optimize_speed" ] - } + if (!is_debug) { + configs -= [ "//build/config/compiler:default_optimization" ] + configs += [ "//build/config/compiler:optimize_speed" ] + } - deps = [ ":minizip" ] + deps = [ ":minizip" ] - configs -= [ "//build/config/compiler:chromium_code" ] - configs += [ "//build/config/compiler:no_chromium_code" ] + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] + } } if (build_with_chromium) { -- cgit v1.2.3 From 00ade15d946d72f75c786dc2e66c419a9d99e2ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 2 Jun 2021 02:30:22 +0000 Subject: [ZipFileCreator] Close directory Mojo pipe on cancellation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closing the Mojo pipe to the Directory object causes the ZipFileCreator service to fail and stop quickly by preventing it from listing and opening files. It still doesn't stop the ZipFileCreator service immediately if it is busy compressing a big file. This will be addressed in a subsequent CL. BUG=chromium:912236 TEST=browser_tests --gtest_filter="ZipFileCreatorTest* TEST=Manual tests Change-Id: Icd1a25ec6344d5e94a0c973c904265d76e69ba10 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2929422 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#888241} NOKEYCHECK=True GitOrigin-RevId: f91a1b3ac6cde3cd92f188d64b779d52c62b43e9 --- google/zip_writer.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/google/zip_writer.cc b/google/zip_writer.cc index b6f3365..69949cd 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -230,10 +230,8 @@ bool ZipWriter::AddFileEntries(Paths paths) { DCHECK_EQ(relative_paths.size(), n); files.clear(); - if (!file_accessor_->Open(relative_paths, &files) || files.size() != n) { - LOG(ERROR) << "Cannot open " << n << " files"; + if (!file_accessor_->Open(relative_paths, &files) || files.size() != n) return false; - } for (size_t i = 0; i < n; i++) { const base::FilePath& relative_path = relative_paths[i]; -- cgit v1.2.3 From 199485df6c234895961519df0e5a150e48026705 Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Thu, 8 Jul 2021 03:43:28 +0000 Subject: Swap base/stl_util.h to base/cxx17_backports.h in many files. Files that use base::size() and base::data() should use cxx17_backports.h directly, instead of getting it transitively through stl_util.h. Bug: 1210983 Change-Id: Icc5f425c23ef4e69283293e0d0d6d733fe3b4ba1 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2974072 Reviewed-by: Nico Weber Owners-Override: Nico Weber Commit-Queue: Lei Zhang Cr-Commit-Position: refs/heads/master@{#899418} NOKEYCHECK=True GitOrigin-RevId: d4a2f8e1ed461d8fc05ed88d1ae2dc94c9773825 --- google/compression_utils_unittest.cc | 2 +- google/zip_reader_unittest.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/google/compression_utils_unittest.cc b/google/compression_utils_unittest.cc index 31c3226..415b9ab 100644 --- a/google/compression_utils_unittest.cc +++ b/google/compression_utils_unittest.cc @@ -9,7 +9,7 @@ #include -#include "base/stl_util.h" +#include "base/cxx17_backports.h" #include "testing/gtest/include/gtest/gtest.h" namespace compression { diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index 44134f8..b203cb5 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -13,13 +13,13 @@ #include "base/bind.h" #include "base/check.h" +#include "base/cxx17_backports.h" #include "base/files/file.h" #include "base/files/file_util.h" #include "base/files/scoped_temp_dir.h" #include "base/hash/md5.h" #include "base/path_service.h" #include "base/run_loop.h" -#include "base/stl_util.h" #include "base/strings/string_piece.h" #include "base/strings/stringprintf.h" #include "base/strings/utf_string_conversions.h" -- cgit v1.2.3 From 1d84e01b0c84e9d5df43aa20c8544317b8fb2578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 12 Jul 2021 11:50:38 +0000 Subject: [zip] Store absolute timestamps in ZIP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1207751, chromium:912236 TEST=browser_tests --gtest_filter="ZipFileCreatorTest*" --gtest_also_run_disabled_tests TEST=zlib_unittests --single-process-test Change-Id: I625927a88fb0c9d26b76445ebb6b84d8eac75e80 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3020541 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#900425} NOKEYCHECK=True GitOrigin-RevId: cf856368a6d63fd9e7cdd8b71b787d003143e943 --- google/zip_internal.cc | 2 +- google/zip_reader.cc | 2 +- google/zip_reader_unittest.cc | 4 ++-- google/zip_unittest.cc | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/google/zip_internal.cc b/google/zip_internal.cc index 354fbf8..653a2ab 100644 --- a/google/zip_internal.cc +++ b/google/zip_internal.cc @@ -243,7 +243,7 @@ int GetErrorOfZipBuffer(void* /*opaque*/, void* /*stream*/) { // Returns a zip_fileinfo struct with the time represented by |file_time|. zip_fileinfo TimeToZipFileInfo(const base::Time& file_time) { base::Time::Exploded file_time_parts; - file_time.LocalExplode(&file_time_parts); + file_time.UTCExplode(&file_time_parts); zip_fileinfo zip_info = {}; if (file_time_parts.year >= 1980) { diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 1910cf2..8ddd8df 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -129,7 +129,7 @@ ZipReader::EntryInfo::EntryInfo(const std::string& file_name_in_zip, exploded_time.second = raw_file_info.tmu_date.tm_sec; exploded_time.millisecond = 0; - if (!base::Time::FromLocalExploded(exploded_time, &last_modified_)) + if (!base::Time::FromUTCExploded(exploded_time, &last_modified_)) last_modified_ = base::Time::UnixEpoch(); } diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index b203cb5..e5bb4b4 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -288,7 +288,7 @@ TEST_F(ZipReaderTest, current_entry_info_RegularFile) { // The expected time stamp: 2009-05-29 06:22:20 base::Time::Exploded exploded = {}; // Zero-clear. - current_entry_info->last_modified().LocalExplode(&exploded); + current_entry_info->last_modified().UTCExplode(&exploded); EXPECT_EQ(2009, exploded.year); EXPECT_EQ(5, exploded.month); EXPECT_EQ(29, exploded.day_of_month); @@ -357,7 +357,7 @@ TEST_F(ZipReaderTest, current_entry_info_Directory) { // The expected time stamp: 2009-05-31 15:49:52 base::Time::Exploded exploded = {}; // Zero-clear. - current_entry_info->last_modified().LocalExplode(&exploded); + current_entry_info->last_modified().UTCExplode(&exploded); EXPECT_EQ(2009, exploded.year); EXPECT_EQ(5, exploded.month); EXPECT_EQ(31, exploded.day_of_month); diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 2d2e66a..03d389e 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -297,11 +297,11 @@ class ZipTest : public PlatformTest { // supports, which is 2 seconds. Note that between this call to Time::Now() // and zip::Zip() the clock can advance a bit, hence the use of EXPECT_GE. base::Time::Exploded now_parts; - base::Time::Now().LocalExplode(&now_parts); + base::Time::Now().UTCExplode(&now_parts); now_parts.second = now_parts.second & ~1; now_parts.millisecond = 0; base::Time now_time; - EXPECT_TRUE(base::Time::FromLocalExploded(now_parts, &now_time)); + EXPECT_TRUE(base::Time::FromUTCExploded(now_parts, &now_time)); EXPECT_EQ(1, base::WriteFile(src_file, "1", 1)); EXPECT_TRUE(base::TouchFile(src_file, base::Time::Now(), test_mtime)); -- cgit v1.2.3 From f376b41126ef858c927194361ec761917f720b85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 12 Jul 2021 13:27:59 +0000 Subject: [zip] Redact filenames from error logs in ZipWriter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User-provided filenames can contain PII and thus shouldn't be found in logs. BUG=chromium:1209718, chromium:912236 TEST=Manual tests Change-Id: I87be2d4c1aa52052fe8deb1267196b902a27d276 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3017620 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#900454} NOKEYCHECK=True GitOrigin-RevId: 83d26ff85360b05f92a31bbb5408a9f8d0c670ea --- google/zip_writer.cc | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/google/zip_writer.cc b/google/zip_writer.cc index 69949cd..90b56ed 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -8,12 +8,25 @@ #include "base/files/file.h" #include "base/logging.h" +#include "base/strings/strcat.h" #include "base/strings/string_util.h" #include "third_party/zlib/google/zip_internal.h" namespace zip { namespace internal { +class Redact { + public: + explicit Redact(const base::FilePath& path) : path_(path) {} + + friend std::ostream& operator<<(std::ostream& out, const Redact&& r) { + return LOG_IS_ON(INFO) ? out << "'" << r.path_ << "'" : out << "(redacted)"; + } + + private: + const base::FilePath& path_; +}; + bool ZipWriter::ShouldContinue() { if (!progress_callback_) return true; @@ -38,7 +51,7 @@ bool ZipWriter::AddFileContent(const base::FilePath& path, base::File file) { file.ReadAtCurrentPos(buf, zip::internal::kZipBufSize); if (num_bytes < 0) { - DPLOG(ERROR) << "Cannot read file '" << path << "'"; + PLOG(ERROR) << "Cannot read file " << Redact(path); return false; } @@ -46,7 +59,8 @@ bool ZipWriter::AddFileContent(const base::FilePath& path, base::File file) { return true; if (zipWriteInFileInZip(zip_file_, buf, num_bytes) != ZIP_OK) { - DLOG(ERROR) << "Cannot write data from file '" << path << "' to ZIP"; + PLOG(ERROR) << "Cannot write data from file " << Redact(path) + << " to ZIP"; return false; } @@ -94,7 +108,7 @@ bool ZipWriter::AddDirectoryEntry(const base::FilePath& path) { return false; if (!info.is_directory) { - LOG(ERROR) << "Not a directory: '" << path << "'"; + LOG(ERROR) << "Not a directory: " << Redact(path); return false; } @@ -141,7 +155,7 @@ std::unique_ptr ZipWriter::Create( APPEND_STATUS_CREATE); if (!zip_file) { - DLOG(ERROR) << "Cannot create ZIP file '" << zip_file_path << "'"; + PLOG(ERROR) << "Cannot create ZIP file " << Redact(zip_file_path); return nullptr; } @@ -239,7 +253,7 @@ bool ZipWriter::AddFileEntries(Paths paths) { base::File& file = files[i]; if (!file.IsValid()) { - LOG(ERROR) << "Cannot open '" << relative_path << "'"; + LOG(ERROR) << "Cannot open " << Redact(relative_path); return false; } -- cgit v1.2.3 From dfbc590f5855bc2765256a743cad0abc56330a30 Mon Sep 17 00:00:00 2001 From: John Stiles Date: Tue, 13 Jul 2021 23:14:41 +0000 Subject: Retire mtklein@ from OWNERS. No longer employed at Google. Change-Id: I20266b80fdc60ab0970fa0f79553d50f9128a2c1 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3025365 Reviewed-by: Leon Scroggins Commit-Queue: John Stiles Cr-Commit-Position: refs/heads/master@{#901207} NOKEYCHECK=True GitOrigin-RevId: 42be3e9a4b9668c310c8ded36eac0236a9f77679 --- OWNERS | 1 - 1 file changed, 1 deletion(-) diff --git a/OWNERS b/OWNERS index 0bfa9fb..ecffb59 100644 --- a/OWNERS +++ b/OWNERS @@ -1,6 +1,5 @@ agl@chromium.org cavalcantii@chromium.org cblume@chromium.org -mtklein@google.com noel@chromium.org scroggo@google.com -- cgit v1.2.3 From 3f05389ec516e40df26921a58c6bcc836b46a704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Sat, 31 Jul 2021 08:40:27 +0000 Subject: [zip] Handle big (4+GB) files with ZIP64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the minizip API with the zip64 flag that allows to store big (4+GB) files in a ZIP. BUG=chromium:1207737, chromium:912236 TEST=browser_tests --gtest_filter="ZipFileCreatorTest*" --gtest_also_run_disabled_tests Change-Id: I525a47e8d7bc18e95ba726b4516330c5b881cefc Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3016489 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#907368} NOKEYCHECK=True GitOrigin-RevId: b0f5fc85e9e0f02127cc696787e4d00b3eb92aa9 --- google/zip_internal.cc | 59 +++++++++++++++++++++++++++----------------------- google/zip_internal.h | 2 +- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/google/zip_internal.cc b/google/zip_internal.cc index 653a2ab..c23e92e 100644 --- a/google/zip_internal.cc +++ b/google/zip_internal.cc @@ -38,7 +38,7 @@ typedef struct { // This function is derived from third_party/minizip/iowin32.c. // Its only difference is that it treats the char* as UTF8 and // uses the Unicode version of CreateFile. -void* ZipOpenFunc(void *opaque, const char* filename, int mode) { +void* ZipOpenFunc(void* opaque, const char* filename, int mode) { DWORD desired_access = 0, creation_disposition = 0; DWORD share_mode = 0, flags_and_attributes = 0; HANDLE file = 0; @@ -105,7 +105,7 @@ void* FdOpenFileFunc(void* opaque, const char* filename, int mode) { int FdCloseFileFunc(void* opaque, void* stream) { fclose(static_cast(stream)); - free(opaque); // malloc'ed in FillFdOpenFileFunc() + free(opaque); // malloc'ed in FillFdOpenFileFunc() return 0; } @@ -138,7 +138,7 @@ void* HandleOpenFileFunc(void* opaque, const char* filename, int mode) { } int HandleCloseFileFunc(void* opaque, void* stream) { - free(stream); // malloc'ed in HandleOpenFileFunc() + free(stream); // malloc'ed in HandleOpenFileFunc() return 0; } #endif @@ -208,8 +208,8 @@ long SeekZipBuffer(void* opaque, void* /*stream*/, uLong offset, int origin) { if (!buffer) return -1; if (origin == ZLIB_FILEFUNC_SEEK_CUR) { - buffer->offset = std::min(buffer->offset + static_cast(offset), - buffer->length); + buffer->offset = + std::min(buffer->offset + static_cast(offset), buffer->length); return 0; } if (origin == ZLIB_FILEFUNC_SEEK_END) { @@ -330,8 +330,7 @@ zipFile OpenForZipping(const std::string& file_name_utf8, int append_flag) { zip_funcs.zopen_file = ZipOpenFunc; zip_func_ptrs = &zip_funcs; #endif - return zipOpen2(file_name_utf8.c_str(), - append_flag, + return zipOpen2(file_name_utf8.c_str(), append_flag, NULL, // global comment zip_func_ptrs); } @@ -352,28 +351,34 @@ bool ZipOpenNewFileInZip(zipFile zip_file, // Setting the Language encoding flag so the file is told to be in utf-8. const uLong LANGUAGE_ENCODING_FLAG = 0x1 << 11; - zip_fileinfo file_info = TimeToZipFileInfo(last_modified_time); - if (ZIP_OK != zipOpenNewFileInZip4(zip_file, // file - str_path.c_str(), // filename - &file_info, // zip_fileinfo - NULL, // extrafield_local, - 0u, // size_extrafield_local - NULL, // extrafield_global - 0u, // size_extrafield_global - NULL, // comment - Z_DEFLATED, // method - Z_DEFAULT_COMPRESSION, // level - 0, // raw - -MAX_WBITS, // windowBits - DEF_MEM_LEVEL, // memLevel - Z_DEFAULT_STRATEGY, // strategy - NULL, // password - 0, // crcForCrypting - 0, // versionMadeBy - LANGUAGE_ENCODING_FLAG)) { // flagBase - DLOG(ERROR) << "Could not open zip file entry " << str_path; + const zip_fileinfo file_info = TimeToZipFileInfo(last_modified_time); + const int err = zipOpenNewFileInZip4_64( + /*file=*/zip_file, + /*filename=*/str_path.c_str(), + /*zip_fileinfo=*/&file_info, + /*extrafield_local=*/nullptr, + /*size_extrafield_local=*/0u, + /*extrafield_global=*/nullptr, + /*size_extrafield_global=*/0u, + /*comment=*/nullptr, + /*method=*/Z_DEFLATED, + /*level=*/Z_DEFAULT_COMPRESSION, + /*raw=*/0, + /*windowBits=*/-MAX_WBITS, + /*memLevel=*/DEF_MEM_LEVEL, + /*strategy=*/Z_DEFAULT_STRATEGY, + /*password=*/nullptr, + /*crcForCrypting=*/0, + /*versionMadeBy=*/0, + /*flagBase=*/LANGUAGE_ENCODING_FLAG, + /*zip64=*/1); + + if (err != ZIP_OK) { + DLOG(ERROR) << "Cannot open ZIP file entry '" << str_path + << "': zipOpenNewFileInZip4_64 returned " << err; return false; } + return true; } diff --git a/google/zip_internal.h b/google/zip_internal.h index 49fb902..9fb5482 100644 --- a/google/zip_internal.h +++ b/google/zip_internal.h @@ -60,7 +60,7 @@ zipFile OpenForZipping(const std::string& file_name_utf8, int append_flag); zipFile OpenFdForZipping(int zip_fd, int append_flag); #endif -// Wrapper around zipOpenNewFileInZip4 which passes most common options. +// Adds a file (or directory) entry to the ZIP archive. bool ZipOpenNewFileInZip(zipFile zip_file, const std::string& str_path, base::Time last_modified_time); -- cgit v1.2.3 From f7a5fc938f402266a46d529d5b1559a75a18ccbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Sun, 1 Aug 2021 00:20:44 +0000 Subject: [zip] Use 64-bit file functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows the zip library to read and write big ZIP files (2+GB), even on 32-bit devices. BUG=chromium:1221447, chromium:912236 TEST=zlib_unittests --single-process-tests --gtest_also_run_disabled_tests TEST=browser_tests --gtest_filter="ZipFileCreatorTest*" --gtest_also_run_disabled_tests TEST=Manual test: Produce 4+GB ZIP on kevin (ARM-32) Change-Id: Iaa8cacb0ab241cc84c00e961c75c93cb590b48bc Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3041009 Commit-Queue: François Degros Reviewed-by: Noel Gordon Cr-Commit-Position: refs/heads/master@{#907396} NOKEYCHECK=True GitOrigin-RevId: 8b199c565c7d6147249cbb4be4d59b3953507917 --- google/zip_internal.cc | 99 ++++++++++++++++++++++++++------------------------ 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/google/zip_internal.cc b/google/zip_internal.cc index c23e92e..12f9c89 100644 --- a/google/zip_internal.cc +++ b/google/zip_internal.cc @@ -36,9 +36,9 @@ typedef struct { } WIN32FILE_IOWIN; // This function is derived from third_party/minizip/iowin32.c. -// Its only difference is that it treats the char* as UTF8 and +// Its only difference is that it treats the filename as UTF-8 and // uses the Unicode version of CreateFile. -void* ZipOpenFunc(void* opaque, const char* filename, int mode) { +void* ZipOpenFunc(void* opaque, const void* filename, int mode) { DWORD desired_access = 0, creation_disposition = 0; DWORD share_mode = 0, flags_and_attributes = 0; HANDLE file = 0; @@ -56,10 +56,11 @@ void* ZipOpenFunc(void* opaque, const char* filename, int mode) { creation_disposition = CREATE_ALWAYS; } - std::wstring filenamew = base::UTF8ToWide(filename); - if ((filename != NULL) && (desired_access != 0)) { - file = CreateFile(filenamew.c_str(), desired_access, share_mode, NULL, - creation_disposition, flags_and_attributes, NULL); + if (filename != nullptr && desired_access != 0) { + file = CreateFileW( + base::UTF8ToWide(static_cast(filename)).c_str(), + desired_access, share_mode, nullptr, creation_disposition, + flags_and_attributes, nullptr); } if (file == INVALID_HANDLE_VALUE) @@ -83,7 +84,7 @@ void* ZipOpenFunc(void* opaque, const char* filename, int mode) { // Callback function for zlib that opens a file stream from a file descriptor. // Since we do not own the file descriptor, dup it so that we can fdopen/fclose // a file stream. -void* FdOpenFileFunc(void* opaque, const char* filename, int mode) { +void* FdOpenFileFunc(void* opaque, const void* filename, int mode) { FILE* file = NULL; const char* mode_fopen = NULL; @@ -111,9 +112,9 @@ int FdCloseFileFunc(void* opaque, void* stream) { // Fills |pzlib_filecunc_def| appropriately to handle the zip file // referred to by |fd|. -void FillFdOpenFileFunc(zlib_filefunc_def* pzlib_filefunc_def, int fd) { - fill_fopen_filefunc(pzlib_filefunc_def); - pzlib_filefunc_def->zopen_file = FdOpenFileFunc; +void FillFdOpenFileFunc(zlib_filefunc64_def* pzlib_filefunc_def, int fd) { + fill_fopen64_filefunc(pzlib_filefunc_def); + pzlib_filefunc_def->zopen64_file = FdOpenFileFunc; pzlib_filefunc_def->zclose_file = FdCloseFileFunc; int* ptr_fd = static_cast(malloc(sizeof(fd))); *ptr_fd = fd; @@ -124,7 +125,7 @@ void FillFdOpenFileFunc(zlib_filefunc_def* pzlib_filefunc_def, int fd) { #if defined(OS_WIN) // Callback function for zlib that opens a file stream from a Windows handle. // Does not take ownership of the handle. -void* HandleOpenFileFunc(void* opaque, const char* filename, int mode) { +void* HandleOpenFileFunc(void* opaque, const void* /*filename*/, int mode) { WIN32FILE_IOWIN file_ret; file_ret.hf = static_cast(opaque); file_ret.error = 0; @@ -148,8 +149,8 @@ int HandleCloseFileFunc(void* opaque, void* stream) { // expect their opaque parameters refer to this struct. struct ZipBuffer { const char* data; // weak - size_t length; - size_t offset; + ZPOS64_T length; + ZPOS64_T offset; }; // Opens the specified file. When this function returns a non-NULL pointer, zlib @@ -158,7 +159,7 @@ struct ZipBuffer { // given opaque parameter and returns it because this parameter stores all // information needed for uncompressing data. (This function does not support // writing compressed data and it returns NULL for this case.) -void* OpenZipBuffer(void* opaque, const char* /*filename*/, int mode) { +void* OpenZipBuffer(void* opaque, const void* /*filename*/, int mode) { if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER) != ZLIB_FILEFUNC_MODE_READ) { NOTREACHED(); return NULL; @@ -175,10 +176,11 @@ void* OpenZipBuffer(void* opaque, const char* /*filename*/, int mode) { uLong ReadZipBuffer(void* opaque, void* /*stream*/, void* buf, uLong size) { ZipBuffer* buffer = static_cast(opaque); DCHECK_LE(buffer->offset, buffer->length); - size_t remaining_bytes = buffer->length - buffer->offset; + ZPOS64_T remaining_bytes = buffer->length - buffer->offset; if (!buffer || !buffer->data || !remaining_bytes) return 0; - size = std::min(size, static_cast(remaining_bytes)); + if (size > remaining_bytes) + size = remaining_bytes; memcpy(buf, &buffer->data[buffer->offset], size); buffer->offset += size; return size; @@ -195,21 +197,23 @@ uLong WriteZipBuffer(void* /*opaque*/, } // Returns the offset from the beginning of the data. -long GetOffsetOfZipBuffer(void* opaque, void* /*stream*/) { +ZPOS64_T GetOffsetOfZipBuffer(void* opaque, void* /*stream*/) { ZipBuffer* buffer = static_cast(opaque); if (!buffer) return -1; - return static_cast(buffer->offset); + return buffer->offset; } // Moves the current offset to the specified position. -long SeekZipBuffer(void* opaque, void* /*stream*/, uLong offset, int origin) { +long SeekZipBuffer(void* opaque, + void* /*stream*/, + ZPOS64_T offset, + int origin) { ZipBuffer* buffer = static_cast(opaque); if (!buffer) return -1; if (origin == ZLIB_FILEFUNC_SEEK_CUR) { - buffer->offset = - std::min(buffer->offset + static_cast(offset), buffer->length); + buffer->offset = std::min(buffer->offset + offset, buffer->length); return 0; } if (origin == ZLIB_FILEFUNC_SEEK_END) { @@ -217,7 +221,7 @@ long SeekZipBuffer(void* opaque, void* /*stream*/, uLong offset, int origin) { return 0; } if (origin == ZLIB_FILEFUNC_SEEK_SET) { - buffer->offset = std::min(buffer->length, static_cast(offset)); + buffer->offset = std::min(buffer->length, offset); return 0; } NOTREACHED(); @@ -268,33 +272,33 @@ namespace zip { namespace internal { unzFile OpenForUnzipping(const std::string& file_name_utf8) { - zlib_filefunc_def* zip_func_ptrs = NULL; + zlib_filefunc64_def* zip_func_ptrs = nullptr; #if defined(OS_WIN) - zlib_filefunc_def zip_funcs; - fill_win32_filefunc(&zip_funcs); - zip_funcs.zopen_file = ZipOpenFunc; + zlib_filefunc64_def zip_funcs; + fill_win32_filefunc64(&zip_funcs); + zip_funcs.zopen64_file = ZipOpenFunc; zip_func_ptrs = &zip_funcs; #endif - return unzOpen2(file_name_utf8.c_str(), zip_func_ptrs); + return unzOpen2_64(file_name_utf8.c_str(), zip_func_ptrs); } #if defined(OS_POSIX) unzFile OpenFdForUnzipping(int zip_fd) { - zlib_filefunc_def zip_funcs; + zlib_filefunc64_def zip_funcs; FillFdOpenFileFunc(&zip_funcs, zip_fd); // Passing dummy "fd" filename to zlib. - return unzOpen2("fd", &zip_funcs); + return unzOpen2_64("fd", &zip_funcs); } #endif #if defined(OS_WIN) unzFile OpenHandleForUnzipping(HANDLE zip_handle) { - zlib_filefunc_def zip_funcs; - fill_win32_filefunc(&zip_funcs); - zip_funcs.zopen_file = HandleOpenFileFunc; + zlib_filefunc64_def zip_funcs; + fill_win32_filefunc64(&zip_funcs); + zip_funcs.zopen64_file = HandleOpenFileFunc; zip_funcs.zclose_file = HandleCloseFileFunc; zip_funcs.opaque = zip_handle; - return unzOpen2("fd", &zip_funcs); + return unzOpen2_64("fd", &zip_funcs); } #endif @@ -310,37 +314,36 @@ unzFile PrepareMemoryForUnzipping(const std::string& data) { buffer->length = data.length(); buffer->offset = 0; - zlib_filefunc_def zip_functions; - zip_functions.zopen_file = OpenZipBuffer; + zlib_filefunc64_def zip_functions; + zip_functions.zopen64_file = OpenZipBuffer; zip_functions.zread_file = ReadZipBuffer; zip_functions.zwrite_file = WriteZipBuffer; - zip_functions.ztell_file = GetOffsetOfZipBuffer; - zip_functions.zseek_file = SeekZipBuffer; + zip_functions.ztell64_file = GetOffsetOfZipBuffer; + zip_functions.zseek64_file = SeekZipBuffer; zip_functions.zclose_file = CloseZipBuffer; zip_functions.zerror_file = GetErrorOfZipBuffer; - zip_functions.opaque = static_cast(buffer); - return unzOpen2(NULL, &zip_functions); + zip_functions.opaque = buffer; + return unzOpen2_64(nullptr, &zip_functions); } zipFile OpenForZipping(const std::string& file_name_utf8, int append_flag) { - zlib_filefunc_def* zip_func_ptrs = NULL; + zlib_filefunc64_def* zip_func_ptrs = nullptr; #if defined(OS_WIN) - zlib_filefunc_def zip_funcs; - fill_win32_filefunc(&zip_funcs); - zip_funcs.zopen_file = ZipOpenFunc; + zlib_filefunc64_def zip_funcs; + fill_win32_filefunc64(&zip_funcs); + zip_funcs.zopen64_file = ZipOpenFunc; zip_func_ptrs = &zip_funcs; #endif - return zipOpen2(file_name_utf8.c_str(), append_flag, - NULL, // global comment - zip_func_ptrs); + return zipOpen2_64(file_name_utf8.c_str(), append_flag, nullptr, + zip_func_ptrs); } #if defined(OS_POSIX) zipFile OpenFdForZipping(int zip_fd, int append_flag) { - zlib_filefunc_def zip_funcs; + zlib_filefunc64_def zip_funcs; FillFdOpenFileFunc(&zip_funcs, zip_fd); // Passing dummy "fd" filename to zlib. - return zipOpen2("fd", append_flag, NULL, &zip_funcs); + return zipOpen2_64("fd", append_flag, nullptr, &zip_funcs); } #endif -- cgit v1.2.3 From 3190a5de448a31514dba42423747f4499785faa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 2 Aug 2021 04:51:00 +0000 Subject: [zip] Simply store already compressed files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use a simple heuristic to determine whether files should be compressed or not. BUG=chromium:1197414, chromium:912236 TEST=zlib_unittests --single-process-tests TEST=browser_tests --gtest_filter="ZipFileCreator*" --gtest_also_run_disabled_tests Change-Id: I37fcf613303eb4372f0fceac84870f7c43432b75 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3058718 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#907451} NOKEYCHECK=True GitOrigin-RevId: 96758475868f886ff6f9ed40b40ba72e1fa0dfc3 --- google/zip_internal.cc | 84 +++++++++++++++++++++++++++++++++++++++++++++-- google/zip_internal.h | 16 ++++++++- google/zip_unittest.cc | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++ google/zip_writer.cc | 12 +++++-- 4 files changed, 195 insertions(+), 5 deletions(-) diff --git a/google/zip_internal.cc b/google/zip_internal.cc index 12f9c89..cea1e88 100644 --- a/google/zip_internal.cc +++ b/google/zip_internal.cc @@ -8,9 +8,13 @@ #include #include +#include +#include "base/files/file_path.h" #include "base/logging.h" +#include "base/no_destructor.h" #include "base/notreached.h" +#include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" #if defined(USE_SYSTEM_MINIZIP) @@ -349,7 +353,8 @@ zipFile OpenFdForZipping(int zip_fd, int append_flag) { bool ZipOpenNewFileInZip(zipFile zip_file, const std::string& str_path, - base::Time last_modified_time) { + base::Time last_modified_time, + Compression compression) { // Section 4.4.4 http://www.pkware.com/documents/casestudies/APPNOTE.TXT // Setting the Language encoding flag so the file is told to be in utf-8. const uLong LANGUAGE_ENCODING_FLAG = 0x1 << 11; @@ -364,7 +369,7 @@ bool ZipOpenNewFileInZip(zipFile zip_file, /*extrafield_global=*/nullptr, /*size_extrafield_global=*/0u, /*comment=*/nullptr, - /*method=*/Z_DEFLATED, + /*method=*/compression, /*level=*/Z_DEFAULT_COMPRESSION, /*raw=*/0, /*windowBits=*/-MAX_WBITS, @@ -385,5 +390,80 @@ bool ZipOpenNewFileInZip(zipFile zip_file, return true; } +Compression GetCompressionMethod(const base::FilePath& path) { + // Get the filename extension in lower case. + const base::FilePath::StringType ext = + base::ToLowerASCII(path.FinalExtension()); + + if (ext.empty()) + return kDeflated; + + using StringPiece = base::FilePath::StringPieceType; + + // Skip the leading dot. + StringPiece ext_without_dot = ext; + DCHECK_EQ(ext_without_dot.front(), FILE_PATH_LITERAL('.')); + ext_without_dot.remove_prefix(1); + + // Well known filename extensions of files that a likely to be already + // compressed. The extensions are in lower case without the leading dot. + static const base::NoDestructor< + std::unordered_set>> + exts(std::initializer_list{ + FILE_PATH_LITERAL("3g2"), // + FILE_PATH_LITERAL("3gp"), // + FILE_PATH_LITERAL("7z"), // + FILE_PATH_LITERAL("7zip"), // + FILE_PATH_LITERAL("aac"), // + FILE_PATH_LITERAL("avi"), // + FILE_PATH_LITERAL("bz"), // + FILE_PATH_LITERAL("bz2"), // + FILE_PATH_LITERAL("crx"), // + FILE_PATH_LITERAL("gif"), // + FILE_PATH_LITERAL("gz"), // + FILE_PATH_LITERAL("jar"), // + FILE_PATH_LITERAL("jpeg"), // + FILE_PATH_LITERAL("jpg"), // + FILE_PATH_LITERAL("lz"), // + FILE_PATH_LITERAL("m2v"), // + FILE_PATH_LITERAL("m4p"), // + FILE_PATH_LITERAL("m4v"), // + FILE_PATH_LITERAL("mng"), // + FILE_PATH_LITERAL("mov"), // + FILE_PATH_LITERAL("mp2"), // + FILE_PATH_LITERAL("mp3"), // + FILE_PATH_LITERAL("mp4"), // + FILE_PATH_LITERAL("mpe"), // + FILE_PATH_LITERAL("mpeg"), // + FILE_PATH_LITERAL("mpg"), // + FILE_PATH_LITERAL("mpv"), // + FILE_PATH_LITERAL("ogg"), // + FILE_PATH_LITERAL("ogv"), // + FILE_PATH_LITERAL("png"), // + FILE_PATH_LITERAL("qt"), // + FILE_PATH_LITERAL("rar"), // + FILE_PATH_LITERAL("taz"), // + FILE_PATH_LITERAL("tb2"), // + FILE_PATH_LITERAL("tbz"), // + FILE_PATH_LITERAL("tbz2"), // + FILE_PATH_LITERAL("tgz"), // + FILE_PATH_LITERAL("tlz"), // + FILE_PATH_LITERAL("tz"), // + FILE_PATH_LITERAL("tz2"), // + FILE_PATH_LITERAL("vob"), // + FILE_PATH_LITERAL("webm"), // + FILE_PATH_LITERAL("wma"), // + FILE_PATH_LITERAL("wmv"), // + FILE_PATH_LITERAL("xz"), // + FILE_PATH_LITERAL("z"), // + FILE_PATH_LITERAL("zip"), // + }); + + if (exts->count(ext_without_dot)) + return kStored; + + return kDeflated; +} + } // namespace internal } // namespace zip diff --git a/google/zip_internal.h b/google/zip_internal.h index 9fb5482..ef5b5d0 100644 --- a/google/zip_internal.h +++ b/google/zip_internal.h @@ -60,10 +60,24 @@ zipFile OpenForZipping(const std::string& file_name_utf8, int append_flag); zipFile OpenFdForZipping(int zip_fd, int append_flag); #endif +// Compression methods. +enum Compression { + kStored = 0, // Stored (no compression) + kDeflated = Z_DEFLATED, // Deflated +}; + // Adds a file (or directory) entry to the ZIP archive. bool ZipOpenNewFileInZip(zipFile zip_file, const std::string& str_path, - base::Time last_modified_time); + base::Time last_modified_time, + Compression compression); + +// Selects the best compression method for the given file. The heuristic is +// based on the filename extension. By default, the compression method is +// kDeflated. But if the given path has an extension indicating a well known +// file format which is likely to be already compressed (eg ZIP, RAR, JPG, +// PNG...) then the compression method is simply kStored. +Compression GetCompressionMethod(const base::FilePath& path); const int kZipMaxPath = 256; const int kZipBufSize = 8192; diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 03d389e..aa6b80c 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -26,8 +26,12 @@ #include "testing/gtest/include/gtest/gtest.h" #include "testing/platform_test.h" #include "third_party/zlib/google/zip.h" +#include "third_party/zlib/google/zip_internal.h" #include "third_party/zlib/google/zip_reader.h" +// Convenience macro to create a file path from a string literal. +#define FP(path) base::FilePath(FILE_PATH_LITERAL(path)) + namespace { bool CreateFile(const std::string& content, @@ -710,4 +714,88 @@ TEST_F(ZipTest, ZipCancel) { } } +// Tests zip::internal::GetCompressionMethod() +TEST_F(ZipTest, GetCompressionMethod) { + using zip::internal::GetCompressionMethod; + using zip::internal::kDeflated; + using zip::internal::kStored; + + EXPECT_EQ(GetCompressionMethod(FP("")), kDeflated); + EXPECT_EQ(GetCompressionMethod(FP("NoExtension")), kDeflated); + EXPECT_EQ(GetCompressionMethod(FP("Folder.zip").Append(FP("NoExtension"))), + kDeflated); + EXPECT_EQ(GetCompressionMethod(FP("Name.txt")), kDeflated); + EXPECT_EQ(GetCompressionMethod(FP("Name.zip")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("Name....zip")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("Name.zip")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("NAME.ZIP")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("Name.gz")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("Name.tar.gz")), kStored); + EXPECT_EQ(GetCompressionMethod(FP("Name.tar")), kDeflated); + + // This one is controversial. + EXPECT_EQ(GetCompressionMethod(FP(".zip")), kStored); +} + +// Tests that files put inside a ZIP are effectively compressed. +TEST_F(ZipTest, Compressed) { + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + + const base::FilePath src_dir = temp_dir.GetPath().AppendASCII("input"); + EXPECT_TRUE(base::CreateDirectory(src_dir)); + + // Create some dummy source files. + for (const base::StringPiece s : {"foo", "bar.txt", ".hidden"}) { + base::File f(src_dir.AppendASCII(s), + base::File::FLAG_CREATE | base::File::FLAG_WRITE); + ASSERT_TRUE(f.SetLength(5000)); + } + + // Zip the source files. + const base::FilePath dest_file = temp_dir.GetPath().AppendASCII("dest.zip"); + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, + .dest_file = dest_file, + .include_hidden_files = true})); + + // Since the source files compress well, the destination ZIP file should be + // smaller than the source files. + int64_t dest_file_size; + ASSERT_TRUE(base::GetFileSize(dest_file, &dest_file_size)); + EXPECT_GT(dest_file_size, 300); + EXPECT_LT(dest_file_size, 1000); +} + +// Tests that a ZIP put inside a ZIP is simply stored instead of being +// compressed. +TEST_F(ZipTest, NestedZip) { + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + + const base::FilePath src_dir = temp_dir.GetPath().AppendASCII("input"); + EXPECT_TRUE(base::CreateDirectory(src_dir)); + + // Create a dummy ZIP file. This is not a valid ZIP file, but for the purpose + // of this test, it doesn't really matter. + const int64_t src_size = 5000; + + { + base::File f(src_dir.AppendASCII("src.zip"), + base::File::FLAG_CREATE | base::File::FLAG_WRITE); + ASSERT_TRUE(f.SetLength(src_size)); + } + + // Zip the dummy ZIP file. + const base::FilePath dest_file = temp_dir.GetPath().AppendASCII("dest.zip"); + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, .dest_file = dest_file})); + + // Since the dummy source (inner) ZIP file should simply be stored in the + // destination (outer) ZIP file, the destination file should be bigger than + // the source file, but not much bigger. + int64_t dest_file_size; + ASSERT_TRUE(base::GetFileSize(dest_file, &dest_file_size)); + EXPECT_GT(dest_file_size, src_size + 100); + EXPECT_LT(dest_file_size, src_size + 300); +} + } // namespace diff --git a/google/zip_writer.cc b/google/zip_writer.cc index 90b56ed..3e2345a 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -74,13 +74,21 @@ bool ZipWriter::OpenNewFileEntry(const base::FilePath& path, bool is_directory, base::Time last_modified) { std::string str_path = path.AsUTF8Unsafe(); + #if defined(OS_WIN) base::ReplaceSubstringsAfterOffset(&str_path, 0u, "\\", "/"); #endif - if (is_directory) + + Compression compression = kDeflated; + + if (is_directory) { str_path += "/"; + } else { + compression = GetCompressionMethod(path); + } - return zip::internal::ZipOpenNewFileInZip(zip_file_, str_path, last_modified); + return zip::internal::ZipOpenNewFileInZip(zip_file_, str_path, last_modified, + compression); } bool ZipWriter::CloseNewFileEntry() { -- cgit v1.2.3 From 6075f1fabdd8125536ccc05ee3851078a9db4f7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 2 Aug 2021 06:22:23 +0000 Subject: [zip] Unit test for big files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test is disabled by default because it times out on some trybots. It can be run on a workstation by using the --gtest_also_run_disabled_tests flag. BUG=chromium:1207737, chromium:1221447, chromium:912236 TEST=zlib_unittests --gtest_also_run_disabled_tests Change-Id: I0732578d9d00a7d3073b313d13c5cf0d66925184 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3065594 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#907460} NOKEYCHECK=True GitOrigin-RevId: d540c627e6dacac36b7ff0ec2701b7e3acd63514 --- google/zip_unittest.cc | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index aa6b80c..1a6a843 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -798,4 +798,37 @@ TEST_F(ZipTest, NestedZip) { EXPECT_LT(dest_file_size, src_size + 300); } +// Tests that there is no 2GB or 4GB limits. Tests that big files can be zipped +// (crbug.com/1207737) and that big ZIP files can be created +// (crbug.com/1221447). +TEST_F(ZipTest, DISABLED_BigFile) { + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + + const base::FilePath src_dir = temp_dir.GetPath().AppendASCII("input"); + EXPECT_TRUE(base::CreateDirectory(src_dir)); + + // Create a big dummy ZIP file. This is not a valid ZIP file, but for the + // purpose of this test, it doesn't really matter. + const int64_t src_size = 5'000'000'000; + + { + base::File f(src_dir.AppendASCII("src.zip"), + base::File::FLAG_CREATE | base::File::FLAG_WRITE); + ASSERT_TRUE(f.SetLength(src_size)); + } + + // Zip the dummy ZIP file. + const base::FilePath dest_file = temp_dir.GetPath().AppendASCII("dest.zip"); + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, .dest_file = dest_file})); + + // Since the dummy source (inner) ZIP file should simply be stored in the + // destination (outer) ZIP file, the destination file should be bigger than + // the source file, but not much bigger. + int64_t dest_file_size; + ASSERT_TRUE(base::GetFileSize(dest_file, &dest_file_size)); + EXPECT_GT(dest_file_size, src_size + 100); + EXPECT_LT(dest_file_size, src_size + 300); +} + } // namespace -- cgit v1.2.3 From 563140dd9c24f84bf40919196e9e7666d351cc0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 2 Aug 2021 09:44:07 +0000 Subject: [zip] Improve unit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ensure that unit tests actually compare file contents. BUG=chromium:774156, chromium:912236 TEST=zlib_unittests --single-process-tests --gtest_also_run_disabled_tests Change-Id: I31622a8e44b15ffed797027791d794ab1e1de4d4 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3060161 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/master@{#907486} NOKEYCHECK=True GitOrigin-RevId: 6ea10bb43162ce2455cfa9f47b8ea7aec6f1d2b1 --- google/zip_unittest.cc | 75 +++++++++++++++++++------------------------------- 1 file changed, 29 insertions(+), 46 deletions(-) diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 1a6a843..876f3eb 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -62,7 +62,7 @@ class VirtualFileSystem : public zip::FileAccessor { VirtualFileSystem() { base::FilePath test_dir; - base::FilePath foo_txt_path = test_dir.Append(FILE_PATH_LITERAL("foo.txt")); + base::FilePath foo_txt_path = test_dir.AppendASCII("foo.txt"); base::FilePath file_path; base::File file; @@ -70,15 +70,13 @@ class VirtualFileSystem : public zip::FileAccessor { DCHECK(success); files_[foo_txt_path] = std::move(file); - base::FilePath bar_dir = test_dir.Append(FILE_PATH_LITERAL("bar")); - base::FilePath bar1_txt_path = - bar_dir.Append(FILE_PATH_LITERAL("bar1.txt")); + base::FilePath bar_dir = test_dir.AppendASCII("bar"); + base::FilePath bar1_txt_path = bar_dir.AppendASCII("bar1.txt"); success = CreateFile(kBar1Content, &file_path, &file); DCHECK(success); files_[bar1_txt_path] = std::move(file); - base::FilePath bar2_txt_path = - bar_dir.Append(FILE_PATH_LITERAL("bar2.txt")); + base::FilePath bar2_txt_path = bar_dir.AppendASCII("bar2.txt"); success = CreateFile(kBar2Content, &file_path, &file); DCHECK(success); files_[bar2_txt_path] = std::move(file); @@ -176,22 +174,20 @@ class ZipTest : public PlatformTest { test_dir_ = temp_dir_.GetPath(); base::FilePath zip_path(test_dir_); - zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("foo.txt"))); - zip_path = zip_path.Append(FILE_PATH_LITERAL("foo")); + zip_contents_.insert(zip_path.AppendASCII("foo.txt")); + zip_path = zip_path.AppendASCII("foo"); zip_contents_.insert(zip_path); - zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("bar.txt"))); - zip_path = zip_path.Append(FILE_PATH_LITERAL("bar")); + zip_contents_.insert(zip_path.AppendASCII("bar.txt")); + zip_path = zip_path.AppendASCII("bar"); zip_contents_.insert(zip_path); - zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("baz.txt"))); - zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL("quux.txt"))); - zip_contents_.insert(zip_path.Append(FILE_PATH_LITERAL(".hidden"))); + zip_contents_.insert(zip_path.AppendASCII("baz.txt")); + zip_contents_.insert(zip_path.AppendASCII("quux.txt")); + zip_contents_.insert(zip_path.AppendASCII(".hidden")); // Include a subset of files in |zip_file_list_| to test ZipFiles(). - zip_file_list_.push_back(base::FilePath(FILE_PATH_LITERAL("foo.txt"))); - zip_file_list_.push_back( - base::FilePath(FILE_PATH_LITERAL("foo/bar/quux.txt"))); - zip_file_list_.push_back( - base::FilePath(FILE_PATH_LITERAL("foo/bar/.hidden"))); + zip_file_list_.push_back(FP("foo.txt")); + zip_file_list_.push_back(FP("foo/bar/quux.txt")); + zip_file_list_.push_back(FP("foo/bar/.hidden")); } virtual void TearDown() { PlatformTest::TearDown(); } @@ -228,7 +224,7 @@ class ZipTest : public PlatformTest { base::FileEnumerator::FILES | base::FileEnumerator::DIRECTORIES); base::FilePath unzipped_entry_path = files.Next(); size_t count = 0; - while (!unzipped_entry_path.value().empty()) { + while (!unzipped_entry_path.empty()) { EXPECT_EQ(zip_contents_.count(unzipped_entry_path), 1U) << "Couldn't find " << unzipped_entry_path.value(); count++; @@ -236,28 +232,15 @@ class ZipTest : public PlatformTest { if (base::PathExists(unzipped_entry_path) && !base::DirectoryExists(unzipped_entry_path)) { // It's a file, check its contents are what we zipped. - // TODO(774156): figure out why the commented out EXPECT_TRUE below - // fails on the build bots (but not on the try-bots). base::FilePath relative_path; - bool append_relative_path_success = - test_dir_.AppendRelativePath(unzipped_entry_path, &relative_path); - if (!append_relative_path_success) { - LOG(ERROR) << "Append relative path failed, params: " << test_dir_ - << " and " << unzipped_entry_path; - } + ASSERT_TRUE( + test_dir_.AppendRelativePath(unzipped_entry_path, &relative_path)) + << "Cannot append relative path failed, params: '" << test_dir_ + << "' and '" << unzipped_entry_path << "'"; base::FilePath original_path = original_dir.Append(relative_path); - const bool equal = - base::ContentsEqual(original_path, unzipped_entry_path); - if (equal) { - LOG(INFO) << "Original and unzipped file '" << relative_path - << "' are equal"; - } else { - LOG(ERROR) << "Original and unzipped file '" << relative_path - << "' are different"; - } - // EXPECT_TRUE(base::ContentsEqual(original_path, unzipped_entry_path)) - // << "Contents differ between original " << original_path.value() - // << " and unzipped file " << unzipped_entry_path.value(); + EXPECT_TRUE(base::ContentsEqual(original_path, unzipped_entry_path)) + << "Original file '" << original_path << "' and unzipped file '" + << unzipped_entry_path << "' have different contents"; } unzipped_entry_path = files.Next(); } @@ -579,17 +562,17 @@ TEST_F(ZipTest, ZipWithFileAccessor) { ASSERT_TRUE(scoped_temp_dir.CreateUniqueTempDir()); const base::FilePath& temp_dir = scoped_temp_dir.GetPath(); ASSERT_TRUE(zip::Unzip(zip_file, temp_dir)); - base::FilePath bar_dir = temp_dir.Append(FILE_PATH_LITERAL("bar")); + base::FilePath bar_dir = temp_dir.AppendASCII("bar"); EXPECT_TRUE(base::DirectoryExists(bar_dir)); std::string file_content; - EXPECT_TRUE(base::ReadFileToString( - temp_dir.Append(FILE_PATH_LITERAL("foo.txt")), &file_content)); + EXPECT_TRUE( + base::ReadFileToString(temp_dir.AppendASCII("foo.txt"), &file_content)); EXPECT_EQ(VirtualFileSystem::kFooContent, file_content); - EXPECT_TRUE(base::ReadFileToString( - bar_dir.Append(FILE_PATH_LITERAL("bar1.txt")), &file_content)); + EXPECT_TRUE( + base::ReadFileToString(bar_dir.AppendASCII("bar1.txt"), &file_content)); EXPECT_EQ(VirtualFileSystem::kBar1Content, file_content); - EXPECT_TRUE(base::ReadFileToString( - bar_dir.Append(FILE_PATH_LITERAL("bar2.txt")), &file_content)); + EXPECT_TRUE( + base::ReadFileToString(bar_dir.AppendASCII("bar2.txt"), &file_content)); EXPECT_EQ(VirtualFileSystem::kBar2Content, file_content); } -- cgit v1.2.3 From 77c132322fe81a1f5518b326e18c99ebd3281627 Mon Sep 17 00:00:00 2001 From: Fabrice de Gans Date: Thu, 26 Aug 2021 16:30:40 +0000 Subject: [zlib] Opt-in to Posix code paths on Fuchsia OS_POSIX is not set on Fuchsia. Here, we can explicitly opt-in to the Posix code on Fuchsia. Bug: 1241675 Change-Id: Icfcf85bb6c1faeae98a66e8188169114a8c88092 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3119663 Auto-Submit: Fabrice de Gans Reviewed-by: Adenilson Cavalcanti Reviewed-by: Wez Commit-Queue: Fabrice de Gans Cr-Commit-Position: refs/heads/main@{#915601} NOKEYCHECK=True GitOrigin-RevId: 7256f7c972c8a126b15ebdd244de79511fda5d8b --- google/zip_internal.cc | 4 ++-- google/zip_internal.h | 2 +- google/zip_reader.cc | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/google/zip_internal.cc b/google/zip_internal.cc index cea1e88..00e9eef 100644 --- a/google/zip_internal.cc +++ b/google/zip_internal.cc @@ -84,7 +84,7 @@ void* ZipOpenFunc(void* opaque, const void* filename, int mode) { } #endif -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) // Callback function for zlib that opens a file stream from a file descriptor. // Since we do not own the file descriptor, dup it so that we can fdopen/fclose // a file stream. @@ -286,7 +286,7 @@ unzFile OpenForUnzipping(const std::string& file_name_utf8) { return unzOpen2_64(file_name_utf8.c_str(), zip_func_ptrs); } -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) unzFile OpenFdForUnzipping(int zip_fd) { zlib_filefunc64_def zip_funcs; FillFdOpenFileFunc(&zip_funcs, zip_fd); diff --git a/google/zip_internal.h b/google/zip_internal.h index ef5b5d0..c7feba6 100644 --- a/google/zip_internal.h +++ b/google/zip_internal.h @@ -35,7 +35,7 @@ namespace internal { // Windows. unzFile OpenForUnzipping(const std::string& file_name_utf8); -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) // Opens the file referred to by |zip_fd| for unzipping. unzFile OpenFdForUnzipping(int zip_fd); #endif diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 8ddd8df..2ad1398 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -157,7 +157,7 @@ bool ZipReader::Open(const base::FilePath& zip_file_path) { bool ZipReader::OpenFromPlatformFile(base::PlatformFile zip_fd) { DCHECK(!zip_file_); -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) zip_file_ = internal::OpenFdForUnzipping(zip_fd); #elif defined(OS_WIN) zip_file_ = internal::OpenHandleForUnzipping(zip_fd); -- cgit v1.2.3 From dfa96e81458fb3b39676e45f7e9e000dff789b05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Bostr=C3=B6m?= Date: Fri, 24 Sep 2021 18:07:21 +0000 Subject: Replace DISALLOW_COPY_AND_ASSIGN in third_party/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This replaces DISALLOW_COPY_AND_ASSIGN with explicit constructor deletes where a local script is able to detect its insertion place (~Foo() is public => insert before this line). This is incomplete as not all classes have a public ~Foo() declared, so not all DISALLOW_COPY_AND_ASSIGN occurrences are replaced. IWYU cleanup is left as a separate pass that is easier when these macros go away. Bug: 1010217 Change-Id: I1daa712b586d5a73c90891e3f9c7bd8245e927a4 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3182423 Auto-Submit: Peter Boström Commit-Queue: Lei Zhang Reviewed-by: Lei Zhang Owners-Override: Lei Zhang Cr-Commit-Position: refs/heads/main@{#924827} NOKEYCHECK=True GitOrigin-RevId: 80c123a1c777b6009baff72e0364bb6137b7caa4 --- google/zip_reader.cc | 6 ++++-- google/zip_reader.h | 11 +++++++---- google/zip_unittest.cc | 5 +++-- google/zip_writer.h | 5 +++-- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 2ad1398..0c6a93a 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -36,6 +36,10 @@ namespace { class StringWriterDelegate : public WriterDelegate { public: StringWriterDelegate(size_t max_read_bytes, std::string* output); + + StringWriterDelegate(const StringWriterDelegate&) = delete; + StringWriterDelegate& operator=(const StringWriterDelegate&) = delete; + ~StringWriterDelegate() override; // WriterDelegate methods: @@ -52,8 +56,6 @@ class StringWriterDelegate : public WriterDelegate { private: size_t max_read_bytes_; std::string* output_; - - DISALLOW_COPY_AND_ASSIGN(StringWriterDelegate); }; StringWriterDelegate::StringWriterDelegate(size_t max_read_bytes, diff --git a/google/zip_reader.h b/google/zip_reader.h index d442d42..9374beb 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -240,6 +240,9 @@ class FileWriterDelegate : public WriterDelegate { // Constructs a FileWriterDelegate that takes ownership of |file|. explicit FileWriterDelegate(std::unique_ptr file); + FileWriterDelegate(const FileWriterDelegate&) = delete; + FileWriterDelegate& operator=(const FileWriterDelegate&) = delete; + // Truncates the file to the number of bytes written. ~FileWriterDelegate() override; @@ -267,14 +270,16 @@ class FileWriterDelegate : public WriterDelegate { std::unique_ptr owned_file_; int64_t file_length_ = 0; - - DISALLOW_COPY_AND_ASSIGN(FileWriterDelegate); }; // A writer delegate that writes a file at a given path. class FilePathWriterDelegate : public WriterDelegate { public: explicit FilePathWriterDelegate(const base::FilePath& output_file_path); + + FilePathWriterDelegate(const FilePathWriterDelegate&) = delete; + FilePathWriterDelegate& operator=(const FilePathWriterDelegate&) = delete; + ~FilePathWriterDelegate() override; // WriterDelegate methods: @@ -292,8 +297,6 @@ class FilePathWriterDelegate : public WriterDelegate { private: base::FilePath output_file_path_; base::File file_; - - DISALLOW_COPY_AND_ASSIGN(FilePathWriterDelegate); }; } // namespace zip diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 876f3eb..daaad21 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -88,6 +88,9 @@ class VirtualFileSystem : public zip::FileAccessor { file_tree_[bar2_txt_path] = {}; } + VirtualFileSystem(const VirtualFileSystem&) = delete; + VirtualFileSystem& operator=(const VirtualFileSystem&) = delete; + ~VirtualFileSystem() override = default; private: @@ -153,8 +156,6 @@ class VirtualFileSystem : public zip::FileAccessor { std::map file_tree_; std::map files_; - - DISALLOW_COPY_AND_ASSIGN(VirtualFileSystem); }; // static diff --git a/google/zip_writer.h b/google/zip_writer.h index c67903d..c58b1b1 100644 --- a/google/zip_writer.h +++ b/google/zip_writer.h @@ -44,6 +44,9 @@ class ZipWriter { static std::unique_ptr Create(const base::FilePath& zip_file, FileAccessor* file_accessor); + ZipWriter(const ZipWriter&) = delete; + ZipWriter& operator=(const ZipWriter&) = delete; + ~ZipWriter(); // Sets the optional progress callback. The callback is called once for each @@ -135,8 +138,6 @@ class ZipWriter { // Should recursively add directories? bool recursive_ = false; - - DISALLOW_COPY_AND_ASSIGN(ZipWriter); }; } // namespace internal -- cgit v1.2.3 From 53c0edcf0264bba68bf85f5071590d7d453cdbdc Mon Sep 17 00:00:00 2001 From: Peter Kasting Date: Sat, 2 Oct 2021 03:06:35 +0000 Subject: Migrate "base::TimeDelta::FromX" to "base:X". MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All changes were done automatically with git grep, sed, xargs, etc. No-Presubmit: true No-Try: true Bug: 1243777 Change-Id: I7cc197e9027f7837cd36afc67a209079f85ec364 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3198824 Commit-Queue: Peter Kasting Owners-Override: Peter Kasting Reviewed-by: Peter Boström Cr-Commit-Position: refs/heads/main@{#927512} NOKEYCHECK=True GitOrigin-RevId: e5a38eddbdf45d7563a00d019debd11b803af1bb --- google/zip_unittest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index daaad21..d10ec05 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -646,7 +646,7 @@ TEST_F(ZipTest, ZipProgressPeriod) { EXPECT_TRUE(zip::Zip({.src_dir = src_dir, .dest_file = zip_file, .progress_callback = std::move(progress_callback), - .progress_period = base::TimeDelta::FromHours(1)})); + .progress_period = base::Hours(1)})); // We expect only 2 progress reports: the first one, and the last one. EXPECT_EQ(progress_count, 2); -- cgit v1.2.3 From bffc82b236567dad6aba5ae14d09bcd1127f7d95 Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Wed, 6 Oct 2021 00:58:51 +0000 Subject: [zlib] Fix warnings on msvc Bug: chromium:1255096 Change-Id: Id4515f5d4a01b0f681b63fcf27f3a2dc8846cb72 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3206541 Commit-Queue: Zhi An Ng Reviewed-by: Nico Weber Cr-Commit-Position: refs/heads/main@{#928458} NOKEYCHECK=True GitOrigin-RevId: 237605b2ad8d0a1b31f282a8f29c2dc9000bf0fa --- BUILD.gn | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/BUILD.gn b/BUILD.gn index 74ecbd1..a3378e4 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -27,6 +27,17 @@ config("zlib_internal_config") { # Enable zlib's asserts in debug and fuzzer builds. defines += [ "ZLIB_DEBUG" ] } + + if (is_win && !is_clang) { + # V8 supports building with msvc, these silence some warnings that + # causes compilation to fail (https://crbug.com/1255096). + cflags = [ + "/wd4244", + "/wd4100", + "/wd4702", + "/wd4127", + ] + } } source_set("zlib_common_headers") { -- cgit v1.2.3 From edc0e0684f38d65b052a19287be00d1ce8dd4d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Bostr=C3=B6m?= Date: Fri, 8 Oct 2021 18:40:17 +0000 Subject: Remove DISALLOW_* macros from third_party/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This inlines all remaining DISALLOW_* macros in third_party/. This is done manually (vim regex + manually finding insertion position). IWYU cleanup is left as a separate pass that is easier when these macros go away. Bug: 1010217 Change-Id: I8286e9235e38594bfa870c0ef31609d020ce5b0f Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3204242 Reviewed-by: Lei Zhang Reviewed-by: Will Harris Reviewed-by: Nektarios Paisios Owners-Override: Lei Zhang Commit-Queue: Peter Boström Cr-Commit-Position: refs/heads/main@{#929808} NOKEYCHECK=True GitOrigin-RevId: 706300abcde8ec4974048562c5d447eb573ac7bc --- google/zip_reader.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/google/zip_reader.h b/google/zip_reader.h index 9374beb..ac47447 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -79,6 +79,9 @@ class ZipReader { EntryInfo(const std::string& filename_in_zip, const unz_file_info& raw_file_info); + EntryInfo(const EntryInfo&) = delete; + EntryInfo& operator=(const EntryInfo&) = delete; + // Returns the file path. The path is usually relative like // "foo/bar.txt", but if it's absolute, is_unsafe() returns true. const base::FilePath& file_path() const { return file_path_; } @@ -117,10 +120,13 @@ class ZipReader { bool is_directory_; bool is_unsafe_; bool is_encrypted_; - DISALLOW_COPY_AND_ASSIGN(EntryInfo); }; ZipReader(); + + ZipReader(const ZipReader&) = delete; + ZipReader& operator=(const ZipReader&) = delete; + ~ZipReader(); // Opens the zip file specified by |zip_file_path|. Returns true on @@ -225,8 +231,6 @@ class ZipReader { std::unique_ptr current_entry_info_; base::WeakPtrFactory weak_ptr_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(ZipReader); }; // A writer delegate that writes to a given File. -- cgit v1.2.3 From 6da1d53b97c89b07e47714d88cab61f1ce003c68 Mon Sep 17 00:00:00 2001 From: Peter Kasting Date: Tue, 12 Oct 2021 17:25:16 +0000 Subject: Disable -Wunused-but-set-variable in various third_party directories. Bug: 1203071 Change-Id: I04ee9d41de53bc5b09ad552fff8e76ca4399d9ef Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3216996 Commit-Queue: Peter Kasting Commit-Queue: Dirk Pranke Auto-Submit: Peter Kasting Reviewed-by: Dirk Pranke Cr-Commit-Position: refs/heads/main@{#930624} NOKEYCHECK=True GitOrigin-RevId: 3884c1f0647dccabe81e148e0184a595c2e030d9 --- BUILD.gn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/BUILD.gn b/BUILD.gn index a3378e4..49f52e1 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -496,6 +496,9 @@ if (build_with_chromium) { "//testing/gtest", ] + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] + include_dirs = [ "//third_party/googletest/src/googletest/include/gtest", ".", -- cgit v1.2.3 From f8964a5ccfffe976f3086c286b8e4cf7c9e2be00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Bostr=C3=B6m?= Date: Fri, 12 Nov 2021 03:40:24 +0000 Subject: Remove most remaining unused "base/macros.h" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes `#include "base/macros.h"` from remaining .cc, .h and .mm files that do not contain `ignore_result(` and do not trigger pre-commit or pre-upload errors. Bug: 1010217 No-Try: true Change-Id: I4cc5207b3deafa7901764d2e633bbb1ad43cfb73 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3276692 Commit-Queue: Peter Boström Reviewed-by: Lei Zhang Owners-Override: Lei Zhang Cr-Commit-Position: refs/heads/main@{#941054} NOKEYCHECK=True GitOrigin-RevId: 5666ff4f5077a7e2f72902f3a95f5d553ea0d88d --- google/zip_reader.cc | 1 - google/zip_reader.h | 1 - google/zip_unittest.cc | 1 - 3 files changed, 3 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 0c6a93a..53fa13f 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -9,7 +9,6 @@ #include "base/bind.h" #include "base/files/file.h" #include "base/logging.h" -#include "base/macros.h" #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" #include "base/threading/sequenced_task_runner_handle.h" diff --git a/google/zip_reader.h b/google/zip_reader.h index ac47447..e1ca7aa 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -14,7 +14,6 @@ #include "base/files/file.h" #include "base/files/file_path.h" #include "base/files/file_util.h" -#include "base/macros.h" #include "base/memory/weak_ptr.h" #include "base/time/time.h" diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index d10ec05..944930f 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -17,7 +17,6 @@ #include "base/files/file_util.h" #include "base/files/scoped_temp_dir.h" #include "base/logging.h" -#include "base/macros.h" #include "base/path_service.h" #include "base/strings/string_util.h" #include "base/strings/stringprintf.h" -- cgit v1.2.3 From 2231b7c84b443385f2bd65dd6c46b3c7b957eae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 29 Nov 2021 00:49:43 +0000 Subject: [zip] Add ZipParams::continue_on_error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1062158 TEST=zlib_unittests Change-Id: I9300616a8c7724319e6835855625debcbf3b3246 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3304996 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#945823} NOKEYCHECK=True GitOrigin-RevId: fe805e5824fb5774a068d0da29e7bc9dcba7168f --- google/zip.cc | 4 +++- google/zip.h | 7 +++++++ google/zip_writer.cc | 19 ++++++++++++------- google/zip_writer.h | 6 ++++++ 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index 7c9bd61..7c46718 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -115,7 +115,8 @@ class DirectFileAccessor : public FileAccessor { std::ostream& operator<<(std::ostream& out, const Progress& progress) { return out << progress.bytes << " bytes, " << progress.files << " files, " - << progress.directories << " dirs"; + << progress.directories << " dirs, " << progress.errors + << " errors"; } bool Zip(const ZipParams& params) { @@ -143,6 +144,7 @@ bool Zip(const ZipParams& params) { zip_writer->SetProgressCallback(params.progress_callback, params.progress_period); zip_writer->SetRecursive(params.recursive); + zip_writer->ContinueOnError(params.continue_on_error); if (!params.include_hidden_files || params.filter_callback) zip_writer->SetFilterCallback(base::BindRepeating( diff --git a/google/zip.h b/google/zip.h index ae08a24..ecaecb1 100644 --- a/google/zip.h +++ b/google/zip.h @@ -68,6 +68,10 @@ struct Progress { // Number of directory entries added to the ZIP so far. // A directory entry is added before items in it. int directories = 0; + + // Number of errors encountered so far (files that cannot be opened, + // directories that cannot be listed). + int errors = 0; }; // Prints Progress to output stream. @@ -130,6 +134,9 @@ struct ZipParams { // Should recursively add subdirectory contents? bool recursive = false; + + // Should ignore errors when discovering files and zipping them? + bool continue_on_error = false; }; // Zip files specified into a ZIP archives. The source files and ZIP destination diff --git a/google/zip_writer.cc b/google/zip_writer.cc index 3e2345a..201f199 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -112,12 +112,10 @@ bool ZipWriter::AddFileEntry(const base::FilePath& path, base::File file) { bool ZipWriter::AddDirectoryEntry(const base::FilePath& path) { FileAccessor::Info info; - if (!file_accessor_->GetInfo(path, &info)) - return false; - - if (!info.is_directory) { + if (!file_accessor_->GetInfo(path, &info) || !info.is_directory) { LOG(ERROR) << "Not a directory: " << Redact(path); - return false; + progress_.errors++; + return continue_on_error_; } if (!OpenNewFileEntry(path, /*is_directory=*/true, info.last_modified)) @@ -262,6 +260,11 @@ bool ZipWriter::AddFileEntries(Paths paths) { if (!file.IsValid()) { LOG(ERROR) << "Cannot open " << Redact(relative_path); + progress_.errors++; + + if (continue_on_error_) + continue; + return false; } @@ -285,8 +288,10 @@ bool ZipWriter::AddDirectoryEntries(Paths paths) { bool ZipWriter::AddDirectoryContents(const base::FilePath& path) { std::vector files, subdirs; - if (!file_accessor_->List(path, &files, &subdirs)) - return false; + if (!file_accessor_->List(path, &files, &subdirs)) { + progress_.errors++; + return continue_on_error_; + } Filter(&files); Filter(&subdirs); diff --git a/google/zip_writer.h b/google/zip_writer.h index c58b1b1..afc9390 100644 --- a/google/zip_writer.h +++ b/google/zip_writer.h @@ -57,6 +57,9 @@ class ZipWriter { progress_period_ = std::move(period); } + // Should ignore missing files and directories? + void ContinueOnError(bool b) { continue_on_error_ = b; } + // Sets the recursive flag, indicating whether the contents of subdirectories // should be included. void SetRecursive(bool b) { recursive_ = b; } @@ -138,6 +141,9 @@ class ZipWriter { // Should recursively add directories? bool recursive_ = false; + + // Should ignore missing files and directories? + bool continue_on_error_ = false; }; } // namespace internal -- cgit v1.2.3 From e8c257c917bc3a48a6167bf177fd3eaeacfa7fd2 Mon Sep 17 00:00:00 2001 From: Chris Blume Date: Tue, 30 Nov 2021 23:07:22 +0000 Subject: [zlib] Improve variable name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A parameter named "b" is not descriptive. However, by context that variable's meaning is clear. void ContinueOnError(bool b); clearly means |b| indicates whether or not to continue. Nevertheless, it is probably wise to provide a better name for that variable. This (profoundly nitpicky) commit renames the parameter |b| to |continue_on_error|. Change-Id: Id6830f2d4ed8ec569ec2091624d40fa687bad8fd Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3307620 Auto-Submit: Chris Blume Commit-Queue: François Degros Reviewed-by: François Degros Cr-Commit-Position: refs/heads/main@{#946733} NOKEYCHECK=True GitOrigin-RevId: 8c09262e21c60ad44d02b1e9ec2bd5b5c41db14a --- google/zip_writer.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/google/zip_writer.h b/google/zip_writer.h index afc9390..fcc9627 100644 --- a/google/zip_writer.h +++ b/google/zip_writer.h @@ -58,7 +58,9 @@ class ZipWriter { } // Should ignore missing files and directories? - void ContinueOnError(bool b) { continue_on_error_ = b; } + void ContinueOnError(bool continue_on_error) { + continue_on_error_ = continue_on_error; + } // Sets the recursive flag, indicating whether the contents of subdirectories // should be included. -- cgit v1.2.3 From efd9399ae01364926be2a38946127fdf463480db Mon Sep 17 00:00:00 2001 From: Alexei Svitkine Date: Fri, 3 Dec 2021 00:33:02 +0000 Subject: [zip] Update callsites to use WIN_-prefixed base::File flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flags have been renamed to make it clear that they only apply to Windows (see associated bug). Bug: 1244149 Change-Id: If625d8fac452c52a1bcad20b5fd09ccce016adfe Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3313308 Auto-Submit: Alexei Svitkine Reviewed-by: François Degros Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#947772} NOKEYCHECK=True GitOrigin-RevId: 9c06ed1d427eab86fba138ed93059dfbc7c7d787 --- google/zip_reader_unittest.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index e5bb4b4..c1d654a 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -632,11 +632,10 @@ class FileWriterDelegateTest : public ::testing::Test { protected: void SetUp() override { ASSERT_TRUE(base::CreateTemporaryFile(&temp_file_path_)); - file_.Initialize(temp_file_path_, (base::File::FLAG_CREATE_ALWAYS | - base::File::FLAG_READ | - base::File::FLAG_WRITE | - base::File::FLAG_TEMPORARY | - base::File::FLAG_DELETE_ON_CLOSE)); + file_.Initialize(temp_file_path_, + (base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_READ | + base::File::FLAG_WRITE | base::File::FLAG_WIN_TEMPORARY | + base::File::FLAG_DELETE_ON_CLOSE)); ASSERT_TRUE(file_.IsValid()); } -- cgit v1.2.3 From fc5cfd78a357d5bb7735a58f383634faaafe706a Mon Sep 17 00:00:00 2001 From: Joshua Pawlicki Date: Wed, 12 Jan 2022 16:11:56 +0000 Subject: zlib: Preserve executable bit from extracted archives on POSIX systems. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a file is readable for U/G/O when created, and the archive entry has an extended file attribute indicating that it is executable for that category, the file will be marked as executable for that category. Fixed: 555011 Change-Id: I79bed3525eada71f34cde38d684e9708fc138113 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3379289 Reviewed-by: François Degros Auto-Submit: Joshua Pawlicki Commit-Queue: Joshua Pawlicki Cr-Commit-Position: refs/heads/main@{#958103} NOKEYCHECK=True GitOrigin-RevId: 35e7398d2d51b23faeb4e8459f8419af99733b44 --- google/test/data/README.md | 15 +++++++ google/test/data/test_posix_permissions.zip | Bin 0 -> 574 bytes google/zip_reader.cc | 61 ++++++++++++++++++++++++++-- google/zip_reader.h | 17 ++++++++ google/zip_reader_unittest.cc | 36 ++++++++++++++++ 5 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 google/test/data/README.md create mode 100644 google/test/data/test_posix_permissions.zip diff --git a/google/test/data/README.md b/google/test/data/README.md new file mode 100644 index 0000000..c76648f --- /dev/null +++ b/google/test/data/README.md @@ -0,0 +1,15 @@ +## test\_posix\_permissions.zip +Rebuild this zip by running: +``` +rm test_posix_permissions.zip && +mkdir z && +cd z && +touch 0.txt 1.txt 2.txt 3.txt && +chmod a+x 0.txt && +chmod o+x 1.txt && +chmod u+x 2.txt && +zip test_posix_permissions.zip * && +mv test_posix_permissions.zip .. && +cd .. && +rm -r z +``` diff --git a/google/test/data/test_posix_permissions.zip b/google/test/data/test_posix_permissions.zip new file mode 100644 index 0000000..a058ba1 Binary files /dev/null and b/google/test/data/test_posix_permissions.zip differ diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 53fa13f..f546a90 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -24,6 +24,10 @@ #endif // defined(OS_WIN) #endif // defined(USE_SYSTEM_MINIZIP) +#if defined(OS_POSIX) +#include +#endif + namespace zip { namespace { @@ -52,6 +56,8 @@ class StringWriterDelegate : public WriterDelegate { void SetTimeModified(const base::Time& time) override; + void SetPosixFilePermissions(int mode) override; + private: size_t max_read_bytes_; std::string* output_; @@ -81,6 +87,33 @@ void StringWriterDelegate::SetTimeModified(const base::Time& time) { // Do nothing. } +void StringWriterDelegate::SetPosixFilePermissions(int mode) { + // Do nothing. +} + +#if defined(OS_POSIX) +void SetPosixFilePermissions(int fd, int mode) { + base::stat_wrapper_t sb; + if (base::File::Fstat(fd, &sb)) { + return; + } + mode_t new_mode = sb.st_mode; + // Transfer the executable bit only if the file is readable. + if ((sb.st_mode & S_IRUSR) == S_IRUSR && (mode & S_IXUSR) == S_IXUSR) { + new_mode |= S_IXUSR; + } + if ((sb.st_mode & S_IRGRP) == S_IRGRP && (mode & S_IXGRP) == S_IXGRP) { + new_mode |= S_IXGRP; + } + if ((sb.st_mode & S_IROTH) == S_IROTH && (mode & S_IXOTH) == S_IXOTH) { + new_mode |= S_IXOTH; + } + if (new_mode != sb.st_mode) { + fchmod(fd, new_mode); + } +} +#endif + } // namespace // TODO(satorux): The implementation assumes that file names in zip files @@ -91,7 +124,8 @@ ZipReader::EntryInfo::EntryInfo(const std::string& file_name_in_zip, : file_path_(base::FilePath::FromUTF8Unsafe(file_name_in_zip)), is_directory_(false), is_unsafe_(false), - is_encrypted_(false) { + is_encrypted_(false), + posix_mode_(0) { original_size_ = raw_file_info.uncompressed_size; // Directory entries in zip files end with "/". @@ -132,6 +166,11 @@ ZipReader::EntryInfo::EntryInfo(const std::string& file_name_in_zip, if (!base::Time::FromUTCExploded(exploded_time, &last_modified_)) last_modified_ = base::Time::UnixEpoch(); + +#if defined(OS_POSIX) + posix_mode_ = + (raw_file_info.external_fa >> 16L) & (S_IRWXU | S_IRWXG | S_IRWXO); +#endif } ZipReader::ZipReader() { @@ -277,9 +316,11 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, unzCloseCurrentFile(zip_file_); - if (entire_file_extracted && - current_entry_info()->last_modified() != base::Time::UnixEpoch()) { - delegate->SetTimeModified(current_entry_info()->last_modified()); + if (entire_file_extracted) { + delegate->SetPosixFilePermissions(current_entry_info()->posix_mode()); + if (current_entry_info()->last_modified() != base::Time::UnixEpoch()) { + delegate->SetTimeModified(current_entry_info()->last_modified()); + } } return entire_file_extracted; @@ -471,6 +512,12 @@ void FileWriterDelegate::SetTimeModified(const base::Time& time) { file_->SetTimes(base::Time::Now(), time); } +void FileWriterDelegate::SetPosixFilePermissions(int mode) { +#if defined(OS_POSIX) + zip::SetPosixFilePermissions(file_->GetPlatformFile(), mode); +#endif +} + // FilePathWriterDelegate ------------------------------------------------------ FilePathWriterDelegate::FilePathWriterDelegate( @@ -499,4 +546,10 @@ void FilePathWriterDelegate::SetTimeModified(const base::Time& time) { base::TouchFile(output_file_path_, base::Time::Now(), time); } +void FilePathWriterDelegate::SetPosixFilePermissions(int mode) { +#if defined(OS_POSIX) + zip::SetPosixFilePermissions(file_.GetPlatformFile(), mode); +#endif +} + } // namespace zip diff --git a/google/zip_reader.h b/google/zip_reader.h index e1ca7aa..c2d3bea 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -41,6 +41,11 @@ class WriterDelegate { // Sets the last-modified time of the data. virtual void SetTimeModified(const base::Time& time) = 0; + + // Called with the POSIX file permissions of the data; POSIX implementations + // may apply some of the permissions (for example, the executable bit) to the + // output file. + virtual void SetPosixFilePermissions(int mode) = 0; }; // This class is used for reading zip files. A typical use case of this @@ -112,6 +117,9 @@ class ZipReader { // Returns true if the entry is encrypted. bool is_encrypted() const { return is_encrypted_; } + // Returns the posix file permissions of the entry. + int posix_mode() const { return posix_mode_; } + private: const base::FilePath file_path_; int64_t original_size_; @@ -119,6 +127,7 @@ class ZipReader { bool is_directory_; bool is_unsafe_; bool is_encrypted_; + int posix_mode_; }; ZipReader(); @@ -261,6 +270,10 @@ class FileWriterDelegate : public WriterDelegate { // Sets the last-modified time of the data. void SetTimeModified(const base::Time& time) override; + // On POSIX systems, sets the file to be executable if the source file was + // executable. + void SetPosixFilePermissions(int mode) override; + // Return the actual size of the file. int64_t file_length() { return file_length_; } @@ -297,6 +310,10 @@ class FilePathWriterDelegate : public WriterDelegate { // Sets the last-modified time of the data. void SetTimeModified(const base::Time& time) override; + // On POSIX systems, sets the file to be executable if the source file was + // executable. + void SetPosixFilePermissions(int mode) override; + private: base::FilePath output_file_path_; base::File file_; diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index c1d654a..a15193b 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -23,8 +23,10 @@ #include "base/strings/string_piece.h" #include "base/strings/stringprintf.h" #include "base/strings/utf_string_conversions.h" +#include "base/test/bind.h" #include "base/test/task_environment.h" #include "base/time/time.h" +#include "build/build_config.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" #include "testing/platform_test.h" @@ -111,6 +113,7 @@ class MockWriterDelegate : public zip::WriterDelegate { MOCK_METHOD0(PrepareOutput, bool()); MOCK_METHOD2(WriteBytes, bool(const char*, int)); MOCK_METHOD1(SetTimeModified, void(const base::Time&)); + MOCK_METHOD1(SetPosixFilePermissions, void(int)); }; bool ExtractCurrentEntryToFilePath(zip::ZipReader* reader, @@ -564,6 +567,38 @@ TEST_F(ZipReaderTest, ExtractPartOfCurrentEntry) { reader.Close(); } +TEST_F(ZipReaderTest, ExtractPosixPermissions) { + base::ScopedTempDir temp_dir; + ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); + + ZipReader reader; + ASSERT_TRUE( + reader.Open(test_data_dir_.AppendASCII("test_posix_permissions.zip"))); + for (auto entry : {"0.txt", "1.txt", "2.txt", "3.txt"}) { + ASSERT_TRUE(LocateAndOpenEntry(&reader, base::FilePath::FromASCII(entry))); + FilePathWriterDelegate delegate(temp_dir.GetPath().AppendASCII(entry)); + ASSERT_TRUE(reader.ExtractCurrentEntry(&delegate, 10000)); + } + reader.Close(); + +#if defined(OS_POSIX) + // This assumes a umask of at least 0400. + int mode = 0; + EXPECT_TRUE(base::GetPosixFilePermissions( + temp_dir.GetPath().AppendASCII("0.txt"), &mode)); + EXPECT_EQ(mode & 0700, 0700); + EXPECT_TRUE(base::GetPosixFilePermissions( + temp_dir.GetPath().AppendASCII("1.txt"), &mode)); + EXPECT_EQ(mode & 0700, 0600); + EXPECT_TRUE(base::GetPosixFilePermissions( + temp_dir.GetPath().AppendASCII("2.txt"), &mode)); + EXPECT_EQ(mode & 0700, 0700); + EXPECT_TRUE(base::GetPosixFilePermissions( + temp_dir.GetPath().AppendASCII("3.txt"), &mode)); + EXPECT_EQ(mode & 0700, 0600); +#endif +} + // This test exposes http://crbug.com/430959, at least on OS X TEST_F(ZipReaderTest, DISABLED_LeakDetectionTest) { for (int i = 0; i < 100000; ++i) { @@ -617,6 +652,7 @@ TEST_F(ZipReaderTest, ExtractCurrentEntrySuccess) { .WillOnce(Return(true)); EXPECT_CALL(mock_writer, WriteBytes(_, _)) .WillRepeatedly(Return(true)); + EXPECT_CALL(mock_writer, SetPosixFilePermissions(_)); EXPECT_CALL(mock_writer, SetTimeModified(_)); base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); -- cgit v1.2.3 From 3fc79233fe8ff5cf39fec4c8b8a46272d4f11cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 7 Feb 2022 04:39:30 +0000 Subject: [zip] ZipReader can handle paths in different encodings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added ZipReader::Entry::file_path_in_original_encoding() that gets the file path in its original encoding from a ZIP archive Entry. This will be used to actually detect the encoding. Added ZipReader::SetEncoding() which allows to specify the expected encoding of the ZIP archive. ZipReader now safely converts the file paths from the assumed encoding to Unicode. This safe conversion is performed even if the assumed encoding is UTF-8. This allows to detect and sanitize invalid UTF-8 sequences by replacing them with the replacement character �. Added tests to zlib_unittests to check the encoding conversion: EncodingSjisAsUtf8, EncodingSjisAs1252, EncodingSjisAsIbm866, and EncodingSjis. BUG=chromium:1287893, chromium:953256 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I534f761734b44bfce8417f97bef66bf4757df6ae Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3430328 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#967674} NOKEYCHECK=True GitOrigin-RevId: d9ecb93965cac30b44eca8fd839b0b8ac0ce1c8f --- google/BUILD.gn | 1 + google/test/data/SJIS Bug 846195.zip | Bin 0 -> 479 bytes google/zip_reader.cc | 80 ++++++------ google/zip_reader.h | 33 +++-- google/zip_reader_unittest.cc | 235 +++++++++++++++++++---------------- google/zip_unittest.cc | 12 +- 6 files changed, 191 insertions(+), 170 deletions(-) create mode 100644 google/test/data/SJIS Bug 846195.zip diff --git a/google/BUILD.gn b/google/BUILD.gn index c29e892..1d5c74f 100644 --- a/google/BUILD.gn +++ b/google/BUILD.gn @@ -18,6 +18,7 @@ if (build_with_chromium) { ] deps = [ "//base", + "//base:i18n", "//third_party/zlib:minizip", ] } diff --git a/google/test/data/SJIS Bug 846195.zip b/google/test/data/SJIS Bug 846195.zip new file mode 100644 index 0000000..0783002 Binary files /dev/null and b/google/test/data/SJIS Bug 846195.zip differ diff --git a/google/zip_reader.cc b/google/zip_reader.cc index f546a90..9209b6b 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -8,7 +8,9 @@ #include "base/bind.h" #include "base/files/file.h" +#include "base/i18n/icu_string_conversions.h" #include "base/logging.h" +#include "base/strings/string_piece.h" #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" #include "base/threading/sequenced_task_runner_handle.h" @@ -65,12 +67,9 @@ class StringWriterDelegate : public WriterDelegate { StringWriterDelegate::StringWriterDelegate(size_t max_read_bytes, std::string* output) - : max_read_bytes_(max_read_bytes), - output_(output) { -} + : max_read_bytes_(max_read_bytes), output_(output) {} -StringWriterDelegate::~StringWriterDelegate() { -} +StringWriterDelegate::~StringWriterDelegate() {} bool StringWriterDelegate::PrepareOutput() { return true; @@ -116,38 +115,23 @@ void SetPosixFilePermissions(int fd, int mode) { } // namespace -// TODO(satorux): The implementation assumes that file names in zip files -// are encoded in UTF-8. This is true for zip files created by Zip() -// function in zip.h, but not true for user-supplied random zip files. -ZipReader::EntryInfo::EntryInfo(const std::string& file_name_in_zip, +ZipReader::EntryInfo::EntryInfo(base::FilePath file_path, + std::string file_path_in_original_encoding, const unz_file_info& raw_file_info) - : file_path_(base::FilePath::FromUTF8Unsafe(file_name_in_zip)), - is_directory_(false), - is_unsafe_(false), - is_encrypted_(false), + : file_path_(std::move(file_path)), + file_path_in_original_encoding_( + std::move(file_path_in_original_encoding)), posix_mode_(0) { original_size_ = raw_file_info.uncompressed_size; // Directory entries in zip files end with "/". - is_directory_ = base::EndsWith(file_name_in_zip, "/", - base::CompareCase::INSENSITIVE_ASCII); + is_directory_ = base::EndsWith(file_path_in_original_encoding_, "/"); // Check the file name here for directory traversal issues. - is_unsafe_ = file_path_.ReferencesParent(); - - // We also consider that the file name is unsafe, if it's invalid UTF-8. - std::u16string file_name_utf16; - if (!base::UTF8ToUTF16(file_name_in_zip.data(), file_name_in_zip.size(), - &file_name_utf16)) { - is_unsafe_ = true; - } - // We also consider that the file name is unsafe, if it's absolute. // On Windows, IsAbsolute() returns false for paths starting with "/". - if (file_path_.IsAbsolute() || - base::StartsWith(file_name_in_zip, "/", - base::CompareCase::INSENSITIVE_ASCII)) - is_unsafe_ = true; + is_unsafe_ = file_path_.ReferencesParent() || file_path_.IsAbsolute() || + base::StartsWith(file_path_in_original_encoding_, "/"); // Whether the file is encrypted is bit 0 of the flag. is_encrypted_ = raw_file_info.flag & 1; @@ -256,21 +240,30 @@ bool ZipReader::OpenCurrentEntryInZip() { DCHECK(zip_file_); unz_file_info raw_file_info = {}; - char raw_file_name_in_zip[internal::kZipMaxPath] = {}; - const int result = unzGetCurrentFileInfo(zip_file_, - &raw_file_info, - raw_file_name_in_zip, - sizeof(raw_file_name_in_zip) - 1, - NULL, // extraField. - 0, // extraFieldBufferSize. - NULL, // szComment. - 0); // commentBufferSize. + char file_path_in_zip[internal::kZipMaxPath] = {}; + const int result = unzGetCurrentFileInfo( + zip_file_, &raw_file_info, file_path_in_zip, sizeof(file_path_in_zip) - 1, + nullptr, // extraField. + 0, // extraFieldBufferSize. + nullptr, // szComment. + 0); // commentBufferSize. if (result != UNZ_OK) return false; - if (raw_file_name_in_zip[0] == '\0') + + // Convert file path from original encoding to Unicode. + const base::StringPiece file_path_in_original_encoding(file_path_in_zip); + std::u16string file_path_in_utf16; + const char* const encoding = encoding_.empty() ? "UTF-8" : encoding_.c_str(); + if (!base::CodepageToUTF16(file_path_in_original_encoding, encoding, + base::OnStringConversionError::SUBSTITUTE, + &file_path_in_utf16)) { + LOG(ERROR) << "Cannot convert file path from encoding " << encoding; return false; - current_entry_info_.reset( - new EntryInfo(raw_file_name_in_zip, raw_file_info)); + } + + current_entry_info_.reset(new EntryInfo( + base::FilePath::FromUTF16Unsafe(file_path_in_utf16), + std::string(file_path_in_original_encoding), raw_file_info)); return true; } @@ -440,7 +433,7 @@ bool ZipReader::OpenInternal() { } void ZipReader::Reset() { - zip_file_ = NULL; + zip_file_ = nullptr; num_entries_ = 0; reached_end_ = false; current_entry_info_.reset(); @@ -453,9 +446,8 @@ void ZipReader::ExtractChunk(base::File output_file, const int64_t offset) { char buffer[internal::kZipBufSize]; - const int num_bytes_read = unzReadCurrentFile(zip_file_, - buffer, - internal::kZipBufSize); + const int num_bytes_read = + unzReadCurrentFile(zip_file_, buffer, internal::kZipBufSize); if (num_bytes_read == 0) { unzCloseCurrentFile(zip_file_); diff --git a/google/zip_reader.h b/google/zip_reader.h index c2d3bea..2199357 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -80,7 +80,8 @@ class ZipReader { // a zip file. class EntryInfo { public: - EntryInfo(const std::string& filename_in_zip, + EntryInfo(base::FilePath file_path, + std::string file_path_in_original_encoding, const unz_file_info& raw_file_info); EntryInfo(const EntryInfo&) = delete; @@ -90,6 +91,14 @@ class ZipReader { // "foo/bar.txt", but if it's absolute, is_unsafe() returns true. const base::FilePath& file_path() const { return file_path_; } + // Returns the file path in its original encoding as it is stored in the ZIP + // archive. The encoding is not specified here. It might not be UTF-8, and + // the caller needs to use other means to determine the encoding if it wants + // to interpret this file path correctly. + const std::string& file_path_in_original_encoding() const { + return file_path_in_original_encoding_; + } + // Returns the size of the original file (i.e. after uncompressed). // Returns 0 if the entry is a directory. // Note: this value should not be trusted, because it is stored as metadata @@ -121,7 +130,8 @@ class ZipReader { int posix_mode() const { return posix_mode_; } private: - const base::FilePath file_path_; + base::FilePath file_path_; + std::string file_path_in_original_encoding_; int64_t original_size_; base::Time last_modified_; bool is_directory_; @@ -154,6 +164,10 @@ class ZipReader { // destructor of the class, so you usually don't need to call this. void Close(); + // Sets the encoding of file paths in the ZIP archive. + // By default, file paths are assumed to be in UTF-8. + void SetEncoding(std::string encoding) { encoding_ = std::move(encoding); } + // Returns true if there is at least one entry to read. This function is // used to scan entries with AdvanceToNextEntry(), like: // @@ -167,12 +181,12 @@ class ZipReader { bool AdvanceToNextEntry(); // Opens the current entry in the zip file. On success, returns true and - // updates the the current entry state (i.e. current_entry_info() is - // updated). This function should be called before operations over the - // current entry like ExtractCurrentEntryToFile(). + // updates the current entry state (i.e. current_entry_info() is updated). + // This function should be called before operations over the current entry + // like ExtractCurrentEntryToFile(). // - // Note that there is no CloseCurrentEntryInZip(). The the current entry - // state is reset automatically as needed. + // Note that there is no CloseCurrentEntryInZip(). The current entry state is + // reset automatically as needed. bool OpenCurrentEntryInZip(); // Extracts |num_bytes_to_extract| bytes of the current entry to |delegate|, @@ -210,9 +224,7 @@ class ZipReader { // Returns the current entry info. Returns NULL if the current entry is // not yet opened. OpenCurrentEntryInZip() must be called beforehand. - EntryInfo* current_entry_info() const { - return current_entry_info_.get(); - } + EntryInfo* current_entry_info() const { return current_entry_info_.get(); } // Returns the number of entries in the zip file. // Open() must be called beforehand. @@ -233,6 +245,7 @@ class ZipReader { const ProgressCallback& progress_callback, const int64_t offset); + std::string encoding_; unzFile zip_file_; int num_entries_; bool reached_end_; diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index a15193b..f4ecfbd 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -8,13 +8,14 @@ #include #include -#include #include +#include #include "base/bind.h" #include "base/check.h" #include "base/cxx17_backports.h" #include "base/files/file.h" +#include "base/files/file_path.h" #include "base/files/file_util.h" #include "base/files/scoped_temp_dir.h" #include "base/hash/md5.h" @@ -32,8 +33,10 @@ #include "testing/platform_test.h" #include "third_party/zlib/google/zip_internal.h" -using ::testing::Return; using ::testing::_; +using ::testing::ElementsAre; +using ::testing::ElementsAreArray; +using ::testing::Return; namespace { @@ -41,10 +44,7 @@ const static std::string kQuuxExpectedMD5 = "d1ae4ac8a17a0e09317113ab284b57a6"; class FileWrapper { public: - typedef enum { - READ_ONLY, - READ_WRITE - } AccessMode; + typedef enum { READ_ONLY, READ_WRITE } AccessMode; FileWrapper(const base::FilePath& path, AccessMode mode) { int flags = base::File::FLAG_READ; @@ -75,18 +75,13 @@ class MockUnzipListener : public base::SupportsWeakPtr { : success_calls_(0), failure_calls_(0), progress_calls_(0), - current_progress_(0) { - } + current_progress_(0) {} // Success callback for async functions. - void OnUnzipSuccess() { - success_calls_++; - } + void OnUnzipSuccess() { success_calls_++; } // Failure callback for async functions. - void OnUnzipFailure() { - failure_calls_++; - } + void OnUnzipFailure() { failure_calls_++; } // Progress callback for async functions. void OnUnzipProgress(int64_t progress) { @@ -137,87 +132,66 @@ bool LocateAndOpenEntry(zip::ZipReader* reader, return false; } -} // namespace +using Paths = std::vector; + +} // namespace namespace zip { // Make the test a PlatformTest to setup autorelease pools properly on Mac. class ZipReaderTest : public PlatformTest { protected: - virtual void SetUp() { + void SetUp() override { PlatformTest::SetUp(); ASSERT_TRUE(temp_dir_.CreateUniqueTempDir()); test_dir_ = temp_dir_.GetPath(); - - ASSERT_TRUE(GetTestDataDirectory(&test_data_dir_)); - - test_zip_file_ = test_data_dir_.AppendASCII("test.zip"); - encrypted_zip_file_ = test_data_dir_.AppendASCII("test_encrypted.zip"); - evil_zip_file_ = test_data_dir_.AppendASCII("evil.zip"); - evil_via_invalid_utf8_zip_file_ = test_data_dir_.AppendASCII( - "evil_via_invalid_utf8.zip"); - evil_via_absolute_file_name_zip_file_ = test_data_dir_.AppendASCII( - "evil_via_absolute_file_name.zip"); - - test_zip_contents_.insert(base::FilePath(FILE_PATH_LITERAL("foo/"))); - test_zip_contents_.insert(base::FilePath(FILE_PATH_LITERAL("foo/bar/"))); - test_zip_contents_.insert( - base::FilePath(FILE_PATH_LITERAL("foo/bar/baz.txt"))); - test_zip_contents_.insert( - base::FilePath(FILE_PATH_LITERAL("foo/bar/quux.txt"))); - test_zip_contents_.insert( - base::FilePath(FILE_PATH_LITERAL("foo/bar.txt"))); - test_zip_contents_.insert(base::FilePath(FILE_PATH_LITERAL("foo.txt"))); - test_zip_contents_.insert( - base::FilePath(FILE_PATH_LITERAL("foo/bar/.hidden"))); } - virtual void TearDown() { - PlatformTest::TearDown(); + static base::FilePath GetTestDataDirectory() { + base::FilePath path; + CHECK(base::PathService::Get(base::DIR_SOURCE_ROOT, &path)); + return path.AppendASCII("third_party") + .AppendASCII("zlib") + .AppendASCII("google") + .AppendASCII("test") + .AppendASCII("data"); } - bool GetTestDataDirectory(base::FilePath* path) { - bool success = base::PathService::Get(base::DIR_SOURCE_ROOT, path); - EXPECT_TRUE(success); - if (!success) - return false; - *path = path->AppendASCII("third_party"); - *path = path->AppendASCII("zlib"); - *path = path->AppendASCII("google"); - *path = path->AppendASCII("test"); - *path = path->AppendASCII("data"); - return true; - } + static Paths GetPaths(const base::FilePath& zip_path, + base::StringPiece encoding = {}) { + Paths paths; - bool CompareFileAndMD5(const base::FilePath& path, - const std::string expected_md5) { - // Read the output file and compute the MD5. - std::string output; - if (!base::ReadFileToString(path, &output)) - return false; - const std::string md5 = base::MD5String(output); - return expected_md5 == md5; + if (ZipReader reader; reader.Open(zip_path)) { + if (!encoding.empty()) + reader.SetEncoding(std::string(encoding)); + + while (reader.HasMore()) { + if (reader.OpenCurrentEntryInZip()) + paths.push_back(reader.current_entry_info()->file_path()); + reader.AdvanceToNextEntry(); + } + } + + return paths; } // The path to temporary directory used to contain the test operations. base::FilePath test_dir_; // The path to the test data directory where test.zip etc. are located. - base::FilePath test_data_dir_; + const base::FilePath data_dir_ = GetTestDataDirectory(); // The path to test.zip in the test data directory. - base::FilePath test_zip_file_; - // The path to test_encrypted.zip in the test data directory. - base::FilePath encrypted_zip_file_; - // The path to evil.zip in the test data directory. - base::FilePath evil_zip_file_; - // The path to evil_via_invalid_utf8.zip in the test data directory. - base::FilePath evil_via_invalid_utf8_zip_file_; - // The path to evil_via_absolute_file_name.zip in the test data directory. - base::FilePath evil_via_absolute_file_name_zip_file_; - std::set test_zip_contents_; - + const base::FilePath test_zip_file_ = data_dir_.AppendASCII("test.zip"); + const Paths test_zip_contents_ = { + base::FilePath(FILE_PATH_LITERAL("foo/")), + base::FilePath(FILE_PATH_LITERAL("foo/bar/")), + base::FilePath(FILE_PATH_LITERAL("foo/bar/baz.txt")), + base::FilePath(FILE_PATH_LITERAL("foo/bar/quux.txt")), + base::FilePath(FILE_PATH_LITERAL("foo/bar.txt")), + base::FilePath(FILE_PATH_LITERAL("foo.txt")), + base::FilePath(FILE_PATH_LITERAL("foo/bar/.hidden")), + }; base::ScopedTempDir temp_dir_; - base::test::TaskEnvironment task_environment_; }; @@ -234,49 +208,47 @@ TEST_F(ZipReaderTest, Open_ValidZipPlatformFile) { TEST_F(ZipReaderTest, Open_NonExistentFile) { ZipReader reader; - ASSERT_FALSE(reader.Open(test_data_dir_.AppendASCII("nonexistent.zip"))); + ASSERT_FALSE(reader.Open(data_dir_.AppendASCII("nonexistent.zip"))); } TEST_F(ZipReaderTest, Open_ExistentButNonZipFile) { ZipReader reader; - ASSERT_FALSE(reader.Open(test_data_dir_.AppendASCII("create_test_zip.sh"))); + ASSERT_FALSE(reader.Open(data_dir_.AppendASCII("create_test_zip.sh"))); } // Iterate through the contents in the test zip file, and compare that the // contents collected from the zip reader matches the expected contents. TEST_F(ZipReaderTest, Iteration) { - std::set actual_contents; + Paths actual_contents; ZipReader reader; ASSERT_TRUE(reader.Open(test_zip_file_)); while (reader.HasMore()) { ASSERT_TRUE(reader.OpenCurrentEntryInZip()); - actual_contents.insert(reader.current_entry_info()->file_path()); + actual_contents.push_back(reader.current_entry_info()->file_path()); ASSERT_TRUE(reader.AdvanceToNextEntry()); } EXPECT_FALSE(reader.AdvanceToNextEntry()); // Shouldn't go further. EXPECT_EQ(test_zip_contents_.size(), static_cast(reader.num_entries())); - EXPECT_EQ(test_zip_contents_.size(), actual_contents.size()); - EXPECT_EQ(test_zip_contents_, actual_contents); + EXPECT_THAT(actual_contents, ElementsAreArray(test_zip_contents_)); } // Open the test zip file from a file descriptor, iterate through its contents, // and compare that they match the expected contents. TEST_F(ZipReaderTest, PlatformFileIteration) { - std::set actual_contents; + Paths actual_contents; ZipReader reader; FileWrapper zip_fd_wrapper(test_zip_file_, FileWrapper::READ_ONLY); ASSERT_TRUE(reader.OpenFromPlatformFile(zip_fd_wrapper.platform_file())); while (reader.HasMore()) { ASSERT_TRUE(reader.OpenCurrentEntryInZip()); - actual_contents.insert(reader.current_entry_info()->file_path()); + actual_contents.push_back(reader.current_entry_info()->file_path()); ASSERT_TRUE(reader.AdvanceToNextEntry()); } EXPECT_FALSE(reader.AdvanceToNextEntry()); // Shouldn't go further. EXPECT_EQ(test_zip_contents_.size(), static_cast(reader.num_entries())); - EXPECT_EQ(test_zip_contents_.size(), actual_contents.size()); - EXPECT_EQ(test_zip_contents_, actual_contents); + EXPECT_THAT(actual_contents, ElementsAreArray(test_zip_contents_)); } TEST_F(ZipReaderTest, current_entry_info_RegularFile) { @@ -306,7 +278,7 @@ TEST_F(ZipReaderTest, current_entry_info_RegularFile) { TEST_F(ZipReaderTest, current_entry_info_DotDotFile) { ZipReader reader; - ASSERT_TRUE(reader.Open(evil_zip_file_)); + ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("evil.zip"))); base::FilePath target_path(FILE_PATH_LITERAL( "../levilevilevilevilevilevilevilevilevilevilevilevil")); ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); @@ -320,22 +292,73 @@ TEST_F(ZipReaderTest, current_entry_info_DotDotFile) { TEST_F(ZipReaderTest, current_entry_info_InvalidUTF8File) { ZipReader reader; - ASSERT_TRUE(reader.Open(evil_via_invalid_utf8_zip_file_)); + ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("evil_via_invalid_utf8.zip"))); // The evil file is the 2nd file in the zip file. // We cannot locate by the file name ".\x80.\\evil.txt", // as FilePath may internally convert the string. ASSERT_TRUE(reader.AdvanceToNextEntry()); ASSERT_TRUE(reader.OpenCurrentEntryInZip()); - ZipReader::EntryInfo* current_entry_info = reader.current_entry_info(); + const ZipReader::EntryInfo* const entry = reader.current_entry_info(); - // This file is unsafe because of invalid UTF-8 in the file name. - EXPECT_TRUE(current_entry_info->is_unsafe()); - EXPECT_FALSE(current_entry_info->is_directory()); + ASSERT_TRUE(entry); + EXPECT_FALSE(entry->is_unsafe()); + EXPECT_FALSE(entry->is_directory()); + EXPECT_EQ(entry->file_path(), + base::FilePath::FromUTF8Unsafe(".�.\\evil.txt")); +} + +// By default, file paths in ZIPs are interpreted as UTF-8. But in this test, +// the ZIP archive contains file paths that are actually encoded in Shift JIS. +// The SJIS-encoded paths are thus wrongly interpreted as UTF-8, resulting in +// garbled paths. Invalid UTF-8 sequences are safely converted to the +// replacement character �. +TEST_F(ZipReaderTest, EncodingSjisAsUtf8) { + EXPECT_THAT( + GetPaths(data_dir_.AppendASCII("SJIS Bug 846195.zip")), + ElementsAre( + base::FilePath::FromUTF8Unsafe("�V�����t�H���_/SJIS_835C_�\\.txt"), + base::FilePath::FromUTF8Unsafe( + "�V�����t�H���_/�V�����e�L�X�g �h�L�������g.txt"))); +} + +// In this test, SJIS-encoded paths are interpreted as Code Page 1252. This +// results in garbled paths. Note the presence of C1 control codes U+0090 and +// U+0081 in the garbled paths. +TEST_F(ZipReaderTest, EncodingSjisAs1252) { + EXPECT_THAT( + GetPaths(data_dir_.AppendASCII("SJIS Bug 846195.zip"), "windows-1252"), + ElementsAre(base::FilePath::FromUTF8Unsafe( + "\u0090V‚µ‚¢ƒtƒHƒ‹ƒ_/SJIS_835C_ƒ\\.txt"), + base::FilePath::FromUTF8Unsafe( + "\u0090V‚µ‚¢ƒtƒHƒ‹ƒ_/\u0090V‚µ‚¢ƒeƒLƒXƒg " + "ƒhƒLƒ…ƒ\u0081ƒ“ƒg.txt"))); +} + +// In this test, SJIS-encoded paths are interpreted as Code Page 866. This +// results in garbled paths. +TEST_F(ZipReaderTest, EncodingSjisAsIbm866) { + EXPECT_THAT( + GetPaths(data_dir_.AppendASCII("SJIS Bug 846195.zip"), "IBM866"), + ElementsAre( + base::FilePath::FromUTF8Unsafe("РVВ╡ВвГtГHГЛГ_/SJIS_835C_Г\\.txt"), + base::FilePath::FromUTF8Unsafe( + "РVВ╡ВвГtГHГЛГ_/РVВ╡ВвГeГLГXГg ГhГLГЕГБГУГg.txt"))); +} + +// Tests that SJIS-encoded paths are correctly converted to Unicode. +TEST_F(ZipReaderTest, EncodingSjis) { + EXPECT_THAT( + GetPaths(data_dir_.AppendASCII("SJIS Bug 846195.zip"), "Shift_JIS"), + ElementsAre( + base::FilePath::FromUTF8Unsafe("新しいフォルダ/SJIS_835C_ソ.txt"), + base::FilePath::FromUTF8Unsafe( + "新しいフォルダ/新しいテキスト ドキュメント.txt"))); } TEST_F(ZipReaderTest, current_entry_info_AbsoluteFile) { ZipReader reader; - ASSERT_TRUE(reader.Open(evil_via_absolute_file_name_zip_file_)); + ASSERT_TRUE( + reader.Open(data_dir_.AppendASCII("evil_via_absolute_file_name.zip"))); base::FilePath target_path(FILE_PATH_LITERAL("/evil.txt")); ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); ZipReader::EntryInfo* current_entry_info = reader.current_entry_info(); @@ -377,7 +400,7 @@ TEST_F(ZipReaderTest, current_entry_info_EncryptedFile) { ZipReader reader; base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); - ASSERT_TRUE(reader.Open(encrypted_zip_file_)); + ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("test_encrypted.zip"))); ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); EXPECT_TRUE(reader.current_entry_info()->is_encrypted()); reader.Close(); @@ -416,8 +439,8 @@ TEST_F(ZipReaderTest, OpenFromString) { test_dir_.AppendASCII("test.txt"))); std::string actual; - ASSERT_TRUE(base::ReadFileToString( - test_dir_.AppendASCII("test.txt"), &actual)); + ASSERT_TRUE( + base::ReadFileToString(test_dir_.AppendASCII("test.txt"), &actual)); EXPECT_EQ(std::string("This is a test.\n"), actual); } @@ -448,8 +471,8 @@ TEST_F(ZipReaderTest, ExtractToFileAsync_RegularFile) { EXPECT_LE(1, listener.progress_calls()); std::string output; - ASSERT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("quux.txt"), - &output)); + ASSERT_TRUE( + base::ReadFileToString(test_dir_.AppendASCII("quux.txt"), &output)); const std::string md5 = base::MD5String(output); EXPECT_EQ(kQuuxExpectedMD5, md5); @@ -493,7 +516,7 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryToString) { // sizes from 0 to 7 bytes respectively, being the contents of each file a // substring of "0123456" starting at '0'. base::FilePath test_zip_file = - test_data_dir_.AppendASCII("test_mismatch_size.zip"); + data_dir_.AppendASCII("test_mismatch_size.zip"); ZipReader reader; std::string contents; @@ -529,7 +552,7 @@ TEST_F(ZipReaderTest, ExtractPartOfCurrentEntry) { // sizes from 0 to 7 bytes respectively, being the contents of each file a // substring of "0123456" starting at '0'. base::FilePath test_zip_file = - test_data_dir_.AppendASCII("test_mismatch_size.zip"); + data_dir_.AppendASCII("test_mismatch_size.zip"); ZipReader reader; std::string contents; @@ -572,8 +595,7 @@ TEST_F(ZipReaderTest, ExtractPosixPermissions) { ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); ZipReader reader; - ASSERT_TRUE( - reader.Open(test_data_dir_.AppendASCII("test_posix_permissions.zip"))); + ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("test_posix_permissions.zip"))); for (auto entry : {"0.txt", "1.txt", "2.txt", "3.txt"}) { ASSERT_TRUE(LocateAndOpenEntry(&reader, base::FilePath::FromASCII(entry))); FilePathWriterDelegate delegate(temp_dir.GetPath().AppendASCII(entry)); @@ -613,8 +635,7 @@ TEST_F(ZipReaderTest, DISABLED_LeakDetectionTest) { TEST_F(ZipReaderTest, ExtractCurrentEntryPrepareFailure) { testing::StrictMock mock_writer; - EXPECT_CALL(mock_writer, PrepareOutput()) - .WillOnce(Return(false)); + EXPECT_CALL(mock_writer, PrepareOutput()).WillOnce(Return(false)); base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); ZipReader reader; @@ -630,10 +651,8 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryPrepareFailure) { TEST_F(ZipReaderTest, ExtractCurrentEntryWriteBytesFailure) { testing::StrictMock mock_writer; - EXPECT_CALL(mock_writer, PrepareOutput()) - .WillOnce(Return(true)); - EXPECT_CALL(mock_writer, WriteBytes(_, _)) - .WillOnce(Return(false)); + EXPECT_CALL(mock_writer, PrepareOutput()).WillOnce(Return(true)); + EXPECT_CALL(mock_writer, WriteBytes(_, _)).WillOnce(Return(false)); base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); ZipReader reader; @@ -648,10 +667,8 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryWriteBytesFailure) { TEST_F(ZipReaderTest, ExtractCurrentEntrySuccess) { testing::StrictMock mock_writer; - EXPECT_CALL(mock_writer, PrepareOutput()) - .WillOnce(Return(true)); - EXPECT_CALL(mock_writer, WriteBytes(_, _)) - .WillRepeatedly(Return(true)); + EXPECT_CALL(mock_writer, PrepareOutput()).WillOnce(Return(true)); + EXPECT_CALL(mock_writer, WriteBytes(_, _)).WillRepeatedly(Return(true)); EXPECT_CALL(mock_writer, SetPosixFilePermissions(_)); EXPECT_CALL(mock_writer, SetTimeModified(_)); diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 944930f..53f76eb 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -346,16 +346,14 @@ TEST_F(ZipTest, UnzipEvil) { TEST_F(ZipTest, UnzipEvil2) { base::FilePath path; ASSERT_TRUE(GetTestDataDirectory(&path)); - // The zip file contains an evil file with invalid UTF-8 in its file - // name. + // The ZIP file contains a file with invalid UTF-8 in its file name. path = path.AppendASCII("evil_via_invalid_utf8.zip"); // See the comment at UnzipEvil() for why we do this. base::FilePath output_dir = test_dir_.AppendASCII("out"); - // This should fail as it contains an evil file. - ASSERT_FALSE(zip::Unzip(path, output_dir)); - base::FilePath evil_file = output_dir; - evil_file = evil_file.AppendASCII("../evil.txt"); - ASSERT_FALSE(base::PathExists(evil_file)); + ASSERT_TRUE(zip::Unzip(path, output_dir)); + ASSERT_TRUE(base::PathExists( + output_dir.Append(base::FilePath::FromUTF8Unsafe(".�.\\evil.txt")))); + ASSERT_FALSE(base::PathExists(output_dir.AppendASCII("../evil.txt"))); } TEST_F(ZipTest, UnzipWithFilter) { -- cgit v1.2.3 From 172367d1f35fb67f09ab577ee35901815b6470af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 9 Feb 2022 06:06:16 +0000 Subject: [zip] Add ZipReader::Entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preparing the transition from (class) ZipReader::EntryInfo to (struct) ZipReader::Entry. This will remove boilerplate code, and will provide optimization and simplification opportunities. Avoid dynamically allocating the Entry over and over again. Reuse the same internal Entry object. BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: Ie51e1bda05408b28f53957ea89c78d37d8009f5c Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3446405 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#968767} NOKEYCHECK=True GitOrigin-RevId: 153ddbabb7339501c16310337d9b240c37929fb6 --- google/zip_reader.cc | 130 ++++++++++++++++++------------------- google/zip_reader.h | 146 +++++++++++++++++++++--------------------- google/zip_reader_unittest.cc | 10 +-- 3 files changed, 141 insertions(+), 145 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 9209b6b..bd36741 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -115,48 +115,6 @@ void SetPosixFilePermissions(int fd, int mode) { } // namespace -ZipReader::EntryInfo::EntryInfo(base::FilePath file_path, - std::string file_path_in_original_encoding, - const unz_file_info& raw_file_info) - : file_path_(std::move(file_path)), - file_path_in_original_encoding_( - std::move(file_path_in_original_encoding)), - posix_mode_(0) { - original_size_ = raw_file_info.uncompressed_size; - - // Directory entries in zip files end with "/". - is_directory_ = base::EndsWith(file_path_in_original_encoding_, "/"); - - // Check the file name here for directory traversal issues. - // We also consider that the file name is unsafe, if it's absolute. - // On Windows, IsAbsolute() returns false for paths starting with "/". - is_unsafe_ = file_path_.ReferencesParent() || file_path_.IsAbsolute() || - base::StartsWith(file_path_in_original_encoding_, "/"); - - // Whether the file is encrypted is bit 0 of the flag. - is_encrypted_ = raw_file_info.flag & 1; - - // Construct the last modified time. The timezone info is not present in - // zip files, so we construct the time as local time. - base::Time::Exploded exploded_time = {}; // Zero-clear. - exploded_time.year = raw_file_info.tmu_date.tm_year; - // The month in zip file is 0-based, whereas ours is 1-based. - exploded_time.month = raw_file_info.tmu_date.tm_mon + 1; - exploded_time.day_of_month = raw_file_info.tmu_date.tm_mday; - exploded_time.hour = raw_file_info.tmu_date.tm_hour; - exploded_time.minute = raw_file_info.tmu_date.tm_min; - exploded_time.second = raw_file_info.tmu_date.tm_sec; - exploded_time.millisecond = 0; - - if (!base::Time::FromUTCExploded(exploded_time, &last_modified_)) - last_modified_ = base::Time::UnixEpoch(); - -#if defined(OS_POSIX) - posix_mode_ = - (raw_file_info.external_fa >> 16L) & (S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - ZipReader::ZipReader() { Reset(); } @@ -165,12 +123,12 @@ ZipReader::~ZipReader() { Close(); } -bool ZipReader::Open(const base::FilePath& zip_file_path) { +bool ZipReader::Open(const base::FilePath& zip_path) { DCHECK(!zip_file_); // Use of "Unsafe" function does not look good, but there is no way to do // this safely on Linux. See file_util.h for details. - zip_file_ = internal::OpenForUnzipping(zip_file_path.AsUTF8Unsafe()); + zip_file_ = internal::OpenForUnzipping(zip_path.AsUTF8Unsafe()); if (!zip_file_) { return false; } @@ -223,7 +181,7 @@ bool ZipReader::AdvanceToNextEntry() { return false; const int current_entry_index = position.num_of_file; // If we are currently at the last entry, then the next position is the - // end of the zip file, so mark that we reached the end. + // end of the ZIP archive, so mark that we reached the end. if (current_entry_index + 1 == num_entries_) { reached_end_ = true; } else { @@ -232,38 +190,75 @@ bool ZipReader::AdvanceToNextEntry() { return false; } } - current_entry_info_.reset(); + + entry_ = {}; + current_entry_ = nullptr; return true; } bool ZipReader::OpenCurrentEntryInZip() { DCHECK(zip_file_); - unz_file_info raw_file_info = {}; - char file_path_in_zip[internal::kZipMaxPath] = {}; - const int result = unzGetCurrentFileInfo( - zip_file_, &raw_file_info, file_path_in_zip, sizeof(file_path_in_zip) - 1, - nullptr, // extraField. - 0, // extraFieldBufferSize. - nullptr, // szComment. - 0); // commentBufferSize. - if (result != UNZ_OK) + current_entry_ = nullptr; + + // Get entry info. + unz_file_info info = {}; + char path_in_zip[internal::kZipMaxPath] = {}; + if (unzGetCurrentFileInfo(zip_file_, &info, path_in_zip, + sizeof(path_in_zip) - 1, nullptr, 0, nullptr, + 0) != UNZ_OK) { return false; + } + + Entry& entry = entry_; + entry.path_in_original_encoding = path_in_zip; - // Convert file path from original encoding to Unicode. - const base::StringPiece file_path_in_original_encoding(file_path_in_zip); - std::u16string file_path_in_utf16; + // Convert path from original encoding to Unicode. + std::u16string path_in_utf16; const char* const encoding = encoding_.empty() ? "UTF-8" : encoding_.c_str(); - if (!base::CodepageToUTF16(file_path_in_original_encoding, encoding, + if (!base::CodepageToUTF16(entry.path_in_original_encoding, encoding, base::OnStringConversionError::SUBSTITUTE, - &file_path_in_utf16)) { - LOG(ERROR) << "Cannot convert file path from encoding " << encoding; + &path_in_utf16)) { + LOG(ERROR) << "Cannot convert path from encoding " << encoding; return false; } - current_entry_info_.reset(new EntryInfo( - base::FilePath::FromUTF16Unsafe(file_path_in_utf16), - std::string(file_path_in_original_encoding), raw_file_info)); + entry.path = base::FilePath::FromUTF16Unsafe(path_in_utf16); + entry.original_size = info.uncompressed_size; + + // Directory entries in ZIP have a path ending with "/". + entry.is_directory = base::EndsWith(path_in_utf16, u"/"); + + // Check the entry path for directory traversal issues. We consider entry + // paths unsafe if they are absolute or if they contain "..". On Windows, + // IsAbsolute() returns false for paths starting with "/". + entry.is_unsafe = entry.path.ReferencesParent() || entry.path.IsAbsolute() || + base::StartsWith(path_in_utf16, u"/"); + + // The file content of this entry is encrypted if flag bit 0 is set. + entry.is_encrypted = info.flag & 1; + + // Construct the last modified time. The timezone info is not present in ZIP + // archives, so we construct the time as UTC. + base::Time::Exploded exploded_time = {}; + exploded_time.year = info.tmu_date.tm_year; + exploded_time.month = info.tmu_date.tm_mon + 1; // 0-based vs 1-based + exploded_time.day_of_month = info.tmu_date.tm_mday; + exploded_time.hour = info.tmu_date.tm_hour; + exploded_time.minute = info.tmu_date.tm_min; + exploded_time.second = info.tmu_date.tm_sec; + exploded_time.millisecond = 0; + + if (!base::Time::FromUTCExploded(exploded_time, &entry.last_modified)) + entry.last_modified = base::Time::UnixEpoch(); + +#if defined(OS_POSIX) + entry.posix_mode = (info.external_fa >> 16L) & (S_IRWXU | S_IRWXG | S_IRWXO); +#else + entry.posix_mode = 0; +#endif + + current_entry_ = &entry_; return true; } @@ -325,7 +320,7 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( FailureCallback failure_callback, const ProgressCallback& progress_callback) { DCHECK(zip_file_); - DCHECK(current_entry_info_.get()); + DCHECK(current_entry_); // If this is a directory, just create it and return. if (current_entry_info()->is_directory()) { @@ -427,7 +422,7 @@ bool ZipReader::OpenInternal() { if (num_entries_ < 0) return false; - // We are already at the end if the zip file is empty. + // We are already at the end if the ZIP archive is empty. reached_end_ = (num_entries_ == 0); return true; } @@ -436,7 +431,8 @@ void ZipReader::Reset() { zip_file_ = nullptr; num_entries_ = 0; reached_end_ = false; - current_entry_info_.reset(); + entry_ = {}; + current_entry_ = nullptr; } void ZipReader::ExtractChunk(base::File output_file, diff --git a/google/zip_reader.h b/google/zip_reader.h index 2199357..1b54622 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -48,12 +48,12 @@ class WriterDelegate { virtual void SetPosixFilePermissions(int mode) = 0; }; -// This class is used for reading zip files. A typical use case of this -// class is to scan entries in a zip file and extract them. The code will -// look like: +// This class is used for reading ZIP archives. A typical use case of this class +// is to scan entries in a ZIP archive and extract them. The code will look +// like: // // ZipReader reader; -// reader.Open(zip_file_path); +// reader.Open(zip_path); // while (reader.HasMore()) { // reader.OpenCurrentEntryInZip(); // const base::FilePath& entry_path = @@ -76,68 +76,67 @@ class ZipReader { // of bytes that have been processed so far. using ProgressCallback = base::RepeatingCallback; - // This class represents information of an entry (file or directory) in - // a zip file. - class EntryInfo { - public: - EntryInfo(base::FilePath file_path, - std::string file_path_in_original_encoding, - const unz_file_info& raw_file_info); - - EntryInfo(const EntryInfo&) = delete; - EntryInfo& operator=(const EntryInfo&) = delete; - - // Returns the file path. The path is usually relative like - // "foo/bar.txt", but if it's absolute, is_unsafe() returns true. - const base::FilePath& file_path() const { return file_path_; } - - // Returns the file path in its original encoding as it is stored in the ZIP - // archive. The encoding is not specified here. It might not be UTF-8, and - // the caller needs to use other means to determine the encoding if it wants - // to interpret this file path correctly. - const std::string& file_path_in_original_encoding() const { - return file_path_in_original_encoding_; - } - - // Returns the size of the original file (i.e. after uncompressed). - // Returns 0 if the entry is a directory. - // Note: this value should not be trusted, because it is stored as metadata - // in the zip archive and can be different from the real uncompressed size. - int64_t original_size() const { return original_size_; } - - // Returns the last modified time. If the time stored in the zip file was - // not valid, the unix epoch will be returned. + // Information of an entry (file or directory) in a ZIP archive. + struct Entry { + // Path of this entry, in its original encoding as it is stored in the ZIP + // archive. The encoding is not specified here. It might or might not be + // UTF-8, and the caller needs to use other means to determine the encoding + // if it wants to interpret this path correctly. + std::string path_in_original_encoding; + + // Path of the entry, converted to Unicode. This path is usually relative + // (eg "foo/bar.txt"), but it can also be absolute (eg "/foo/bar.txt") or + // parent-relative (eg "../foo/bar.txt"). See also |is_unsafe|. + base::FilePath path; + + // Size of the original uncompressed file, or 0 if the entry is a directory. + // This value should not be trusted, because it is stored as metadata in the + // ZIP archive and can be different from the real uncompressed size. + int64_t original_size; + + // Last modified time. If the timestamp stored in the ZIP archive is not + // valid, the Unix epoch will be returned. + // + // The timestamp stored in the ZIP archive uses the MS-DOS date and time + // format. // - // The time stored in the zip archive uses the MS-DOS date and time format. // http://msdn.microsoft.com/en-us/library/ms724247(v=vs.85).aspx + // // As such the following limitations apply: - // * only years from 1980 to 2107 can be represented. - // * the time stamp has a 2 second resolution. - // * there's no timezone information, so the time is interpreted as local. - base::Time last_modified() const { return last_modified_; } - - // Returns true if the entry is a directory. - bool is_directory() const { return is_directory_; } - - // Returns true if the entry is unsafe, like having ".." or invalid - // UTF-8 characters in its file name, or the file path is absolute. - bool is_unsafe() const { return is_unsafe_; } - - // Returns true if the entry is encrypted. - bool is_encrypted() const { return is_encrypted_; } - - // Returns the posix file permissions of the entry. - int posix_mode() const { return posix_mode_; } - - private: - base::FilePath file_path_; - std::string file_path_in_original_encoding_; - int64_t original_size_; - base::Time last_modified_; - bool is_directory_; - bool is_unsafe_; - bool is_encrypted_; - int posix_mode_; + // * Only years from 1980 to 2107 can be represented. + // * The timestamp has a 2-second resolution. + // * There is no timezone information, so the time is interpreted as UTC. + base::Time last_modified; + + // True if the entry is a directory. + // False if the entry is a file. + bool is_directory; + + // True if the entry path is considered unsafe, ie if it is absolute or if + // it contains "..". + bool is_unsafe; + + // True if the file content is encrypted. + bool is_encrypted; + + // Entry POSIX permissions (POSIX systems only). + int posix_mode; + }; + + // TODO(crbug.com/1295127) Remove this struct once transition to Entry is + // finished. + struct EntryInfo : Entry { + const Entry& entry() const { return *this; } + const std::string& file_path_in_original_encoding() const { + return entry().path_in_original_encoding; + } + const base::FilePath& file_path() const { return entry().path; } + int64_t original_size() const { return entry().original_size; } + base::Time last_modified() const { return entry().last_modified; } + bool is_directory() const { return entry().is_directory; } + bool is_unsafe() const { return entry().is_unsafe; } + bool is_encrypted() const { return entry().is_encrypted; } + int posix_mode() const { return entry().posix_mode; } }; ZipReader(); @@ -147,11 +146,11 @@ class ZipReader { ~ZipReader(); - // Opens the zip file specified by |zip_file_path|. Returns true on + // Opens the ZIP archive specified by |zip_path|. Returns true on // success. - bool Open(const base::FilePath& zip_file_path); + bool Open(const base::FilePath& zip_path); - // Opens the zip file referred to by the platform file |zip_fd|, without + // Opens the ZIP archive referred to by the platform file |zip_fd|, without // taking ownership of |zip_fd|. Returns true on success. bool OpenFromPlatformFile(base::PlatformFile zip_fd); @@ -160,12 +159,12 @@ class ZipReader { // string until it finishes extracting files. bool OpenFromString(const std::string& data); - // Closes the currently opened zip file. This function is called in the + // Closes the currently opened ZIP archive. This function is called in the // destructor of the class, so you usually don't need to call this. void Close(); - // Sets the encoding of file paths in the ZIP archive. - // By default, file paths are assumed to be in UTF-8. + // Sets the encoding of entry paths in the ZIP archive. + // By default, paths are assumed to be in UTF-8. void SetEncoding(std::string encoding) { encoding_ = std::move(encoding); } // Returns true if there is at least one entry to read. This function is @@ -180,7 +179,7 @@ class ZipReader { // Advances the next entry. Returns true on success. bool AdvanceToNextEntry(); - // Opens the current entry in the zip file. On success, returns true and + // Opens the current entry in the ZIP archive. On success, returns true and // updates the current entry state (i.e. current_entry_info() is updated). // This function should be called before operations over the current entry // like ExtractCurrentEntryToFile(). @@ -224,9 +223,9 @@ class ZipReader { // Returns the current entry info. Returns NULL if the current entry is // not yet opened. OpenCurrentEntryInZip() must be called beforehand. - EntryInfo* current_entry_info() const { return current_entry_info_.get(); } + EntryInfo* current_entry_info() const { return current_entry_; } - // Returns the number of entries in the zip file. + // Returns the number of entries in the ZIP archive. // Open() must be called beforehand. int num_entries() const { return num_entries_; } @@ -249,7 +248,8 @@ class ZipReader { unzFile zip_file_; int num_entries_; bool reached_end_; - std::unique_ptr current_entry_info_; + EntryInfo entry_ = {}; + EntryInfo* current_entry_ = nullptr; base::WeakPtrFactory weak_ptr_factory_{this}; }; diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index f4ecfbd..b0347e8 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -216,8 +216,8 @@ TEST_F(ZipReaderTest, Open_ExistentButNonZipFile) { ASSERT_FALSE(reader.Open(data_dir_.AppendASCII("create_test_zip.sh"))); } -// Iterate through the contents in the test zip file, and compare that the -// contents collected from the zip reader matches the expected contents. +// Iterate through the contents in the test ZIP archive, and compare that the +// contents collected from the ZipReader matches the expected contents. TEST_F(ZipReaderTest, Iteration) { Paths actual_contents; ZipReader reader; @@ -233,8 +233,8 @@ TEST_F(ZipReaderTest, Iteration) { EXPECT_THAT(actual_contents, ElementsAreArray(test_zip_contents_)); } -// Open the test zip file from a file descriptor, iterate through its contents, -// and compare that they match the expected contents. +// Open the test ZIP archive from a file descriptor, iterate through its +// contents, and compare that they match the expected contents. TEST_F(ZipReaderTest, PlatformFileIteration) { Paths actual_contents; ZipReader reader; @@ -293,7 +293,7 @@ TEST_F(ZipReaderTest, current_entry_info_DotDotFile) { TEST_F(ZipReaderTest, current_entry_info_InvalidUTF8File) { ZipReader reader; ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("evil_via_invalid_utf8.zip"))); - // The evil file is the 2nd file in the zip file. + // The evil file is the 2nd file in the ZIP archive. // We cannot locate by the file name ".\x80.\\evil.txt", // as FilePath may internally convert the string. ASSERT_TRUE(reader.AdvanceToNextEntry()); -- cgit v1.2.3 From 9dcc674af38c1ccb5dbe03a77f828a4bfc427676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 9 Feb 2022 10:11:39 +0000 Subject: [zip] Simplify ZipReader::ExtractCurrentEntry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No need to use a dynamically allocated buffer. Also added ERROR logs. BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I99ffcaa24815be1f74164132109f1855aca1be08 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3445923 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#968827} NOKEYCHECK=True GitOrigin-RevId: 123ca7a96860c51ed7bbfce32493920317890bc4 --- google/zip_reader.cc | 79 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index bd36741..1457b45 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -31,9 +31,28 @@ #endif namespace zip { - namespace { +enum UnzipError : int; + +std::ostream& operator<<(std::ostream& out, UnzipError error) { +#define SWITCH_ERR(X) \ + case X: \ + return out << #X; + switch (error) { + SWITCH_ERR(UNZ_OK); + SWITCH_ERR(UNZ_END_OF_LIST_OF_FILE); + SWITCH_ERR(UNZ_ERRNO); + SWITCH_ERR(UNZ_PARAMERROR); + SWITCH_ERR(UNZ_BADZIPFILE); + SWITCH_ERR(UNZ_INTERNALERROR); + SWITCH_ERR(UNZ_CRCERROR); + default: + return out << "UNZ" << static_cast(error); + } +#undef SWITCH_ERR +} + // StringWriterDelegate -------------------------------------------------------- // A writer delegate that writes no more than |max_read_bytes| to a given @@ -186,7 +205,8 @@ bool ZipReader::AdvanceToNextEntry() { reached_end_ = true; } else { DCHECK_LT(current_entry_index + 1, num_entries_); - if (unzGoToNextFile(zip_file_) != UNZ_OK) { + if (const int err = unzGoToNextFile(zip_file_); err != UNZ_OK) { + LOG(ERROR) << "Cannot go to next entry in ZIP: " << UnzipError(err); return false; } } @@ -266,40 +286,47 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, uint64_t num_bytes_to_extract) const { DCHECK(zip_file_); - const int open_result = unzOpenCurrentFile(zip_file_); - if (open_result != UNZ_OK) + if (const int err = unzOpenCurrentFile(zip_file_); err != UNZ_OK) { + LOG(ERROR) << "Cannot open file from ZIP entry: " << UnzipError(err); return false; + } if (!delegate->PrepareOutput()) return false; - std::unique_ptr buf(new char[internal::kZipBufSize]); uint64_t remaining_capacity = num_bytes_to_extract; bool entire_file_extracted = false; while (remaining_capacity > 0) { + char buf[internal::kZipBufSize]; const int num_bytes_read = - unzReadCurrentFile(zip_file_, buf.get(), internal::kZipBufSize); + unzReadCurrentFile(zip_file_, buf, internal::kZipBufSize); if (num_bytes_read == 0) { entire_file_extracted = true; break; - } else if (num_bytes_read < 0) { - // If num_bytes_read < 0, then it's a specific UNZ_* error code. + } + + if (num_bytes_read < 0) { + LOG(ERROR) << "Error " << UnzipError(num_bytes_read) + << " while reading data from file in ZIP"; + break; + } + + DCHECK_LT(0, num_bytes_read); + CHECK_LE(num_bytes_read, internal::kZipBufSize); + + uint64_t num_bytes_to_write = std::min( + remaining_capacity, base::checked_cast(num_bytes_read)); + if (!delegate->WriteBytes(buf, num_bytes_to_write)) break; - } else if (num_bytes_read > 0) { - uint64_t num_bytes_to_write = std::min( - remaining_capacity, base::checked_cast(num_bytes_read)); - if (!delegate->WriteBytes(buf.get(), num_bytes_to_write)) - break; - if (remaining_capacity == base::checked_cast(num_bytes_read)) { - // Ensures function returns true if the entire file has been read. - entire_file_extracted = - (unzReadCurrentFile(zip_file_, buf.get(), 1) == 0); - } - CHECK_GE(remaining_capacity, num_bytes_to_write); - remaining_capacity -= num_bytes_to_write; + + if (remaining_capacity == base::checked_cast(num_bytes_read)) { + // Ensures function returns true if the entire file has been read. + entire_file_extracted = (unzReadCurrentFile(zip_file_, buf, 1) == 0); } + CHECK_GE(remaining_capacity, num_bytes_to_write); + remaining_capacity -= num_bytes_to_write; } unzCloseCurrentFile(zip_file_); @@ -335,8 +362,8 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( return; } - if (unzOpenCurrentFile(zip_file_) != UNZ_OK) { - DVLOG(1) << "Unzip failed: unable to open current zip entry."; + if (const int err = unzOpenCurrentFile(zip_file_); err != UNZ_OK) { + LOG(ERROR) << "Cannot open file from ZIP entry: " << UnzipError(err); base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, std::move(failure_callback)); return; @@ -415,12 +442,16 @@ bool ZipReader::OpenInternal() { DCHECK(zip_file_); unz_global_info zip_info = {}; // Zero-clear. - if (unzGetGlobalInfo(zip_file_, &zip_info) != UNZ_OK) { + if (const int err = unzGetGlobalInfo(zip_file_, &zip_info); err != UNZ_OK) { + LOG(ERROR) << "Cannot get ZIP info: " << UnzipError(err); return false; } + num_entries_ = zip_info.number_entry; - if (num_entries_ < 0) + if (num_entries_ < 0) { + LOG(ERROR) << "Cannot get ZIP info: " << UnzipError(num_entries_); return false; + } // We are already at the end if the ZIP archive is empty. reached_end_ = (num_entries_ == 0); -- cgit v1.2.3 From 9973d8bc57b59a3ce4cec5494d2153b2c8f85974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 9 Feb 2022 21:51:25 +0000 Subject: [zip] Add test ZipReaderTest.Open_EmptyFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: Ia50d5911047ec6347872e62f1f4c4de9650d7394 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3447717 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#969125} NOKEYCHECK=True GitOrigin-RevId: 57f8a7470fe498336643bd0ef8a7d2499b5d280f --- google/test/data/empty.zip | Bin 0 -> 22 bytes google/zip_reader_unittest.cc | 5 +++++ 2 files changed, 5 insertions(+) create mode 100644 google/test/data/empty.zip diff --git a/google/test/data/empty.zip b/google/test/data/empty.zip new file mode 100644 index 0000000..15cb0ec Binary files /dev/null and b/google/test/data/empty.zip differ diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index b0347e8..d2752ee 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -216,6 +216,11 @@ TEST_F(ZipReaderTest, Open_ExistentButNonZipFile) { ASSERT_FALSE(reader.Open(data_dir_.AppendASCII("create_test_zip.sh"))); } +TEST_F(ZipReaderTest, Open_EmptyFile) { + ZipReader reader; + EXPECT_FALSE(reader.Open(data_dir_.AppendASCII("empty.zip"))); +} + // Iterate through the contents in the test ZIP archive, and compare that the // contents collected from the ZipReader matches the expected contents. TEST_F(ZipReaderTest, Iteration) { -- cgit v1.2.3 From 6e2ac2abe8769429111570802caf0a470794f980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 9 Feb 2022 23:22:18 +0000 Subject: [zip] Add ERROR logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I2b72366fe33a2446fbed0a55c48422741379ff53 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3450253 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#969171} NOKEYCHECK=True GitOrigin-RevId: 415ce793d5dafc2678fe7f034a8cb03a6673dc70 --- google/zip_reader.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 1457b45..bc2e01b 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -10,6 +10,7 @@ #include "base/files/file.h" #include "base/i18n/icu_string_conversions.h" #include "base/logging.h" +#include "base/strings/strcat.h" #include "base/strings/string_piece.h" #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" @@ -149,6 +150,10 @@ bool ZipReader::Open(const base::FilePath& zip_path) { // this safely on Linux. See file_util.h for details. zip_file_ = internal::OpenForUnzipping(zip_path.AsUTF8Unsafe()); if (!zip_file_) { + LOG(ERROR) << "Cannot open ZIP archive " + << (LOG_IS_ON(INFO) + ? base::StrCat({"'", zip_path.AsUTF8Unsafe(), "'"}) + : "(redacted)"); return false; } @@ -164,6 +169,7 @@ bool ZipReader::OpenFromPlatformFile(base::PlatformFile zip_fd) { zip_file_ = internal::OpenHandleForUnzipping(zip_fd); #endif if (!zip_file_) { + LOG(ERROR) << "Cannot open ZIP from file handle " << zip_fd; return false; } @@ -224,9 +230,11 @@ bool ZipReader::OpenCurrentEntryInZip() { // Get entry info. unz_file_info info = {}; char path_in_zip[internal::kZipMaxPath] = {}; - if (unzGetCurrentFileInfo(zip_file_, &info, path_in_zip, - sizeof(path_in_zip) - 1, nullptr, 0, nullptr, - 0) != UNZ_OK) { + if (const int err = unzGetCurrentFileInfo(zip_file_, &info, path_in_zip, + sizeof(path_in_zip) - 1, nullptr, 0, + nullptr, 0); + err != UNZ_OK) { + LOG(ERROR) << "Cannot get entry from ZIP: " << UnzipError(err); return false; } -- cgit v1.2.3 From 87b4a54c2a38d9a673831dd43c29ad0e5a48a856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Thu, 10 Feb 2022 01:23:36 +0000 Subject: [zip] Simplify ZipReader::AdvanceToNextEntry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: Ie54cc0843eb0fa6f0bb89f738a09ddb9ecf58ee5 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3450013 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#969222} NOKEYCHECK=True GitOrigin-RevId: 3649bcde642cbaa8c7dfc04977f1a6b154403495 --- google/zip_reader.cc | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index bc2e01b..2e3f877 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -201,17 +201,9 @@ bool ZipReader::AdvanceToNextEntry() { if (reached_end_) return false; - unz_file_pos position = {}; - if (unzGetFilePos(zip_file_, &position) != UNZ_OK) - return false; - const int current_entry_index = position.num_of_file; - // If we are currently at the last entry, then the next position is the - // end of the ZIP archive, so mark that we reached the end. - if (current_entry_index + 1 == num_entries_) { + if (const int err = unzGoToNextFile(zip_file_); err != UNZ_OK) { reached_end_ = true; - } else { - DCHECK_LT(current_entry_index + 1, num_entries_); - if (const int err = unzGoToNextFile(zip_file_); err != UNZ_OK) { + if (err != UNZ_END_OF_LIST_OF_FILE) { LOG(ERROR) << "Cannot go to next entry in ZIP: " << UnzipError(err); return false; } -- cgit v1.2.3 From 14f4303f3d588e8fc3e70624b2bd81bd99724707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Thu, 10 Feb 2022 02:38:25 +0000 Subject: [zip] Simplify ZipReader::OpenInternal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I75d5598be2c413e4974cc3ee57e7850de89f2711 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3449731 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#969256} NOKEYCHECK=True GitOrigin-RevId: fde2706f6c5aa059ba1985211e4b2f5b63b5b1b8 --- google/zip_reader.cc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 2e3f877..e841cab 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -448,13 +448,7 @@ bool ZipReader::OpenInternal() { } num_entries_ = zip_info.number_entry; - if (num_entries_ < 0) { - LOG(ERROR) << "Cannot get ZIP info: " << UnzipError(num_entries_); - return false; - } - - // We are already at the end if the ZIP archive is empty. - reached_end_ = (num_entries_ == 0); + reached_end_ = (num_entries_ <= 0); return true; } -- cgit v1.2.3 From 1a9943c74cb89e6e3d5e15725764f39faf876cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Thu, 10 Feb 2022 22:58:43 +0000 Subject: [zip] Add ZipReader::Next() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This method is not used yet. It will make ZipReader easier to use, in a more idiomatic way with fewer error checking points. It will eventually replace the methods HasMore(), OpenCurrentEntryInZip(), AdvanceToNextEntry() and current_entry_info(). BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: Ibe8732ccb29f628b168c587503a90dc7d66010ba Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3448626 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#969684} NOKEYCHECK=True GitOrigin-RevId: 394d2f7b127a685a82a6323c1549ebdae3b49347 --- google/zip_reader.cc | 24 ++++++++++++++++++++++++ google/zip_reader.h | 50 ++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index e841cab..71f6340 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -205,6 +205,7 @@ bool ZipReader::AdvanceToNextEntry() { reached_end_ = true; if (err != UNZ_END_OF_LIST_OF_FILE) { LOG(ERROR) << "Cannot go to next entry in ZIP: " << UnzipError(err); + ok_ = false; return false; } } @@ -214,6 +215,26 @@ bool ZipReader::AdvanceToNextEntry() { return true; } +const ZipReader::Entry* ZipReader::Next() { + DCHECK(zip_file_); + + if (reached_end_) + return nullptr; + + if (next_index_ > 0 && (!AdvanceToNextEntry() || reached_end_)) + return nullptr; + + next_index_++; + + if (!OpenCurrentEntryInZip()) { + reached_end_ = true; + ok_ = false; + return nullptr; + } + + return &entry_; +} + bool ZipReader::OpenCurrentEntryInZip() { DCHECK(zip_file_); @@ -449,13 +470,16 @@ bool ZipReader::OpenInternal() { num_entries_ = zip_info.number_entry; reached_end_ = (num_entries_ <= 0); + ok_ = true; return true; } void ZipReader::Reset() { zip_file_ = nullptr; num_entries_ = 0; + next_index_ = 0; reached_end_ = false; + ok_ = false; entry_ = {}; current_entry_ = nullptr; } diff --git a/google/zip_reader.h b/google/zip_reader.h index 1b54622..b66dc79 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -53,18 +53,24 @@ class WriterDelegate { // like: // // ZipReader reader; -// reader.Open(zip_path); -// while (reader.HasMore()) { -// reader.OpenCurrentEntryInZip(); -// const base::FilePath& entry_path = -// reader.current_entry_info()->file_path(); -// auto writer = CreateFilePathWriterDelegate(extract_dir, entry_path); -// reader.ExtractCurrentEntry(writer, std::numeric_limits::max()); -// reader.AdvanceToNextEntry(); +// if (!reader.Open(zip_path)) { +// // Cannot open +// return; // } // -// For simplicity, error checking is omitted in the example code above. The -// production code should check return values from all of these functions. +// while (const ZipReader::entry* entry = reader.Next()) { +// auto writer = CreateFilePathWriterDelegate(extract_dir, entry->path); +// if (!reader.ExtractCurrentEntry( +// writer, std::numeric_limits::max())) { +// // Cannot extract +// return; +// } +// } +// +// if (!reader.ok()) { +// // Error while enumerating entries +// return; +// } // class ZipReader { public: @@ -167,6 +173,20 @@ class ZipReader { // By default, paths are assumed to be in UTF-8. void SetEncoding(std::string encoding) { encoding_ = std::move(encoding); } + // Gets the next entry. Returns null if there is no more entry. The returned + // Entry is owned by this ZipReader, and is valid until Next() is called + // again or until this ZipReader is closed. + // + // This function is used to scan entries: + // while (const ZipReader::Entry* entry = reader.Next()) { + // // Do something with the current entry here. + // ... + // } + const Entry* Next(); + + // Returns true if the enumeration of entries was successful. + bool ok() const { return ok_; } + // Returns true if there is at least one entry to read. This function is // used to scan entries with AdvanceToNextEntry(), like: // @@ -174,9 +194,13 @@ class ZipReader { // // Do something with the current file here. // reader.AdvanceToNextEntry(); // } + // + // TODO(crbug.com/1295127) Remove this method. bool HasMore(); // Advances the next entry. Returns true on success. + // + // TODO(crbug.com/1295127) Remove this method. bool AdvanceToNextEntry(); // Opens the current entry in the ZIP archive. On success, returns true and @@ -186,6 +210,8 @@ class ZipReader { // // Note that there is no CloseCurrentEntryInZip(). The current entry state is // reset automatically as needed. + // + // TODO(crbug.com/1295127) Remove this method. bool OpenCurrentEntryInZip(); // Extracts |num_bytes_to_extract| bytes of the current entry to |delegate|, @@ -223,6 +249,8 @@ class ZipReader { // Returns the current entry info. Returns NULL if the current entry is // not yet opened. OpenCurrentEntryInZip() must be called beforehand. + // + // TODO(crbug.com/1295127) Remove this method. EntryInfo* current_entry_info() const { return current_entry_; } // Returns the number of entries in the ZIP archive. @@ -247,7 +275,9 @@ class ZipReader { std::string encoding_; unzFile zip_file_; int num_entries_; + int next_index_; bool reached_end_; + bool ok_; EntryInfo entry_ = {}; EntryInfo* current_entry_ = nullptr; -- cgit v1.2.3 From 96f8062041a40851d5ce62844465e3e8a44cbb69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Fri, 11 Feb 2022 00:06:17 +0000 Subject: [zip] Use ZipReader::Next() in UnzipWithFilterAndWriters() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactored UnzipWithFilterAndWriters() to use ZipReader::Next(). BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: Ibf57b4308eadaa31c3ea19a208220460ecef8743 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3450067 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#969701} NOKEYCHECK=True GitOrigin-RevId: 754cf4a440eb3b144444a9bb16a9a74ddf7e03c2 --- google/zip.cc | 49 ++++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index 7c46718..790969e 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -198,41 +198,40 @@ bool UnzipWithFilterAndWriters(const base::PlatformFile& src_file, bool log_skipped_files) { ZipReader reader; if (!reader.OpenFromPlatformFile(src_file)) { - DLOG(WARNING) << "Cannot open '" << src_file << "'"; + DLOG(WARNING) << "Cannot open ZIP from file handle " << src_file; return false; } - while (reader.HasMore()) { - if (!reader.OpenCurrentEntryInZip()) { - DLOG(WARNING) << "Failed to open the current file in zip"; + + while (const ZipReader::Entry* const entry = reader.Next()) { + if (entry->is_unsafe) { + DLOG(WARNING) << "Found unsafe entry in ZIP: " << entry->path; return false; } - const base::FilePath& entry_path = reader.current_entry_info()->file_path(); - if (reader.current_entry_info()->is_unsafe()) { - DLOG(WARNING) << "Found an unsafe file in zip " << entry_path; - return false; + + if (filter_cb && !filter_cb.Run(entry->path)) { + DLOG_IF(WARNING, log_skipped_files) + << "Skipped ZIP entry " << entry->path; + continue; } - if (filter_cb.Run(entry_path)) { - if (reader.current_entry_info()->is_directory()) { - if (!directory_creator.Run(entry_path)) - return false; - } else { - std::unique_ptr writer = writer_factory.Run(entry_path); - if (!reader.ExtractCurrentEntry(writer.get(), - std::numeric_limits::max())) { - DLOG(WARNING) << "Failed to extract " << entry_path; - return false; - } - } - } else if (log_skipped_files) { - DLOG(WARNING) << "Skipped file " << entry_path; + + if (entry->is_directory) { + // It's a directory. + if (!directory_creator.Run(entry->path)) + return false; + + continue; } - if (!reader.AdvanceToNextEntry()) { - DLOG(WARNING) << "Failed to advance to the next file"; + // It's a file. + std::unique_ptr writer = writer_factory.Run(entry->path); + if (!reader.ExtractCurrentEntry(writer.get(), + std::numeric_limits::max())) { + DLOG(WARNING) << "Cannot extract " << entry->path; return false; } } - return true; + + return reader.ok(); } bool ZipWithFilterCallback(const base::FilePath& src_dir, -- cgit v1.2.3 From dd9a1330e5aeb2f194d711746a03e01391d6f17c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Fri, 11 Feb 2022 02:11:52 +0000 Subject: [zip] Use ZipReader::Next() in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: Ic1048c46013190bf251f62b7e6dd82c065ea8298 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3450064 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#969788} NOKEYCHECK=True GitOrigin-RevId: 013178d0b5ccd034179d4da78129a225bb4fa359 --- google/zip_reader_unittest.cc | 181 +++++++++++++++++++++++------------------- google/zip_unittest.cc | 12 ++- 2 files changed, 104 insertions(+), 89 deletions(-) diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index d2752ee..f5e184e 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -37,6 +37,7 @@ using ::testing::_; using ::testing::ElementsAre; using ::testing::ElementsAreArray; using ::testing::Return; +using ::testing::SizeIs; namespace { @@ -118,18 +119,22 @@ bool ExtractCurrentEntryToFilePath(zip::ZipReader* reader, std::numeric_limits::max()); } -bool LocateAndOpenEntry(zip::ZipReader* reader, - const base::FilePath& path_in_zip) { +const zip::ZipReader::Entry* LocateAndOpenEntry( + zip::ZipReader* const reader, + const base::FilePath& path_in_zip) { + DCHECK(reader); + EXPECT_TRUE(reader->ok()); + // The underlying library can do O(1) access, but ZipReader does not expose // that. O(N) access is acceptable for these tests. - while (reader->HasMore()) { - if (!reader->OpenCurrentEntryInZip()) - return false; - if (reader->current_entry_info()->file_path() == path_in_zip) - return true; - reader->AdvanceToNextEntry(); + while (const zip::ZipReader::Entry* const entry = reader->Next()) { + EXPECT_TRUE(reader->ok()); + if (entry->path == path_in_zip) + return entry; } - return false; + + EXPECT_TRUE(reader->ok()); + return nullptr; } using Paths = std::vector; @@ -166,11 +171,12 @@ class ZipReaderTest : public PlatformTest { if (!encoding.empty()) reader.SetEncoding(std::string(encoding)); - while (reader.HasMore()) { - if (reader.OpenCurrentEntryInZip()) - paths.push_back(reader.current_entry_info()->file_path()); - reader.AdvanceToNextEntry(); + while (const ZipReader::Entry* const entry = reader.Next()) { + EXPECT_TRUE(reader.ok()); + paths.push_back(entry->path); } + + EXPECT_TRUE(reader.ok()); } return paths; @@ -197,28 +203,37 @@ class ZipReaderTest : public PlatformTest { TEST_F(ZipReaderTest, Open_ValidZipFile) { ZipReader reader; - ASSERT_TRUE(reader.Open(test_zip_file_)); + EXPECT_TRUE(reader.Open(test_zip_file_)); + EXPECT_TRUE(reader.ok()); } TEST_F(ZipReaderTest, Open_ValidZipPlatformFile) { ZipReader reader; + EXPECT_FALSE(reader.ok()); FileWrapper zip_fd_wrapper(test_zip_file_, FileWrapper::READ_ONLY); - ASSERT_TRUE(reader.OpenFromPlatformFile(zip_fd_wrapper.platform_file())); + EXPECT_TRUE(reader.OpenFromPlatformFile(zip_fd_wrapper.platform_file())); + EXPECT_TRUE(reader.ok()); } TEST_F(ZipReaderTest, Open_NonExistentFile) { ZipReader reader; - ASSERT_FALSE(reader.Open(data_dir_.AppendASCII("nonexistent.zip"))); + EXPECT_FALSE(reader.ok()); + EXPECT_FALSE(reader.Open(data_dir_.AppendASCII("nonexistent.zip"))); + EXPECT_FALSE(reader.ok()); } TEST_F(ZipReaderTest, Open_ExistentButNonZipFile) { ZipReader reader; - ASSERT_FALSE(reader.Open(data_dir_.AppendASCII("create_test_zip.sh"))); + EXPECT_FALSE(reader.ok()); + EXPECT_FALSE(reader.Open(data_dir_.AppendASCII("create_test_zip.sh"))); + EXPECT_FALSE(reader.ok()); } TEST_F(ZipReaderTest, Open_EmptyFile) { ZipReader reader; + EXPECT_FALSE(reader.ok()); EXPECT_FALSE(reader.Open(data_dir_.AppendASCII("empty.zip"))); + EXPECT_FALSE(reader.ok()); } // Iterate through the contents in the test ZIP archive, and compare that the @@ -226,15 +241,19 @@ TEST_F(ZipReaderTest, Open_EmptyFile) { TEST_F(ZipReaderTest, Iteration) { Paths actual_contents; ZipReader reader; - ASSERT_TRUE(reader.Open(test_zip_file_)); - while (reader.HasMore()) { - ASSERT_TRUE(reader.OpenCurrentEntryInZip()); - actual_contents.push_back(reader.current_entry_info()->file_path()); - ASSERT_TRUE(reader.AdvanceToNextEntry()); + EXPECT_FALSE(reader.ok()); + EXPECT_TRUE(reader.Open(test_zip_file_)); + EXPECT_TRUE(reader.ok()); + while (const ZipReader::Entry* const entry = reader.Next()) { + EXPECT_TRUE(reader.ok()); + actual_contents.push_back(entry->path); } - EXPECT_FALSE(reader.AdvanceToNextEntry()); // Shouldn't go further. - EXPECT_EQ(test_zip_contents_.size(), - static_cast(reader.num_entries())); + + EXPECT_TRUE(reader.ok()); + EXPECT_FALSE(reader.Next()); // Shouldn't go further. + EXPECT_TRUE(reader.ok()); + + EXPECT_THAT(actual_contents, SizeIs(reader.num_entries())); EXPECT_THAT(actual_contents, ElementsAreArray(test_zip_contents_)); } @@ -244,31 +263,35 @@ TEST_F(ZipReaderTest, PlatformFileIteration) { Paths actual_contents; ZipReader reader; FileWrapper zip_fd_wrapper(test_zip_file_, FileWrapper::READ_ONLY); - ASSERT_TRUE(reader.OpenFromPlatformFile(zip_fd_wrapper.platform_file())); - while (reader.HasMore()) { - ASSERT_TRUE(reader.OpenCurrentEntryInZip()); - actual_contents.push_back(reader.current_entry_info()->file_path()); - ASSERT_TRUE(reader.AdvanceToNextEntry()); + EXPECT_TRUE(reader.OpenFromPlatformFile(zip_fd_wrapper.platform_file())); + EXPECT_TRUE(reader.ok()); + while (const ZipReader::Entry* const entry = reader.Next()) { + EXPECT_TRUE(reader.ok()); + actual_contents.push_back(entry->path); } - EXPECT_FALSE(reader.AdvanceToNextEntry()); // Shouldn't go further. - EXPECT_EQ(test_zip_contents_.size(), - static_cast(reader.num_entries())); + + EXPECT_TRUE(reader.ok()); + EXPECT_FALSE(reader.Next()); // Shouldn't go further. + EXPECT_TRUE(reader.ok()); + + EXPECT_THAT(actual_contents, SizeIs(reader.num_entries())); EXPECT_THAT(actual_contents, ElementsAreArray(test_zip_contents_)); } -TEST_F(ZipReaderTest, current_entry_info_RegularFile) { +TEST_F(ZipReaderTest, RegularFile) { ZipReader reader; ASSERT_TRUE(reader.Open(test_zip_file_)); base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); - ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); - ZipReader::EntryInfo* current_entry_info = reader.current_entry_info(); - EXPECT_EQ(target_path, current_entry_info->file_path()); - EXPECT_EQ(13527, current_entry_info->original_size()); + const ZipReader::Entry* entry = LocateAndOpenEntry(&reader, target_path); + ASSERT_TRUE(entry); + + EXPECT_EQ(target_path, entry->path); + EXPECT_EQ(13527, entry->original_size); // The expected time stamp: 2009-05-29 06:22:20 base::Time::Exploded exploded = {}; // Zero-clear. - current_entry_info->last_modified().UTCExplode(&exploded); + entry->last_modified.UTCExplode(&exploded); EXPECT_EQ(2009, exploded.year); EXPECT_EQ(5, exploded.month); EXPECT_EQ(29, exploded.day_of_month); @@ -277,39 +300,32 @@ TEST_F(ZipReaderTest, current_entry_info_RegularFile) { EXPECT_EQ(20, exploded.second); EXPECT_EQ(0, exploded.millisecond); - EXPECT_FALSE(current_entry_info->is_unsafe()); - EXPECT_FALSE(current_entry_info->is_directory()); + EXPECT_FALSE(entry->is_unsafe); + EXPECT_FALSE(entry->is_directory); } -TEST_F(ZipReaderTest, current_entry_info_DotDotFile) { +TEST_F(ZipReaderTest, DotDotFile) { ZipReader reader; ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("evil.zip"))); base::FilePath target_path(FILE_PATH_LITERAL( "../levilevilevilevilevilevilevilevilevilevilevilevil")); - ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); - ZipReader::EntryInfo* current_entry_info = reader.current_entry_info(); - EXPECT_EQ(target_path, current_entry_info->file_path()); - + const ZipReader::Entry* entry = LocateAndOpenEntry(&reader, target_path); + ASSERT_TRUE(entry); + EXPECT_EQ(target_path, entry->path); // This file is unsafe because of ".." in the file name. - EXPECT_TRUE(current_entry_info->is_unsafe()); - EXPECT_FALSE(current_entry_info->is_directory()); + EXPECT_TRUE(entry->is_unsafe); + EXPECT_FALSE(entry->is_directory); } -TEST_F(ZipReaderTest, current_entry_info_InvalidUTF8File) { +TEST_F(ZipReaderTest, InvalidUTF8File) { ZipReader reader; ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("evil_via_invalid_utf8.zip"))); - // The evil file is the 2nd file in the ZIP archive. - // We cannot locate by the file name ".\x80.\\evil.txt", - // as FilePath may internally convert the string. - ASSERT_TRUE(reader.AdvanceToNextEntry()); - ASSERT_TRUE(reader.OpenCurrentEntryInZip()); - const ZipReader::EntryInfo* const entry = reader.current_entry_info(); - + base::FilePath target_path = base::FilePath::FromUTF8Unsafe(".�.\\evil.txt"); + const ZipReader::Entry* entry = LocateAndOpenEntry(&reader, target_path); ASSERT_TRUE(entry); - EXPECT_FALSE(entry->is_unsafe()); - EXPECT_FALSE(entry->is_directory()); - EXPECT_EQ(entry->file_path(), - base::FilePath::FromUTF8Unsafe(".�.\\evil.txt")); + EXPECT_EQ(target_path, entry->path); + EXPECT_FALSE(entry->is_unsafe); + EXPECT_FALSE(entry->is_directory); } // By default, file paths in ZIPs are interpreted as UTF-8. But in this test, @@ -360,35 +376,32 @@ TEST_F(ZipReaderTest, EncodingSjis) { "新しいフォルダ/新しいテキスト ドキュメント.txt"))); } -TEST_F(ZipReaderTest, current_entry_info_AbsoluteFile) { +TEST_F(ZipReaderTest, AbsoluteFile) { ZipReader reader; ASSERT_TRUE( reader.Open(data_dir_.AppendASCII("evil_via_absolute_file_name.zip"))); base::FilePath target_path(FILE_PATH_LITERAL("/evil.txt")); - ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); - ZipReader::EntryInfo* current_entry_info = reader.current_entry_info(); - EXPECT_EQ(target_path, current_entry_info->file_path()); - + const ZipReader::Entry* entry = LocateAndOpenEntry(&reader, target_path); + ASSERT_TRUE(entry); + EXPECT_EQ(target_path, entry->path); // This file is unsafe because of the absolute file name. - EXPECT_TRUE(current_entry_info->is_unsafe()); - EXPECT_FALSE(current_entry_info->is_directory()); + EXPECT_TRUE(entry->is_unsafe); + EXPECT_FALSE(entry->is_directory); } -TEST_F(ZipReaderTest, current_entry_info_Directory) { +TEST_F(ZipReaderTest, Directory) { ZipReader reader; ASSERT_TRUE(reader.Open(test_zip_file_)); base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/")); - ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); - ZipReader::EntryInfo* current_entry_info = reader.current_entry_info(); - - EXPECT_EQ(base::FilePath(FILE_PATH_LITERAL("foo/bar/")), - current_entry_info->file_path()); + const ZipReader::Entry* entry = LocateAndOpenEntry(&reader, target_path); + ASSERT_TRUE(entry); + EXPECT_EQ(target_path, entry->path); // The directory size should be zero. - EXPECT_EQ(0, current_entry_info->original_size()); + EXPECT_EQ(0, entry->original_size); // The expected time stamp: 2009-05-31 15:49:52 base::Time::Exploded exploded = {}; // Zero-clear. - current_entry_info->last_modified().UTCExplode(&exploded); + entry->last_modified.UTCExplode(&exploded); EXPECT_EQ(2009, exploded.year); EXPECT_EQ(5, exploded.month); EXPECT_EQ(31, exploded.day_of_month); @@ -397,22 +410,26 @@ TEST_F(ZipReaderTest, current_entry_info_Directory) { EXPECT_EQ(52, exploded.second); EXPECT_EQ(0, exploded.millisecond); - EXPECT_FALSE(current_entry_info->is_unsafe()); - EXPECT_TRUE(current_entry_info->is_directory()); + EXPECT_FALSE(entry->is_unsafe); + EXPECT_TRUE(entry->is_directory); } -TEST_F(ZipReaderTest, current_entry_info_EncryptedFile) { +TEST_F(ZipReaderTest, EncryptedFile) { ZipReader reader; base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("test_encrypted.zip"))); - ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); - EXPECT_TRUE(reader.current_entry_info()->is_encrypted()); + const ZipReader::Entry* entry = LocateAndOpenEntry(&reader, target_path); + ASSERT_TRUE(entry); + EXPECT_EQ(target_path, entry->path); + EXPECT_TRUE(entry->is_encrypted); reader.Close(); ASSERT_TRUE(reader.Open(test_zip_file_)); - ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); - EXPECT_FALSE(reader.current_entry_info()->is_encrypted()); + entry = LocateAndOpenEntry(&reader, target_path); + ASSERT_TRUE(entry); + EXPECT_EQ(target_path, entry->path); + EXPECT_FALSE(entry->is_encrypted); } // Verifies that the ZipReader class can extract a file from a zip archive diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 53f76eb..a8fdfc6 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -212,7 +212,7 @@ class ZipTest : public PlatformTest { } void TestUnzipFile(const base::FilePath& path, bool expect_hidden_files) { - ASSERT_TRUE(base::PathExists(path)) << "no file " << path.value(); + ASSERT_TRUE(base::PathExists(path)) << "no file " << path; ASSERT_TRUE(zip::Unzip(path, test_dir_)); base::FilePath original_dir; @@ -226,7 +226,7 @@ class ZipTest : public PlatformTest { size_t count = 0; while (!unzipped_entry_path.empty()) { EXPECT_EQ(zip_contents_.count(unzipped_entry_path), 1U) - << "Couldn't find " << unzipped_entry_path.value(); + << "Couldn't find " << unzipped_entry_path; count++; if (base::PathExists(unzipped_entry_path) && @@ -511,11 +511,9 @@ TEST_F(ZipTest, ZipFiles) { EXPECT_TRUE(reader.Open(zip_name)); EXPECT_EQ(zip_file_list_.size(), static_cast(reader.num_entries())); for (size_t i = 0; i < zip_file_list_.size(); ++i) { - EXPECT_TRUE(reader.HasMore()); - EXPECT_TRUE(reader.OpenCurrentEntryInZip()); - const zip::ZipReader::EntryInfo* entry_info = reader.current_entry_info(); - EXPECT_EQ(entry_info->file_path(), zip_file_list_[i]); - reader.AdvanceToNextEntry(); + const zip::ZipReader::Entry* const entry = reader.Next(); + ASSERT_TRUE(entry); + EXPECT_EQ(entry->path, zip_file_list_[i]); } } #endif // defined(OS_POSIX) -- cgit v1.2.3 From e798a8d0730b0f2cad2dc1ae08d91f9793117026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Sat, 12 Feb 2022 19:32:11 +0000 Subject: [zip] Add log messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I08f8c306beaa56ffb3a5424f23dfe3d000cf7229 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3458044 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#970340} NOKEYCHECK=True GitOrigin-RevId: 43d9ed269917c0b59d93cb33b2d51193635d4e98 --- google/zip_reader.cc | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 71f6340..a40ecd8 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -54,6 +54,15 @@ std::ostream& operator<<(std::ostream& out, UnzipError error) { #undef SWITCH_ERR } +struct Redact { + explicit Redact(const base::FilePath& path) : path(path) {} + const base::FilePath& path; +}; + +std::ostream& operator<<(std::ostream& out, Redact r) { + return LOG_IS_ON(INFO) ? out << "'" << r.path << "'" : out << "(redacted)"; +} + // StringWriterDelegate -------------------------------------------------------- // A writer delegate that writes no more than |max_read_bytes| to a given @@ -150,10 +159,7 @@ bool ZipReader::Open(const base::FilePath& zip_path) { // this safely on Linux. See file_util.h for details. zip_file_ = internal::OpenForUnzipping(zip_path.AsUTF8Unsafe()); if (!zip_file_) { - LOG(ERROR) << "Cannot open ZIP archive " - << (LOG_IS_ON(INFO) - ? base::StrCat({"'", zip_path.AsUTF8Unsafe(), "'"}) - : "(redacted)"); + LOG(ERROR) << "Cannot open ZIP archive " << Redact(zip_path); return false; } @@ -308,7 +314,8 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, DCHECK(zip_file_); if (const int err = unzOpenCurrentFile(zip_file_); err != UNZ_OK) { - LOG(ERROR) << "Cannot open file from ZIP entry: " << UnzipError(err); + LOG(ERROR) << "Cannot open file " << Redact(entry_.path) + << " from ZIP: " << UnzipError(err); return false; } @@ -329,8 +336,8 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, } if (num_bytes_read < 0) { - LOG(ERROR) << "Error " << UnzipError(num_bytes_read) - << " while reading data from file in ZIP"; + LOG(ERROR) << "Cannot read file " << Redact(entry_.path) + << " from ZIP: " << UnzipError(num_bytes_read); break; } @@ -344,8 +351,12 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, if (remaining_capacity == base::checked_cast(num_bytes_read)) { // Ensures function returns true if the entire file has been read. - entire_file_extracted = (unzReadCurrentFile(zip_file_, buf, 1) == 0); + const int n = unzReadCurrentFile(zip_file_, buf, 1); + entire_file_extracted = (n == 0); + LOG_IF(ERROR, n < 0) << "Cannot read file " << Redact(entry_.path) + << " from ZIP: " << UnzipError(n); } + CHECK_GE(remaining_capacity, num_bytes_to_write); remaining_capacity -= num_bytes_to_write; } @@ -376,7 +387,7 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, std::move(success_callback)); } else { - DVLOG(1) << "Unzip failed: unable to create directory."; + LOG(ERROR) << "Cannot create directory " << Redact(output_file_path); base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, std::move(failure_callback)); } @@ -384,7 +395,8 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( } if (const int err = unzOpenCurrentFile(zip_file_); err != UNZ_OK) { - LOG(ERROR) << "Cannot open file from ZIP entry: " << UnzipError(err); + LOG(ERROR) << "Cannot open file " << Redact(entry_.path) + << " from ZIP: " << UnzipError(err); base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, std::move(failure_callback)); return; @@ -392,7 +404,7 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( base::FilePath output_dir_path = output_file_path.DirName(); if (!base::CreateDirectory(output_dir_path)) { - DVLOG(1) << "Unzip failed: unable to create containing directory."; + LOG(ERROR) << "Cannot create directory " << Redact(output_dir_path); base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, std::move(failure_callback)); return; @@ -402,8 +414,7 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( base::File output_file(output_file_path, flags); if (!output_file.IsValid()) { - DVLOG(1) << "Unzip failed: unable to create platform file at " - << output_file_path.value(); + LOG(ERROR) << "Cannot create file " << Redact(output_file_path); base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, std::move(failure_callback)); return; @@ -498,12 +509,13 @@ void ZipReader::ExtractChunk(base::File output_file, unzCloseCurrentFile(zip_file_); std::move(success_callback).Run(); } else if (num_bytes_read < 0) { - DVLOG(1) << "Unzip failed: error while reading zipfile " - << "(" << num_bytes_read << ")"; + LOG(ERROR) << "Cannot read file " << Redact(entry_.path) + << " from ZIP: " << UnzipError(num_bytes_read); std::move(failure_callback).Run(); } else { if (num_bytes_read != output_file.Write(offset, buffer, num_bytes_read)) { - DVLOG(1) << "Unzip failed: unable to write all bytes to target."; + LOG(ERROR) << "Cannot write " << num_bytes_read + << " bytes to file at offset " << offset; std::move(failure_callback).Run(); return; } -- cgit v1.2.3 From d8f49e6ae1efe8618a088f0c872df2b1edf16436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Sat, 12 Feb 2022 23:59:41 +0000 Subject: [zip] Check CRC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ZipReader's Extract* methods check the file CRC, and report any error accordingly. Added tests: ZipReaderTest.WrongCrc ZipReaderTest.ExtractToFileAsync_WrongCrc BUG=chromium:1296826 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: Id3b253141601d14b55f4cbdfb4b0e0e35df7dbc3 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3458662 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#970364} NOKEYCHECK=True GitOrigin-RevId: 07568a33cc5475e7cf5e36c807e1b18fe586eb4f --- google/test/data/Wrong CRC.zip | Bin 0 -> 231 bytes google/zip_reader.cc | 54 +++++++++++++++++++++++++---------------- google/zip_reader_unittest.cc | 47 +++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 google/test/data/Wrong CRC.zip diff --git a/google/test/data/Wrong CRC.zip b/google/test/data/Wrong CRC.zip new file mode 100644 index 0000000..ee9a1ef Binary files /dev/null and b/google/test/data/Wrong CRC.zip differ diff --git a/google/zip_reader.cc b/google/zip_reader.cc index a40ecd8..039b887 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -361,8 +361,6 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, remaining_capacity -= num_bytes_to_write; } - unzCloseCurrentFile(zip_file_); - if (entire_file_extracted) { delegate->SetPosixFilePermissions(current_entry_info()->posix_mode()); if (current_entry_info()->last_modified() != base::Time::UnixEpoch()) { @@ -370,6 +368,12 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, } } + if (const int err = unzCloseCurrentFile(zip_file_); err != UNZ_OK) { + LOG(ERROR) << "Cannot extract file " << Redact(entry_.path) + << " from ZIP: " << UnzipError(err); + return false; + } + return entire_file_extracted; } @@ -499,38 +503,46 @@ void ZipReader::ExtractChunk(base::File output_file, SuccessCallback success_callback, FailureCallback failure_callback, const ProgressCallback& progress_callback, - const int64_t offset) { + int64_t offset) { char buffer[internal::kZipBufSize]; const int num_bytes_read = unzReadCurrentFile(zip_file_, buffer, internal::kZipBufSize); if (num_bytes_read == 0) { - unzCloseCurrentFile(zip_file_); - std::move(success_callback).Run(); - } else if (num_bytes_read < 0) { - LOG(ERROR) << "Cannot read file " << Redact(entry_.path) - << " from ZIP: " << UnzipError(num_bytes_read); - std::move(failure_callback).Run(); - } else { - if (num_bytes_read != output_file.Write(offset, buffer, num_bytes_read)) { - LOG(ERROR) << "Cannot write " << num_bytes_read - << " bytes to file at offset " << offset; + if (const int err = unzCloseCurrentFile(zip_file_); err != UNZ_OK) { + LOG(ERROR) << "Cannot extract file " << Redact(entry_.path) + << " from ZIP: " << UnzipError(err); std::move(failure_callback).Run(); return; } - int64_t current_progress = offset + num_bytes_read; + std::move(success_callback).Run(); + return; + } - progress_callback.Run(current_progress); + if (num_bytes_read < 0) { + LOG(ERROR) << "Cannot read file " << Redact(entry_.path) + << " from ZIP: " << UnzipError(num_bytes_read); + std::move(failure_callback).Run(); + return; + } - base::SequencedTaskRunnerHandle::Get()->PostTask( - FROM_HERE, - base::BindOnce(&ZipReader::ExtractChunk, weak_ptr_factory_.GetWeakPtr(), - std::move(output_file), std::move(success_callback), - std::move(failure_callback), progress_callback, - current_progress)); + if (num_bytes_read != output_file.Write(offset, buffer, num_bytes_read)) { + LOG(ERROR) << "Cannot write " << num_bytes_read + << " bytes to file at offset " << offset; + std::move(failure_callback).Run(); + return; } + + offset += num_bytes_read; + progress_callback.Run(offset); + + base::SequencedTaskRunnerHandle::Get()->PostTask( + FROM_HERE, + base::BindOnce(&ZipReader::ExtractChunk, weak_ptr_factory_.GetWeakPtr(), + std::move(output_file), std::move(success_callback), + std::move(failure_callback), progress_callback, offset)); } // FileWriterDelegate ---------------------------------------------------------- diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index f5e184e..4a41896 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -504,6 +504,41 @@ TEST_F(ZipReaderTest, ExtractToFileAsync_RegularFile) { EXPECT_EQ(file_size, listener.current_progress()); } +TEST_F(ZipReaderTest, ExtractToFileAsync_WrongCrc) { + MockUnzipListener listener; + + ZipReader reader; + ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("Wrong CRC.zip"))); + ASSERT_TRUE( + LocateAndOpenEntry(&reader, base::FilePath::FromASCII("Corrupted.txt"))); + const base::FilePath target_path = test_dir_.AppendASCII("extracted"); + reader.ExtractCurrentEntryToFilePathAsync( + target_path, + base::BindOnce(&MockUnzipListener::OnUnzipSuccess, listener.AsWeakPtr()), + base::BindOnce(&MockUnzipListener::OnUnzipFailure, listener.AsWeakPtr()), + base::BindRepeating(&MockUnzipListener::OnUnzipProgress, + listener.AsWeakPtr())); + + EXPECT_EQ(0, listener.success_calls()); + EXPECT_EQ(0, listener.failure_calls()); + EXPECT_EQ(0, listener.progress_calls()); + + base::RunLoop().RunUntilIdle(); + + EXPECT_EQ(0, listener.success_calls()); + EXPECT_EQ(1, listener.failure_calls()); + EXPECT_LE(1, listener.progress_calls()); + + std::string contents; + ASSERT_TRUE(base::ReadFileToString(target_path, &contents)); + EXPECT_EQ("This file has been changed after its CRC was computed.\n", + contents); + + int64_t file_size = 0; + ASSERT_TRUE(base::GetFileSize(target_path, &file_size)); + EXPECT_EQ(file_size, listener.current_progress()); +} + // Verifies that the asynchronous extraction to a file works. TEST_F(ZipReaderTest, ExtractToFileAsync_Directory) { MockUnzipListener listener; @@ -703,6 +738,18 @@ TEST_F(ZipReaderTest, ExtractCurrentEntrySuccess) { std::numeric_limits::max())); } +TEST_F(ZipReaderTest, WrongCrc) { + ZipReader reader; + ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("Wrong CRC.zip"))); + + const ZipReader::Entry* const entry = + LocateAndOpenEntry(&reader, base::FilePath::FromASCII("Corrupted.txt")); + ASSERT_TRUE(entry); + std::string contents = "dummy"; + EXPECT_FALSE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_EQ("", contents); +} + class FileWriterDelegateTest : public ::testing::Test { protected: void SetUp() override { -- cgit v1.2.3 From 03f3212b985a0fb224f9afaca68a7d97ee74eab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 14 Feb 2022 07:46:26 +0000 Subject: [zip] Add ZipReader::SetPassword() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ZipReader can now decrypt encrypted files from ZIP archives. Use password only for encrypted files. Using a non-null password triggers an error for non-encrypted files, even if this password is an empty string. Note that only the legacy ZipCrypto encryption scheme is currently supported. The underlying minizip library does not support the more modern AES encryption schemes. Added tests: ZipReaderTest.EncryptedFile_WrongPassword ZipReaderTest.EncryptedFile_RightPassword ZipReaderTest.ExtractToFileAsync_Encrypted_NoPassword ZipReaderTest.ExtractToFileAsync_Encrypted_RightPassword BUG=chromium:1296838, chromium:953256 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I01ee66e9c5db2181de565fdf1fa0081b6dd4aa7d Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3458783 Reviewed-by: Noel Gordon Commit-Queue: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#970508} NOKEYCHECK=True GitOrigin-RevId: 4e9e01a391947244e4082302c1739807950c6e71 --- google/test/data/Different Encryptions.zip | Bin 0 -> 1083 bytes google/zip_reader.cc | 14 ++- google/zip_reader.h | 5 + google/zip_reader_unittest.cc | 157 ++++++++++++++++++++++++++--- 4 files changed, 161 insertions(+), 15 deletions(-) create mode 100644 google/test/data/Different Encryptions.zip diff --git a/google/test/data/Different Encryptions.zip b/google/test/data/Different Encryptions.zip new file mode 100644 index 0000000..858e7c7 Binary files /dev/null and b/google/test/data/Different Encryptions.zip differ diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 039b887..9ed9d07 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -313,7 +313,12 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, uint64_t num_bytes_to_extract) const { DCHECK(zip_file_); - if (const int err = unzOpenCurrentFile(zip_file_); err != UNZ_OK) { + // Use password only for encrypted files. For non-encrypted files, no password + // is needed, and must be nullptr. + const char* const password = + entry_.is_encrypted() ? password_.c_str() : nullptr; + if (const int err = unzOpenCurrentFilePassword(zip_file_, password); + err != UNZ_OK) { LOG(ERROR) << "Cannot open file " << Redact(entry_.path) << " from ZIP: " << UnzipError(err); return false; @@ -398,7 +403,12 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( return; } - if (const int err = unzOpenCurrentFile(zip_file_); err != UNZ_OK) { + // Use password only for encrypted files. For non-encrypted files, no password + // is needed, and must be nullptr. + const char* const password = + entry_.is_encrypted() ? password_.c_str() : nullptr; + if (const int err = unzOpenCurrentFilePassword(zip_file_, password); + err != UNZ_OK) { LOG(ERROR) << "Cannot open file " << Redact(entry_.path) << " from ZIP: " << UnzipError(err); base::SequencedTaskRunnerHandle::Get()->PostTask( diff --git a/google/zip_reader.h b/google/zip_reader.h index b66dc79..3e46ca9 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -173,6 +173,10 @@ class ZipReader { // By default, paths are assumed to be in UTF-8. void SetEncoding(std::string encoding) { encoding_ = std::move(encoding); } + // Sets the decryption password that will be used to decrypt encrypted file in + // the ZIP archive. + void SetPassword(std::string password) { password_ = std::move(password); } + // Gets the next entry. Returns null if there is no more entry. The returned // Entry is owned by this ZipReader, and is valid until Next() is called // again or until this ZipReader is closed. @@ -273,6 +277,7 @@ class ZipReader { const int64_t offset); std::string encoding_; + std::string password_; unzFile zip_file_; int num_entries_; int next_index_; diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index 4a41896..0680e5c 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -414,22 +414,88 @@ TEST_F(ZipReaderTest, Directory) { EXPECT_TRUE(entry->is_directory); } -TEST_F(ZipReaderTest, EncryptedFile) { +TEST_F(ZipReaderTest, EncryptedFile_WrongPassword) { ZipReader reader; - base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); + ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("Different Encryptions.zip"))); + reader.SetPassword("wrong password"); - ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("test_encrypted.zip"))); - const ZipReader::Entry* entry = LocateAndOpenEntry(&reader, target_path); - ASSERT_TRUE(entry); - EXPECT_EQ(target_path, entry->path); - EXPECT_TRUE(entry->is_encrypted); - reader.Close(); + { + const ZipReader::Entry* entry = reader.Next(); + ASSERT_TRUE(entry); + EXPECT_EQ(base::FilePath::FromASCII("ClearText.txt"), entry->path); + EXPECT_FALSE(entry->is_directory); + EXPECT_FALSE(entry->is_encrypted); + std::string contents = "dummy"; + EXPECT_TRUE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_EQ("This is not encrypted.\n", contents); + } - ASSERT_TRUE(reader.Open(test_zip_file_)); - entry = LocateAndOpenEntry(&reader, target_path); - ASSERT_TRUE(entry); - EXPECT_EQ(target_path, entry->path); - EXPECT_FALSE(entry->is_encrypted); + for (const base::StringPiece path : { + "Encrypted AES-128.txt", + "Encrypted AES-192.txt", + "Encrypted AES-256.txt", + "Encrypted ZipCrypto.txt", + }) { + const ZipReader::Entry* entry = reader.Next(); + ASSERT_TRUE(entry); + EXPECT_EQ(base::FilePath::FromASCII(path), entry->path); + EXPECT_FALSE(entry->is_directory); + EXPECT_TRUE(entry->is_encrypted); + std::string contents = "dummy"; + EXPECT_FALSE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_EQ("", contents); + } + + EXPECT_FALSE(reader.Next()); + EXPECT_TRUE(reader.ok()); +} + +TEST_F(ZipReaderTest, EncryptedFile_RightPassword) { + ZipReader reader; + ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("Different Encryptions.zip"))); + reader.SetPassword("password"); + + { + const ZipReader::Entry* entry = reader.Next(); + ASSERT_TRUE(entry); + EXPECT_EQ(base::FilePath::FromASCII("ClearText.txt"), entry->path); + EXPECT_FALSE(entry->is_directory); + EXPECT_FALSE(entry->is_encrypted); + std::string contents = "dummy"; + EXPECT_TRUE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_EQ("This is not encrypted.\n", contents); + } + + // TODO(crbug.com/1296838) Support AES encryption. + for (const base::StringPiece path : { + "Encrypted AES-128.txt", + "Encrypted AES-192.txt", + "Encrypted AES-256.txt", + }) { + const ZipReader::Entry* entry = reader.Next(); + ASSERT_TRUE(entry); + EXPECT_EQ(base::FilePath::FromASCII(path), entry->path); + EXPECT_FALSE(entry->is_directory); + EXPECT_TRUE(entry->is_encrypted); + std::string contents = "dummy"; + EXPECT_FALSE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_EQ("", contents); + } + + { + const ZipReader::Entry* entry = reader.Next(); + ASSERT_TRUE(entry); + EXPECT_EQ(base::FilePath::FromASCII("Encrypted ZipCrypto.txt"), + entry->path); + EXPECT_FALSE(entry->is_directory); + EXPECT_TRUE(entry->is_encrypted); + std::string contents = "dummy"; + EXPECT_TRUE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_EQ("This is encrypted with ZipCrypto.\n", contents); + } + + EXPECT_FALSE(reader.Next()); + EXPECT_TRUE(reader.ok()); } // Verifies that the ZipReader class can extract a file from a zip archive @@ -504,6 +570,71 @@ TEST_F(ZipReaderTest, ExtractToFileAsync_RegularFile) { EXPECT_EQ(file_size, listener.current_progress()); } +TEST_F(ZipReaderTest, ExtractToFileAsync_Encrypted_NoPassword) { + MockUnzipListener listener; + + ZipReader reader; + ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("Different Encryptions.zip"))); + ASSERT_TRUE(LocateAndOpenEntry( + &reader, base::FilePath::FromASCII("Encrypted ZipCrypto.txt"))); + const base::FilePath target_path = test_dir_.AppendASCII("extracted"); + reader.ExtractCurrentEntryToFilePathAsync( + target_path, + base::BindOnce(&MockUnzipListener::OnUnzipSuccess, listener.AsWeakPtr()), + base::BindOnce(&MockUnzipListener::OnUnzipFailure, listener.AsWeakPtr()), + base::BindRepeating(&MockUnzipListener::OnUnzipProgress, + listener.AsWeakPtr())); + + EXPECT_EQ(0, listener.success_calls()); + EXPECT_EQ(0, listener.failure_calls()); + EXPECT_EQ(0, listener.progress_calls()); + + base::RunLoop().RunUntilIdle(); + + EXPECT_EQ(0, listener.success_calls()); + EXPECT_EQ(1, listener.failure_calls()); + EXPECT_LE(1, listener.progress_calls()); + + // The extracted file contains rubbish data. + // We probably shouldn't even look at it. + std::string contents; + ASSERT_TRUE(base::ReadFileToString(target_path, &contents)); + EXPECT_NE("", contents); + EXPECT_EQ(contents.size(), listener.current_progress()); +} + +TEST_F(ZipReaderTest, ExtractToFileAsync_Encrypted_RightPassword) { + MockUnzipListener listener; + + ZipReader reader; + reader.SetPassword("password"); + ASSERT_TRUE(reader.Open(data_dir_.AppendASCII("Different Encryptions.zip"))); + ASSERT_TRUE(LocateAndOpenEntry( + &reader, base::FilePath::FromASCII("Encrypted ZipCrypto.txt"))); + const base::FilePath target_path = test_dir_.AppendASCII("extracted"); + reader.ExtractCurrentEntryToFilePathAsync( + target_path, + base::BindOnce(&MockUnzipListener::OnUnzipSuccess, listener.AsWeakPtr()), + base::BindOnce(&MockUnzipListener::OnUnzipFailure, listener.AsWeakPtr()), + base::BindRepeating(&MockUnzipListener::OnUnzipProgress, + listener.AsWeakPtr())); + + EXPECT_EQ(0, listener.success_calls()); + EXPECT_EQ(0, listener.failure_calls()); + EXPECT_EQ(0, listener.progress_calls()); + + base::RunLoop().RunUntilIdle(); + + EXPECT_EQ(1, listener.success_calls()); + EXPECT_EQ(0, listener.failure_calls()); + EXPECT_LE(1, listener.progress_calls()); + + std::string contents; + ASSERT_TRUE(base::ReadFileToString(target_path, &contents)); + EXPECT_EQ("This is encrypted with ZipCrypto.\n", contents); + EXPECT_EQ(contents.size(), listener.current_progress()); +} + TEST_F(ZipReaderTest, ExtractToFileAsync_WrongCrc) { MockUnzipListener listener; -- cgit v1.2.3 From 4e87a80d55fc075a46b591505a2d8c5e3027b5ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 15 Feb 2022 05:45:25 +0000 Subject: [Unzipper] Remove DudWriterDelegate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1295127 TEST=autoninja -C out/Default components_unittests && out/Default/components_unittests --gtest_filter='UnzipTest.*' Change-Id: Ic097431f6c7beabee449e66ba07df75bee2d97fa Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3461150 Reviewed-by: Sorin Jianu Reviewed-by: Noel Gordon Commit-Queue: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#971059} NOKEYCHECK=True GitOrigin-RevId: cdd1d83385d88212c55990d3ad9780ef03ef3144 --- google/zip.cc | 5 +++-- google/zip_reader.cc | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index 790969e..feedc22 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -4,6 +4,7 @@ #include "third_party/zlib/google/zip.h" +#include #include #include @@ -224,8 +225,8 @@ bool UnzipWithFilterAndWriters(const base::PlatformFile& src_file, // It's a file. std::unique_ptr writer = writer_factory.Run(entry->path); - if (!reader.ExtractCurrentEntry(writer.get(), - std::numeric_limits::max())) { + if (!writer || !reader.ExtractCurrentEntry( + writer.get(), std::numeric_limits::max())) { DLOG(WARNING) << "Cannot extract " << entry->path; return false; } diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 9ed9d07..1532ca3 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -7,6 +7,7 @@ #include #include "base/bind.h" +#include "base/check.h" #include "base/files/file.h" #include "base/i18n/icu_string_conversions.h" #include "base/logging.h" @@ -324,6 +325,7 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, return false; } + DCHECK(delegate); if (!delegate->PrepareOutput()) return false; -- cgit v1.2.3 From 9538f4194f6e5eff1bd59f2396ed9d05b1a8d801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 15 Feb 2022 22:08:31 +0000 Subject: [zip] Add struct UnzipOptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added struct UnzipOptions to pass optional arguments to the Unzip functions (including the decryption password, if needed.) Removed function UnzipWithFilterCallback(). Use Unzip() + UnzipOptions instead. Renamed function UnzipWithFilterAndWriters() to simply Unzip(), and pass to it an UnzipOptions struct too. Added tests: ZipTest.UnzipEncryptedWithRightPassword ZipTest.UnzipEncryptedWithWrongPassword ZipTest.UnzipEncryptedWithNoPassword BUG=chromium:1296838, chromium:953256 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests TEST=autoninja -C out/Default components_unittests && out/Default/components_unittests --gtest_filter='UnzipTest.*' Change-Id: I8a29f57f2cec3007e419a94dd6a4d8ec83c8a59c Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3463278 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#971432} NOKEYCHECK=True GitOrigin-RevId: 68766ecf2214aca9bd122c00ece1f7cfaba168e5 --- google/zip.cc | 44 ++++++--------- google/zip.h | 50 +++++++++-------- google/zip_unittest.cc | 148 ++++++++++++++++++++++++++++++------------------- 3 files changed, 135 insertions(+), 107 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index feedc22..f48306a 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -26,10 +26,6 @@ bool IsHiddenFile(const base::FilePath& file_path) { return file_path.BaseName().value()[0] == '.'; } -bool ExcludeNoFilesFilter(const base::FilePath& file_path) { - return true; -} - // Creates a directory at |extract_dir|/|entry_path|, including any parents. bool CreateDirectory(const base::FilePath& extract_dir, const base::FilePath& entry_path) { @@ -170,34 +166,28 @@ bool Zip(const ZipParams& params) { return zip_writer->Close(); } -bool Unzip(const base::FilePath& src_file, const base::FilePath& dest_dir) { - return UnzipWithFilterCallback( - src_file, dest_dir, base::BindRepeating(&ExcludeNoFilesFilter), true); -} - -bool UnzipWithFilterCallback(const base::FilePath& src_file, - const base::FilePath& dest_dir, - FilterCallback filter_cb, - bool log_skipped_files) { +bool Unzip(const base::FilePath& src_file, + const base::FilePath& dest_dir, + UnzipOptions options) { base::File file(src_file, base::File::FLAG_OPEN | base::File::FLAG_READ); if (!file.IsValid()) { DLOG(WARNING) << "Cannot open '" << src_file << "'"; return false; } - return UnzipWithFilterAndWriters( - file.GetPlatformFile(), - base::BindRepeating(&CreateFilePathWriterDelegate, dest_dir), - base::BindRepeating(&CreateDirectory, dest_dir), std::move(filter_cb), - log_skipped_files); + return Unzip(file.GetPlatformFile(), + base::BindRepeating(&CreateFilePathWriterDelegate, dest_dir), + base::BindRepeating(&CreateDirectory, dest_dir), + std::move(options)); } -bool UnzipWithFilterAndWriters(const base::PlatformFile& src_file, - WriterFactory writer_factory, - DirectoryCreator directory_creator, - FilterCallback filter_cb, - bool log_skipped_files) { +bool Unzip(const base::PlatformFile& src_file, + WriterFactory writer_factory, + DirectoryCreator directory_creator, + UnzipOptions options) { ZipReader reader; + reader.SetPassword(std::move(options.password)); + if (!reader.OpenFromPlatformFile(src_file)) { DLOG(WARNING) << "Cannot open ZIP from file handle " << src_file; return false; @@ -209,8 +199,8 @@ bool UnzipWithFilterAndWriters(const base::PlatformFile& src_file, return false; } - if (filter_cb && !filter_cb.Run(entry->path)) { - DLOG_IF(WARNING, log_skipped_files) + if (options.filter && !options.filter.Run(entry->path)) { + DLOG_IF(WARNING, options.log_skipped_files) << "Skipped ZIP entry " << entry->path; continue; } @@ -237,11 +227,11 @@ bool UnzipWithFilterAndWriters(const base::PlatformFile& src_file, bool ZipWithFilterCallback(const base::FilePath& src_dir, const base::FilePath& dest_file, - FilterCallback filter_cb) { + FilterCallback filter) { DCHECK(base::DirectoryExists(src_dir)); return Zip({.src_dir = src_dir, .dest_file = dest_file, - .filter_callback = std::move(filter_cb)}); + .filter_callback = std::move(filter)}); } bool Zip(const base::FilePath& src_dir, diff --git a/google/zip.h b/google/zip.h index ecaecb1..a13dbca 100644 --- a/google/zip.h +++ b/google/zip.h @@ -170,33 +170,37 @@ bool ZipFiles(const base::FilePath& src_dir, int dest_fd); #endif // defined(OS_POSIX) -// Unzip the contents of zip_file into dest_dir. -// For each file in zip_file, include it only if the callback |filter_cb| -// returns true. Otherwise omit it. -// If |log_skipped_files| is true, files skipped during extraction are printed -// to debug log. -bool UnzipWithFilterCallback(const base::FilePath& zip_file, - const base::FilePath& dest_dir, - FilterCallback filter_cb, - bool log_skipped_files); - -// Unzip the contents of zip_file, using the writers provided by writer_factory. -// For each file in zip_file, include it only if the callback |filter_cb| -// returns true. Otherwise omit it. -// If |log_skipped_files| is true, files skipped during extraction are printed -// to debug log. +// Options of the Unzip function, with valid default values. +struct UnzipOptions { + // Only extract the entries for which |filter_cb| returns true. If no filter + // is provided, everything gets extracted. + FilterCallback filter; + + // Password to decrypt the encrypted files. + std::string password; + + // If |log_skipped_files| is true, files skipped during extraction are printed + // to debug log. + bool log_skipped_files = true; +}; + typedef base::RepeatingCallback( const base::FilePath&)> WriterFactory; + typedef base::RepeatingCallback DirectoryCreator; -bool UnzipWithFilterAndWriters(const base::PlatformFile& zip_file, - WriterFactory writer_factory, - DirectoryCreator directory_creator, - FilterCallback filter_cb, - bool log_skipped_files); - -// Unzip the contents of zip_file into dest_dir. -bool Unzip(const base::FilePath& zip_file, const base::FilePath& dest_dir); + +// Unzips the contents of |zip_file|, using the writers provided by +// |writer_factory|. +bool Unzip(const base::PlatformFile& zip_file, + WriterFactory writer_factory, + DirectoryCreator directory_creator, + UnzipOptions options = {}); + +// Unzips the contents of |zip_file| into |dest_dir|. +bool Unzip(const base::FilePath& zip_file, + const base::FilePath& dest_dir, + UnzipOptions options = {}); } // namespace zip diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index a8fdfc6..e383dbf 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -192,32 +192,28 @@ class ZipTest : public PlatformTest { virtual void TearDown() { PlatformTest::TearDown(); } - bool GetTestDataDirectory(base::FilePath* path) { - bool success = base::PathService::Get(base::DIR_SOURCE_ROOT, path); + static base::FilePath GetDataDirectory() { + base::FilePath path; + bool success = base::PathService::Get(base::DIR_SOURCE_ROOT, &path); EXPECT_TRUE(success); - if (!success) - return false; - for (const base::StringPiece s : - {"third_party", "zlib", "google", "test", "data"}) { - *path = path->AppendASCII(s); - } - return true; + return std::move(path) + .AppendASCII("third_party") + .AppendASCII("zlib") + .AppendASCII("google") + .AppendASCII("test") + .AppendASCII("data"); } void TestUnzipFile(const base::FilePath::StringType& filename, bool expect_hidden_files) { - base::FilePath test_dir; - ASSERT_TRUE(GetTestDataDirectory(&test_dir)); - TestUnzipFile(test_dir.Append(filename), expect_hidden_files); + TestUnzipFile(GetDataDirectory().Append(filename), expect_hidden_files); } void TestUnzipFile(const base::FilePath& path, bool expect_hidden_files) { ASSERT_TRUE(base::PathExists(path)) << "no file " << path; ASSERT_TRUE(zip::Unzip(path, test_dir_)); - base::FilePath original_dir; - ASSERT_TRUE(GetTestDataDirectory(&original_dir)); - original_dir = original_dir.AppendASCII("test"); + base::FilePath original_dir = GetDataDirectory().AppendASCII("test"); base::FileEnumerator files( test_dir_, true, @@ -329,9 +325,7 @@ TEST_F(ZipTest, UnzipUncompressed) { } TEST_F(ZipTest, UnzipEvil) { - base::FilePath path; - ASSERT_TRUE(GetTestDataDirectory(&path)); - path = path.AppendASCII("evil.zip"); + base::FilePath path = GetDataDirectory().AppendASCII("evil.zip"); // Unzip the zip file into a sub directory of test_dir_ so evil.zip // won't create a persistent file outside test_dir_ in case of a // failure. @@ -344,10 +338,9 @@ TEST_F(ZipTest, UnzipEvil) { } TEST_F(ZipTest, UnzipEvil2) { - base::FilePath path; - ASSERT_TRUE(GetTestDataDirectory(&path)); // The ZIP file contains a file with invalid UTF-8 in its file name. - path = path.AppendASCII("evil_via_invalid_utf8.zip"); + base::FilePath path = + GetDataDirectory().AppendASCII("evil_via_invalid_utf8.zip"); // See the comment at UnzipEvil() for why we do this. base::FilePath output_dir = test_dir_.AppendASCII("out"); ASSERT_TRUE(zip::Unzip(path, output_dir)); @@ -360,10 +353,8 @@ TEST_F(ZipTest, UnzipWithFilter) { auto filter = base::BindRepeating([](const base::FilePath& path) { return path.BaseName().MaybeAsASCII() == "foo.txt"; }); - base::FilePath path; - ASSERT_TRUE(GetTestDataDirectory(&path)); - ASSERT_TRUE(zip::UnzipWithFilterCallback(path.AppendASCII("test.zip"), - test_dir_, filter, false)); + ASSERT_TRUE(zip::Unzip(GetDataDirectory().AppendASCII("test.zip"), test_dir_, + {.filter = std::move(filter)})); // Only foo.txt should have been extracted. The following paths should not // be extracted: // foo/ @@ -392,9 +383,71 @@ TEST_F(ZipTest, UnzipWithFilter) { ASSERT_EQ(0, extracted_count); } +TEST_F(ZipTest, UnzipEncryptedWithRightPassword) { + // TODO(crbug.com/1296838) Also check the AES-encrypted files. + auto filter = base::BindRepeating([](const base::FilePath& path) { + return !base::StartsWith(path.MaybeAsASCII(), "Encrypted AES"); + }); + + ASSERT_TRUE(zip::Unzip( + GetDataDirectory().AppendASCII("Different Encryptions.zip"), test_dir_, + {.filter = std::move(filter), .password = "password"})); + + std::string contents; + ASSERT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("ClearText.txt"), + &contents)); + EXPECT_EQ("This is not encrypted.\n", contents); + + ASSERT_TRUE(base::ReadFileToString( + test_dir_.AppendASCII("Encrypted ZipCrypto.txt"), &contents)); + EXPECT_EQ("This is encrypted with ZipCrypto.\n", contents); +} + +TEST_F(ZipTest, UnzipEncryptedWithWrongPassword) { + // TODO(crbug.com/1296838) Also check the AES-encrypted files. + auto filter = base::BindRepeating([](const base::FilePath& path) { + return !base::StartsWith(path.MaybeAsASCII(), "Encrypted AES"); + }); + + ASSERT_FALSE(zip::Unzip( + GetDataDirectory().AppendASCII("Different Encryptions.zip"), test_dir_, + {.filter = std::move(filter), .password = "wrong"})); + + std::string contents; + ASSERT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("ClearText.txt"), + &contents)); + EXPECT_EQ("This is not encrypted.\n", contents); + + // This extracted file contains rubbish data. + ASSERT_TRUE(base::ReadFileToString( + test_dir_.AppendASCII("Encrypted ZipCrypto.txt"), &contents)); + EXPECT_NE("", contents); + EXPECT_NE("This is encrypted with ZipCrypto.\n", contents); +} + +TEST_F(ZipTest, UnzipEncryptedWithNoPassword) { + // TODO(crbug.com/1296838) Also check the AES-encrypted files. + auto filter = base::BindRepeating([](const base::FilePath& path) { + return !base::StartsWith(path.MaybeAsASCII(), "Encrypted AES"); + }); + + ASSERT_FALSE( + zip::Unzip(GetDataDirectory().AppendASCII("Different Encryptions.zip"), + test_dir_, {.filter = std::move(filter)})); + + std::string contents; + ASSERT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("ClearText.txt"), + &contents)); + EXPECT_EQ("This is not encrypted.\n", contents); + + // This extracted file contains rubbish data. + ASSERT_TRUE(base::ReadFileToString( + test_dir_.AppendASCII("Encrypted ZipCrypto.txt"), &contents)); + EXPECT_NE("", contents); + EXPECT_NE("This is encrypted with ZipCrypto.\n", contents); +} + TEST_F(ZipTest, UnzipWithDelegates) { - auto filter = - base::BindRepeating([](const base::FilePath& path) { return true; }); auto dir_creator = base::BindRepeating( [](const base::FilePath& extract_dir, const base::FilePath& entry_path) { return base::CreateDirectory(extract_dir.Append(entry_path)); @@ -407,12 +460,10 @@ TEST_F(ZipTest, UnzipWithDelegates) { extract_dir.Append(entry_path)); }, test_dir_); - base::FilePath path; - ASSERT_TRUE(GetTestDataDirectory(&path)); - base::File file(path.AppendASCII("test.zip"), + + base::File file(GetDataDirectory().AppendASCII("test.zip"), base::File::Flags::FLAG_OPEN | base::File::Flags::FLAG_READ); - ASSERT_TRUE(zip::UnzipWithFilterAndWriters(file.GetPlatformFile(), writer, - dir_creator, filter, false)); + ASSERT_TRUE(zip::Unzip(file.GetPlatformFile(), writer, dir_creator)); base::FilePath dir = test_dir_; base::FilePath dir_foo = dir.AppendASCII("foo"); base::FilePath dir_foo_bar = dir_foo.AppendASCII("bar"); @@ -426,9 +477,7 @@ TEST_F(ZipTest, UnzipWithDelegates) { } TEST_F(ZipTest, Zip) { - base::FilePath src_dir; - ASSERT_TRUE(GetTestDataDirectory(&src_dir)); - src_dir = src_dir.AppendASCII("test"); + base::FilePath src_dir = GetDataDirectory().AppendASCII("test"); base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); @@ -439,9 +488,7 @@ TEST_F(ZipTest, Zip) { } TEST_F(ZipTest, ZipIgnoreHidden) { - base::FilePath src_dir; - ASSERT_TRUE(GetTestDataDirectory(&src_dir)); - src_dir = src_dir.AppendASCII("test"); + base::FilePath src_dir = GetDataDirectory().AppendASCII("test"); base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); @@ -452,9 +499,7 @@ TEST_F(ZipTest, ZipIgnoreHidden) { } TEST_F(ZipTest, ZipNonASCIIDir) { - base::FilePath src_dir; - ASSERT_TRUE(GetTestDataDirectory(&src_dir)); - src_dir = src_dir.AppendASCII("test"); + base::FilePath src_dir = GetDataDirectory().AppendASCII("test"); base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); @@ -492,9 +537,7 @@ TEST_F(ZipTest, ZipTimeStamp) { #if defined(OS_POSIX) TEST_F(ZipTest, ZipFiles) { - base::FilePath src_dir; - ASSERT_TRUE(GetTestDataDirectory(&src_dir)); - src_dir = src_dir.AppendASCII("test"); + base::FilePath src_dir = GetDataDirectory().AppendASCII("test"); base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); @@ -519,15 +562,12 @@ TEST_F(ZipTest, ZipFiles) { #endif // defined(OS_POSIX) TEST_F(ZipTest, UnzipFilesWithIncorrectSize) { - base::FilePath test_data_folder; - ASSERT_TRUE(GetTestDataDirectory(&test_data_folder)); - // test_mismatch_size.zip contains files with names from 0.txt to 7.txt with // sizes from 0 to 7 bytes respectively, but the metadata in the zip file says // the uncompressed size is 3 bytes. The ZipReader and minizip code needs to // be clever enough to get all the data out. base::FilePath test_zip_file = - test_data_folder.AppendASCII("test_mismatch_size.zip"); + GetDataDirectory().AppendASCII("test_mismatch_size.zip"); base::ScopedTempDir scoped_temp_dir; ASSERT_TRUE(scoped_temp_dir.CreateUniqueTempDir()); @@ -574,9 +614,7 @@ TEST_F(ZipTest, ZipWithFileAccessor) { // Tests progress reporting while zipping files. TEST_F(ZipTest, ZipProgress) { - base::FilePath src_dir; - ASSERT_TRUE(GetTestDataDirectory(&src_dir)); - src_dir = src_dir.AppendASCII("test"); + base::FilePath src_dir = GetDataDirectory().AppendASCII("test"); base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); @@ -613,9 +651,7 @@ TEST_F(ZipTest, ZipProgress) { // Tests throttling of progress reporting while zipping files. TEST_F(ZipTest, ZipProgressPeriod) { - base::FilePath src_dir; - ASSERT_TRUE(GetTestDataDirectory(&src_dir)); - src_dir = src_dir.AppendASCII("test"); + base::FilePath src_dir = GetDataDirectory().AppendASCII("test"); base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); @@ -654,9 +690,7 @@ TEST_F(ZipTest, ZipProgressPeriod) { // Tests cancellation while zipping files. TEST_F(ZipTest, ZipCancel) { - base::FilePath src_dir; - ASSERT_TRUE(GetTestDataDirectory(&src_dir)); - src_dir = src_dir.AppendASCII("test"); + base::FilePath src_dir = GetDataDirectory().AppendASCII("test"); base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); -- cgit v1.2.3 From 6a005193afe178305e2c956304bec8be06e05d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 16 Feb 2022 03:40:59 +0000 Subject: [zip] Simplify ZipReader::ExtractCurrentEntryToString() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I1baa2180fb6987e99d41f607962c71211aac5caa Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3461142 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#971606} NOKEYCHECK=True GitOrigin-RevId: 398f67e5871d1258e93c719b814b0e112775dd7a --- google/zip_reader.cc | 96 ++++++++++++++-------------------------------------- google/zip_reader.h | 8 ++--- 2 files changed, 30 insertions(+), 74 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 1532ca3..c6da9a8 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -4,6 +4,7 @@ #include "third_party/zlib/google/zip_reader.h" +#include #include #include "base/bind.h" @@ -11,6 +12,7 @@ #include "base/files/file.h" #include "base/i18n/icu_string_conversions.h" #include "base/logging.h" +#include "base/numerics/safe_conversions.h" #include "base/strings/strcat.h" #include "base/strings/string_piece.h" #include "base/strings/string_util.h" @@ -64,62 +66,21 @@ std::ostream& operator<<(std::ostream& out, Redact r) { return LOG_IS_ON(INFO) ? out << "'" << r.path << "'" : out << "(redacted)"; } -// StringWriterDelegate -------------------------------------------------------- - -// A writer delegate that writes no more than |max_read_bytes| to a given -// std::string. +// A writer delegate that writes to a given string. class StringWriterDelegate : public WriterDelegate { public: - StringWriterDelegate(size_t max_read_bytes, std::string* output); - - StringWriterDelegate(const StringWriterDelegate&) = delete; - StringWriterDelegate& operator=(const StringWriterDelegate&) = delete; - - ~StringWriterDelegate() override; + explicit StringWriterDelegate(std::string* output) : output_(output) {} // WriterDelegate methods: - - // Returns true. - bool PrepareOutput() override; - - // Appends |num_bytes| bytes from |data| to the output string. Returns false - // if |num_bytes| will cause the string to exceed |max_read_bytes|. - bool WriteBytes(const char* data, int num_bytes) override; - - void SetTimeModified(const base::Time& time) override; - - void SetPosixFilePermissions(int mode) override; + bool WriteBytes(const char* data, int num_bytes) override { + output_->append(data, num_bytes); + return true; + } private: - size_t max_read_bytes_; - std::string* output_; + std::string* const output_; }; -StringWriterDelegate::StringWriterDelegate(size_t max_read_bytes, - std::string* output) - : max_read_bytes_(max_read_bytes), output_(output) {} - -StringWriterDelegate::~StringWriterDelegate() {} - -bool StringWriterDelegate::PrepareOutput() { - return true; -} - -bool StringWriterDelegate::WriteBytes(const char* data, int num_bytes) { - if (output_->size() + num_bytes > max_read_bytes_) - return false; - output_->append(data, num_bytes); - return true; -} - -void StringWriterDelegate::SetTimeModified(const base::Time& time) { - // Do nothing. -} - -void StringWriterDelegate::SetPosixFilePermissions(int mode) { - // Do nothing. -} - #if defined(OS_POSIX) void SetPosixFilePermissions(int fd, int mode) { base::stat_wrapper_t sb; @@ -448,41 +409,36 @@ bool ZipReader::ExtractCurrentEntryToString(uint64_t max_read_bytes, std::string* output) const { DCHECK(output); DCHECK(zip_file_); + DCHECK(ok_); + DCHECK(!reached_end_); + + output->clear(); - if (max_read_bytes == 0) { - output->clear(); + if (max_read_bytes == 0) return true; - } - if (current_entry_info()->is_directory()) { - output->clear(); + if (entry_.is_directory()) return true; - } - // The original_size() is the best hint for the real size, so it saves - // doing reallocations for the common case when the uncompressed size is - // correct. However, we need to assume that the uncompressed size could be - // incorrect therefore this function needs to read as much data as possible. - std::string contents; - contents.reserve( - static_cast(std::min(base::checked_cast(max_read_bytes), - current_entry_info()->original_size()))); + // The original_size is the best hint for the real size, so it saves doing + // reallocations for the common case when the uncompressed size is correct. + // However, we need to assume that the uncompressed size could be incorrect + // therefore this function needs to read as much data as possible. + output->reserve(static_cast(std::min( + base::checked_cast(max_read_bytes), entry_.original_size()))); - StringWriterDelegate writer(max_read_bytes, &contents); + StringWriterDelegate writer(output); if (!ExtractCurrentEntry(&writer, max_read_bytes)) { - if (contents.length() < max_read_bytes) { + if (output->size() < max_read_bytes) { // There was an error in extracting entry. If ExtractCurrentEntry() // returns false, the entire file was not read - in which case - // contents.length() should equal |max_read_bytes| unless an error - // occurred which caused extraction to be aborted. + // output->size() should equal |max_read_bytes| unless an error occurred + // which caused extraction to be aborted. output->clear(); - } else { - // |num_bytes| is less than the length of current entry. - output->swap(contents); } return false; } - output->swap(contents); + return true; } diff --git a/google/zip_reader.h b/google/zip_reader.h index 3e46ca9..4eea422 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -33,19 +33,19 @@ class WriterDelegate { // Invoked once before any data is streamed out to pave the way (e.g., to open // the output file). Return false on failure to cancel extraction. - virtual bool PrepareOutput() = 0; + virtual bool PrepareOutput() { return true; } // Invoked to write the next chunk of data. Return false on failure to cancel // extraction. - virtual bool WriteBytes(const char* data, int num_bytes) = 0; + virtual bool WriteBytes(const char* data, int num_bytes) { return true; } // Sets the last-modified time of the data. - virtual void SetTimeModified(const base::Time& time) = 0; + virtual void SetTimeModified(const base::Time& time) {} // Called with the POSIX file permissions of the data; POSIX implementations // may apply some of the permissions (for example, the executable bit) to the // output file. - virtual void SetPosixFilePermissions(int mode) = 0; + virtual void SetPosixFilePermissions(int mode) {} }; // This class is used for reading ZIP archives. A typical use case of this class -- cgit v1.2.3 From 901a5d97c2ac1420e0fe065c584d259ae610f355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 16 Feb 2022 06:39:40 +0000 Subject: [zip] Add log message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit minizip can potentially report an error when closing a ZIP. Just to make sure that no error goes unnoticed, log this error to make debugging easier. BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I37ebb68ffe017c34f94f46de51c95c5a4f16ac3e Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3464761 Commit-Queue: François Degros Reviewed-by: Noel Gordon Cr-Commit-Position: refs/heads/main@{#971665} NOKEYCHECK=True GitOrigin-RevId: 5c759f4ac60940953a4ff06a3c840efd27d4d344 --- google/zip_reader.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index c6da9a8..71600bd 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -153,7 +153,9 @@ bool ZipReader::OpenFromString(const std::string& data) { void ZipReader::Close() { if (zip_file_) { - unzClose(zip_file_); + if (const int err = unzClose(zip_file_); err != UNZ_OK) { + LOG(ERROR) << "Error while closing ZIP archive: " << UnzipError(err); + } } Reset(); } -- cgit v1.2.3 From 31e5ad1c421db1bf116d9f19aa2677fed544343a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Thu, 17 Feb 2022 07:29:28 +0000 Subject: [zip] ZipReader correctly reports file size on 32-bit devices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed ZipReader to use minizip's unzGetCurrentFileInfo64() instead of unzGetCurrentFileInfo(). This allows to use 64-bit file sizes even on 32-bit devices. Enhanced test ZipTest.DISABLED_BigFile. BUG=chromium:1298347 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests --single-process-tests --gtest_also_run_disabled_tests Change-Id: I468ebebac369d8df3973919b5818f33873367d03 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3468880 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#972322} NOKEYCHECK=True GitOrigin-RevId: 1d24e7d817e06938a07d907341895c581a6a0d2b --- google/zip_reader.cc | 8 ++--- google/zip_unittest.cc | 83 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 71600bd..6fff03d 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -211,11 +211,11 @@ bool ZipReader::OpenCurrentEntryInZip() { current_entry_ = nullptr; // Get entry info. - unz_file_info info = {}; + unz_file_info64 info = {}; char path_in_zip[internal::kZipMaxPath] = {}; - if (const int err = unzGetCurrentFileInfo(zip_file_, &info, path_in_zip, - sizeof(path_in_zip) - 1, nullptr, 0, - nullptr, 0); + if (const int err = unzGetCurrentFileInfo64(zip_file_, &info, path_in_zip, + sizeof(path_in_zip) - 1, nullptr, + 0, nullptr, 0); err != UNZ_OK) { LOG(ERROR) << "Cannot get entry from ZIP: " << UnzipError(err); return false; diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index e383dbf..8b54f9b 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include #include @@ -21,6 +23,7 @@ #include "base/strings/string_util.h" #include "base/strings/stringprintf.h" #include "base/test/bind.h" +#include "base/time/time.h" #include "build/build_config.h" #include "testing/gtest/include/gtest/gtest.h" #include "testing/platform_test.h" @@ -47,6 +50,46 @@ bool CreateFile(const std::string& content, return file->IsValid(); } +// A WriterDelegate that logs progress once per second. +class ProgressWriterDelegate : public zip::WriterDelegate { + public: + explicit ProgressWriterDelegate(int64_t expected_size) + : expected_size_(expected_size) { + CHECK_GT(expected_size_, 0); + } + + bool WriteBytes(const char* data, int num_bytes) override { + received_bytes_ += num_bytes; + LogProgressIfNecessary(); + return true; + } + + void SetTimeModified(const base::Time& time) override { LogProgress(); } + + int64_t received_bytes() const { return received_bytes_; } + + private: + void LogProgressIfNecessary() { + const base::TimeTicks now = base::TimeTicks::Now(); + if (next_progress_report_time_ > now) + return; + + next_progress_report_time_ = now + progress_period_; + LogProgress(); + } + + void LogProgress() const { + LOG(INFO) << "Unzipping... " << std::setw(3) + << (100 * received_bytes_ / expected_size_) << "%"; + } + + const base::TimeDelta progress_period_ = base::Seconds(1); + base::TimeTicks next_progress_report_time_ = + base::TimeTicks::Now() + progress_period_; + const uint64_t expected_size_; + int64_t received_bytes_ = 0; +}; + // A virtual file system containing: // /test // /test/foo.txt @@ -813,7 +856,9 @@ TEST_F(ZipTest, NestedZip) { // Tests that there is no 2GB or 4GB limits. Tests that big files can be zipped // (crbug.com/1207737) and that big ZIP files can be created -// (crbug.com/1221447). +// (crbug.com/1221447). Tests that the big ZIP can be opened, that its entries +// are correctly enumerated (crbug.com/1298347), and that the big file can be +// extracted. TEST_F(ZipTest, DISABLED_BigFile) { base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); @@ -825,15 +870,26 @@ TEST_F(ZipTest, DISABLED_BigFile) { // purpose of this test, it doesn't really matter. const int64_t src_size = 5'000'000'000; + const base::FilePath src_file = src_dir.AppendASCII("src.zip"); + LOG(INFO) << "Creating big file " << src_file; { - base::File f(src_dir.AppendASCII("src.zip"), - base::File::FLAG_CREATE | base::File::FLAG_WRITE); + base::File f(src_file, base::File::FLAG_CREATE | base::File::FLAG_WRITE); ASSERT_TRUE(f.SetLength(src_size)); } // Zip the dummy ZIP file. const base::FilePath dest_file = temp_dir.GetPath().AppendASCII("dest.zip"); - EXPECT_TRUE(zip::Zip({.src_dir = src_dir, .dest_file = dest_file})); + LOG(INFO) << "Zipping big file into " << dest_file; + zip::ProgressCallback progress_callback = + base::BindLambdaForTesting([&](const zip::Progress& progress) { + LOG(INFO) << "Zipping... " << std::setw(3) + << (100 * progress.bytes / src_size) << "%"; + return true; + }); + EXPECT_TRUE(zip::Zip({.src_dir = src_dir, + .dest_file = dest_file, + .progress_callback = std::move(progress_callback), + .progress_period = base::Seconds(1)})); // Since the dummy source (inner) ZIP file should simply be stored in the // destination (outer) ZIP file, the destination file should be bigger than @@ -842,6 +898,25 @@ TEST_F(ZipTest, DISABLED_BigFile) { ASSERT_TRUE(base::GetFileSize(dest_file, &dest_file_size)); EXPECT_GT(dest_file_size, src_size + 100); EXPECT_LT(dest_file_size, src_size + 300); + + LOG(INFO) << "Reading big ZIP " << dest_file; + zip::ZipReader reader; + ASSERT_TRUE(reader.Open(dest_file)); + + const zip::ZipReader::Entry* const entry = reader.Next(); + ASSERT_TRUE(entry); + EXPECT_EQ(FP("src.zip"), entry->path); + EXPECT_EQ(src_size, entry->original_size); + EXPECT_FALSE(entry->is_directory); + EXPECT_FALSE(entry->is_encrypted); + + ProgressWriterDelegate writer(src_size); + EXPECT_TRUE(reader.ExtractCurrentEntry(&writer, + std::numeric_limits::max())); + EXPECT_EQ(src_size, writer.received_bytes()); + + EXPECT_FALSE(reader.Next()); + EXPECT_TRUE(reader.ok()); } } // namespace -- cgit v1.2.3 From 4aca88e3555bca33756b7c072bb347a249232441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Sat, 19 Feb 2022 23:04:51 +0000 Subject: [zip] Remove ZipReader's HasMore() and related methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed: ZipReader::HasMore() ZipReader::OpenCurrentEntryInZip() ZipReader::AdvanceToNextEntry() ZipReader::current_entry_info() struct ZipReader::EntryInfo These methods and struct are not used anymore. BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I8763c287f362dd1ea7125901a256702b29b6267d Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3454223 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#973315} NOKEYCHECK=True GitOrigin-RevId: 905679b37654552f1d83dad23c3bb5c843885572 --- google/zip_reader.cc | 100 +++++++++++++++++++----------------------- google/zip_reader.h | 120 +++++++++++++++++++++------------------------------ 2 files changed, 94 insertions(+), 126 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 6fff03d..b134cde 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -160,43 +160,29 @@ void ZipReader::Close() { Reset(); } -bool ZipReader::HasMore() { - return !reached_end_; -} - -bool ZipReader::AdvanceToNextEntry() { - DCHECK(zip_file_); - - // Should not go further if we already reached the end. - if (reached_end_) - return false; - - if (const int err = unzGoToNextFile(zip_file_); err != UNZ_OK) { - reached_end_ = true; - if (err != UNZ_END_OF_LIST_OF_FILE) { - LOG(ERROR) << "Cannot go to next entry in ZIP: " << UnzipError(err); - ok_ = false; - return false; - } - } - - entry_ = {}; - current_entry_ = nullptr; - return true; -} - const ZipReader::Entry* ZipReader::Next() { DCHECK(zip_file_); if (reached_end_) return nullptr; - if (next_index_ > 0 && (!AdvanceToNextEntry() || reached_end_)) - return nullptr; + DCHECK(ok_); + + // Move to the next entry if we're not trying to open the first entry. + if (next_index_ > 0) { + if (const int err = unzGoToNextFile(zip_file_); err != UNZ_OK) { + reached_end_ = true; + if (err != UNZ_END_OF_LIST_OF_FILE) { + LOG(ERROR) << "Cannot go to next entry in ZIP: " << UnzipError(err); + ok_ = false; + } + return nullptr; + } + } next_index_++; - if (!OpenCurrentEntryInZip()) { + if (!OpenEntry()) { reached_end_ = true; ok_ = false; return nullptr; @@ -205,11 +191,9 @@ const ZipReader::Entry* ZipReader::Next() { return &entry_; } -bool ZipReader::OpenCurrentEntryInZip() { +bool ZipReader::OpenEntry() { DCHECK(zip_file_); - current_entry_ = nullptr; - // Get entry info. unz_file_info64 info = {}; char path_in_zip[internal::kZipMaxPath] = {}; @@ -221,33 +205,33 @@ bool ZipReader::OpenCurrentEntryInZip() { return false; } - Entry& entry = entry_; - entry.path_in_original_encoding = path_in_zip; + entry_.path_in_original_encoding = path_in_zip; // Convert path from original encoding to Unicode. std::u16string path_in_utf16; const char* const encoding = encoding_.empty() ? "UTF-8" : encoding_.c_str(); - if (!base::CodepageToUTF16(entry.path_in_original_encoding, encoding, + if (!base::CodepageToUTF16(entry_.path_in_original_encoding, encoding, base::OnStringConversionError::SUBSTITUTE, &path_in_utf16)) { LOG(ERROR) << "Cannot convert path from encoding " << encoding; return false; } - entry.path = base::FilePath::FromUTF16Unsafe(path_in_utf16); - entry.original_size = info.uncompressed_size; + entry_.path = base::FilePath::FromUTF16Unsafe(path_in_utf16); + entry_.original_size = info.uncompressed_size; // Directory entries in ZIP have a path ending with "/". - entry.is_directory = base::EndsWith(path_in_utf16, u"/"); + entry_.is_directory = base::EndsWith(path_in_utf16, u"/"); // Check the entry path for directory traversal issues. We consider entry // paths unsafe if they are absolute or if they contain "..". On Windows, // IsAbsolute() returns false for paths starting with "/". - entry.is_unsafe = entry.path.ReferencesParent() || entry.path.IsAbsolute() || - base::StartsWith(path_in_utf16, u"/"); + entry_.is_unsafe = entry_.path.ReferencesParent() || + entry_.path.IsAbsolute() || + base::StartsWith(path_in_utf16, u"/"); // The file content of this entry is encrypted if flag bit 0 is set. - entry.is_encrypted = info.flag & 1; + entry_.is_encrypted = info.flag & 1; // Construct the last modified time. The timezone info is not present in ZIP // archives, so we construct the time as UTC. @@ -260,27 +244,29 @@ bool ZipReader::OpenCurrentEntryInZip() { exploded_time.second = info.tmu_date.tm_sec; exploded_time.millisecond = 0; - if (!base::Time::FromUTCExploded(exploded_time, &entry.last_modified)) - entry.last_modified = base::Time::UnixEpoch(); + if (!base::Time::FromUTCExploded(exploded_time, &entry_.last_modified)) + entry_.last_modified = base::Time::UnixEpoch(); #if defined(OS_POSIX) - entry.posix_mode = (info.external_fa >> 16L) & (S_IRWXU | S_IRWXG | S_IRWXO); + entry_.posix_mode = (info.external_fa >> 16L) & (S_IRWXU | S_IRWXG | S_IRWXO); #else - entry.posix_mode = 0; + entry_.posix_mode = 0; #endif - current_entry_ = &entry_; return true; } bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, uint64_t num_bytes_to_extract) const { DCHECK(zip_file_); + DCHECK_LT(0, next_index_); + DCHECK(ok_); + DCHECK(!reached_end_); // Use password only for encrypted files. For non-encrypted files, no password // is needed, and must be nullptr. const char* const password = - entry_.is_encrypted() ? password_.c_str() : nullptr; + entry_.is_encrypted ? password_.c_str() : nullptr; if (const int err = unzOpenCurrentFilePassword(zip_file_, password); err != UNZ_OK) { LOG(ERROR) << "Cannot open file " << Redact(entry_.path) @@ -332,9 +318,9 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, } if (entire_file_extracted) { - delegate->SetPosixFilePermissions(current_entry_info()->posix_mode()); - if (current_entry_info()->last_modified() != base::Time::UnixEpoch()) { - delegate->SetTimeModified(current_entry_info()->last_modified()); + delegate->SetPosixFilePermissions(entry_.posix_mode); + if (entry_.last_modified != base::Time::UnixEpoch()) { + delegate->SetTimeModified(entry_.last_modified); } } @@ -353,10 +339,12 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( FailureCallback failure_callback, const ProgressCallback& progress_callback) { DCHECK(zip_file_); - DCHECK(current_entry_); + DCHECK_LT(0, next_index_); + DCHECK(ok_); + DCHECK(!reached_end_); // If this is a directory, just create it and return. - if (current_entry_info()->is_directory()) { + if (entry_.is_directory) { if (base::CreateDirectory(output_file_path)) { base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, std::move(success_callback)); @@ -371,7 +359,7 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( // Use password only for encrypted files. For non-encrypted files, no password // is needed, and must be nullptr. const char* const password = - entry_.is_encrypted() ? password_.c_str() : nullptr; + entry_.is_encrypted ? password_.c_str() : nullptr; if (const int err = unzOpenCurrentFilePassword(zip_file_, password); err != UNZ_OK) { LOG(ERROR) << "Cannot open file " << Redact(entry_.path) @@ -411,6 +399,7 @@ bool ZipReader::ExtractCurrentEntryToString(uint64_t max_read_bytes, std::string* output) const { DCHECK(output); DCHECK(zip_file_); + DCHECK_LT(0, next_index_); DCHECK(ok_); DCHECK(!reached_end_); @@ -419,7 +408,7 @@ bool ZipReader::ExtractCurrentEntryToString(uint64_t max_read_bytes, if (max_read_bytes == 0) return true; - if (entry_.is_directory()) + if (entry_.is_directory) return true; // The original_size is the best hint for the real size, so it saves doing @@ -427,7 +416,7 @@ bool ZipReader::ExtractCurrentEntryToString(uint64_t max_read_bytes, // However, we need to assume that the uncompressed size could be incorrect // therefore this function needs to read as much data as possible. output->reserve(static_cast(std::min( - base::checked_cast(max_read_bytes), entry_.original_size()))); + base::checked_cast(max_read_bytes), entry_.original_size))); StringWriterDelegate writer(output); if (!ExtractCurrentEntry(&writer, max_read_bytes)) { @@ -463,10 +452,9 @@ void ZipReader::Reset() { zip_file_ = nullptr; num_entries_ = 0; next_index_ = 0; - reached_end_ = false; + reached_end_ = true; ok_ = false; entry_ = {}; - current_entry_ = nullptr; } void ZipReader::ExtractChunk(base::File output_file, diff --git a/google/zip_reader.h b/google/zip_reader.h index 4eea422..1f309bd 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -129,22 +129,6 @@ class ZipReader { int posix_mode; }; - // TODO(crbug.com/1295127) Remove this struct once transition to Entry is - // finished. - struct EntryInfo : Entry { - const Entry& entry() const { return *this; } - const std::string& file_path_in_original_encoding() const { - return entry().path_in_original_encoding; - } - const base::FilePath& file_path() const { return entry().path; } - int64_t original_size() const { return entry().original_size; } - base::Time last_modified() const { return entry().last_modified; } - bool is_directory() const { return entry().is_directory; } - bool is_unsafe() const { return entry().is_unsafe; } - bool is_encrypted() const { return entry().is_encrypted; } - int posix_mode() const { return entry().posix_mode; } - }; - ZipReader(); ZipReader(const ZipReader&) = delete; @@ -177,59 +161,48 @@ class ZipReader { // the ZIP archive. void SetPassword(std::string password) { password_ = std::move(password); } - // Gets the next entry. Returns null if there is no more entry. The returned - // Entry is owned by this ZipReader, and is valid until Next() is called - // again or until this ZipReader is closed. + // Gets the next entry. Returns null if there is no more entry, or if an error + // occurred while scanning entries. The returned Entry is owned by this + // ZipReader, and is valid until Next() is called again or until this + // ZipReader is closed. + // + // This function should be called before operations over the current entry + // like ExtractCurrentEntryToFile(). // - // This function is used to scan entries: // while (const ZipReader::Entry* entry = reader.Next()) { // // Do something with the current entry here. // ... // } - const Entry* Next(); - - // Returns true if the enumeration of entries was successful. - bool ok() const { return ok_; } - - // Returns true if there is at least one entry to read. This function is - // used to scan entries with AdvanceToNextEntry(), like: // - // while (reader.HasMore()) { - // // Do something with the current file here. - // reader.AdvanceToNextEntry(); + // // Finished scanning entries. + // // Check if the scanning stopped because of an error. + // if (!reader.ok()) { + // // There was an error. + // ... // } - // - // TODO(crbug.com/1295127) Remove this method. - bool HasMore(); - - // Advances the next entry. Returns true on success. - // - // TODO(crbug.com/1295127) Remove this method. - bool AdvanceToNextEntry(); + const Entry* Next(); - // Opens the current entry in the ZIP archive. On success, returns true and - // updates the current entry state (i.e. current_entry_info() is updated). - // This function should be called before operations over the current entry - // like ExtractCurrentEntryToFile(). - // - // Note that there is no CloseCurrentEntryInZip(). The current entry state is - // reset automatically as needed. - // - // TODO(crbug.com/1295127) Remove this method. - bool OpenCurrentEntryInZip(); + // Returns true if the enumeration of entries was successful, or false if it + // stopped because of an error. + bool ok() const { return ok_; } // Extracts |num_bytes_to_extract| bytes of the current entry to |delegate|, // starting from the beginning of the entry. Return value specifies whether // the entire file was extracted. + // + // Precondition: Next() returned a non-null Entry. bool ExtractCurrentEntry(WriterDelegate* delegate, uint64_t num_bytes_to_extract) const; - // Asynchronously extracts the current entry to the given output file path. - // If the current entry is a directory it just creates the directory - // synchronously instead. OpenCurrentEntryInZip() must be called beforehand. + // Asynchronously extracts the current entry to the given output file path. If + // the current entry is a directory it just creates the directory + // synchronously instead. + // // success_callback will be called on success and failure_callback will be - // called on failure. progress_callback will be called at least once. + // called on failure. progress_callback will be called at least once. // Callbacks will be posted to the current MessageLoop in-order. + // + // Precondition: Next() returned a non-null Entry. void ExtractCurrentEntryToFilePathAsync( const base::FilePath& output_file_path, SuccessCallback success_callback, @@ -239,26 +212,27 @@ class ZipReader { // Extracts the current entry into memory. If the current entry is a // directory, the |output| parameter is set to the empty string. If the // current entry is a file, the |output| parameter is filled with its - // contents. OpenCurrentEntryInZip() must be called beforehand. Note: the - // |output| parameter can be filled with a big amount of data, avoid passing - // it around by value, but by reference or pointer. Note: the value returned - // by EntryInfo::original_size() cannot be trusted, so the real size of the - // uncompressed contents can be different. |max_read_bytes| limits the ammount - // of memory used to carry the entry. Returns true if the entire content is - // read. If the entry is bigger than |max_read_bytes|, returns false and - // |output| is filled with |max_read_bytes| of data. If an error occurs, - // returns false, and |output| is set to the empty string. + // contents. + // + // The |output| parameter can be filled with a big amount of data, avoid + // passing it around by value, but by reference or pointer. + // + // The value in Entry::original_size cannot be trusted, so the real size of + // the uncompressed contents can be different. |max_read_bytes| limits the + // ammount of memory used to carry the entry. + // + // Returns true if the entire content is read. If the entry is bigger than + // |max_read_bytes|, returns false and |output| is filled with + // |max_read_bytes| of data. If an error occurs, returns false, and |output| + // is set to the empty string. + // + // Precondition: Next() returned a non-null Entry. bool ExtractCurrentEntryToString(uint64_t max_read_bytes, std::string* output) const; - // Returns the current entry info. Returns NULL if the current entry is - // not yet opened. OpenCurrentEntryInZip() must be called beforehand. - // - // TODO(crbug.com/1295127) Remove this method. - EntryInfo* current_entry_info() const { return current_entry_; } - // Returns the number of entries in the ZIP archive. - // Open() must be called beforehand. + // + // Precondition: one of the Open() methods returned true. int num_entries() const { return num_entries_; } private: @@ -268,6 +242,13 @@ class ZipReader { // Resets the internal state. void Reset(); + // Opens the current entry in the ZIP archive. On success, returns true and + // updates the current entry state |entry_|. + // + // Note that there is no matching CloseEntry(). The current entry state is + // reset automatically as needed. + bool OpenEntry(); + // Extracts a chunk of the file to the target. Will post a task for the next // chunk and success/failure/progress callbacks as necessary. void ExtractChunk(base::File target_file, @@ -283,8 +264,7 @@ class ZipReader { int next_index_; bool reached_end_; bool ok_; - EntryInfo entry_ = {}; - EntryInfo* current_entry_ = nullptr; + Entry entry_; base::WeakPtrFactory weak_ptr_factory_{this}; }; -- cgit v1.2.3 From 06d838f793fe0b308b1c3f25109bd2fc7e17337a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 22 Feb 2022 00:11:00 +0000 Subject: [zip] Enable test ZipTest.BigFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUG=chromium:1298347 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I0a6ccef11ffcec93082dc3e6ee4c90fa79ef5f21 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3469120 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#973591} NOKEYCHECK=True GitOrigin-RevId: d9fb4a003d1447be29a5cff3e4bd3b73202f29eb --- google/zip_unittest.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 8b54f9b..7dac143 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -859,7 +859,16 @@ TEST_F(ZipTest, NestedZip) { // (crbug.com/1221447). Tests that the big ZIP can be opened, that its entries // are correctly enumerated (crbug.com/1298347), and that the big file can be // extracted. +// +// This test is too slow with TSAN. +// OS Fuchsia does not seem to support large files. +// The trybot android-asan is running out of space when performing this test. +#if defined(THREAD_SANITIZER) || defined(OS_FUCHSIA) || \ + (defined(OS_ANDROID) && defined(ADDRESS_SANITIZER)) TEST_F(ZipTest, DISABLED_BigFile) { +#else +TEST_F(ZipTest, BigFile) { +#endif base::ScopedTempDir temp_dir; ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); -- cgit v1.2.3 From cd494c4016fcb3a6bca8ae617e0f754c4ac33f2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 22 Feb 2022 02:17:02 +0000 Subject: [zip] Add UnzipOptions::encoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added tests: ZipTest.UnzipSjis ZipTest.UnzipSjisAsUtf8 BUG=chromium:1287893, chromium:953256 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: Ibdf79b80c76362a768af7cfa134267ef400c30aa Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3478934 Reviewed-by: Noel Gordon Commit-Queue: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#973605} NOKEYCHECK=True GitOrigin-RevId: e1fbfd300044b4b029a3a3faee660ca93f1bcd42 --- google/zip.cc | 1 + google/zip.h | 8 ++++++-- google/zip_unittest.cc | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index f48306a..25e4fcb 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -186,6 +186,7 @@ bool Unzip(const base::PlatformFile& src_file, DirectoryCreator directory_creator, UnzipOptions options) { ZipReader reader; + reader.SetEncoding(std::move(options.encoding)); reader.SetPassword(std::move(options.password)); if (!reader.OpenFromPlatformFile(src_file)) { diff --git a/google/zip.h b/google/zip.h index a13dbca..f5b8a3a 100644 --- a/google/zip.h +++ b/google/zip.h @@ -172,8 +172,12 @@ bool ZipFiles(const base::FilePath& src_dir, // Options of the Unzip function, with valid default values. struct UnzipOptions { - // Only extract the entries for which |filter_cb| returns true. If no filter - // is provided, everything gets extracted. + // Encoding of entry paths in the ZIP archive. By default, paths are assumed + // to be in UTF-8. + std::string encoding; + + // Only extract the entries for which |filter_cb| returns true. By default, + // everything gets extracted. FilterCallback filter; // Password to decrypt the encrypted files. diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 7dac143..2254a3d 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -519,6 +519,62 @@ TEST_F(ZipTest, UnzipWithDelegates) { ASSERT_TRUE(base::PathExists(dir_foo_bar.AppendASCII("quux.txt"))); } +// Tests that a ZIP archive containing SJIS-encoded file names can be correctly +// extracted if the encoding is specified. +TEST_F(ZipTest, UnzipSjis) { + ASSERT_TRUE(zip::Unzip(GetDataDirectory().AppendASCII("SJIS Bug 846195.zip"), + test_dir_, {.encoding = "Shift_JIS"})); + + const base::FilePath dir = + test_dir_.Append(base::FilePath::FromUTF8Unsafe("新しいフォルダ")); + EXPECT_TRUE(base::DirectoryExists(dir)); + + std::string contents; + ASSERT_TRUE(base::ReadFileToString( + dir.Append(base::FilePath::FromUTF8Unsafe("SJIS_835C_ソ.txt")), + &contents)); + EXPECT_EQ( + "This file's name contains 0x5c (backslash) as the 2nd byte of Japanese " + "characater \"\x83\x5c\" when encoded in Shift JIS.", + contents); + + ASSERT_TRUE(base::ReadFileToString(dir.Append(base::FilePath::FromUTF8Unsafe( + "新しいテキスト ドキュメント.txt")), + &contents)); + EXPECT_EQ("This file name is coded in Shift JIS in the archive.", contents); +} + +// Tests that a ZIP archive containing SJIS-encoded file names can be extracted +// even if the encoding is not specified. In this case, file names are +// interpreted as UTF-8, which leads to garbled names where invalid UTF-8 +// sequences are replaced with the character �. Nevertheless, the files are +// safely extracted and readable. +TEST_F(ZipTest, UnzipSjisAsUtf8) { + ASSERT_TRUE(zip::Unzip(GetDataDirectory().AppendASCII("SJIS Bug 846195.zip"), + test_dir_)); + + EXPECT_FALSE(base::DirectoryExists( + test_dir_.Append(base::FilePath::FromUTF8Unsafe("新しいフォルダ")))); + + const base::FilePath dir = + test_dir_.Append(base::FilePath::FromUTF8Unsafe("�V�����t�H���_")); + EXPECT_TRUE(base::DirectoryExists(dir)); + + std::string contents; + ASSERT_TRUE(base::ReadFileToString( + dir.Append(base::FilePath::FromUTF8Unsafe("SJIS_835C_�\\.txt")), + &contents)); + EXPECT_EQ( + "This file's name contains 0x5c (backslash) as the 2nd byte of Japanese " + "characater \"\x83\x5c\" when encoded in Shift JIS.", + contents); + + ASSERT_TRUE(base::ReadFileToString(dir.Append(base::FilePath::FromUTF8Unsafe( + "�V�����e�L�X�g �h�L�������g.txt")), + &contents)); + EXPECT_EQ("This file name is coded in Shift JIS in the archive.", contents); +} + TEST_F(ZipTest, Zip) { base::FilePath src_dir = GetDataDirectory().AppendASCII("test"); -- cgit v1.2.3 From 79737a2a54d03cf67f405133d0ef3b1a45488b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 22 Feb 2022 03:43:30 +0000 Subject: [zip] Disable test ZipTest.BigFile on Android MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some Android waterfall and CQ try bots are running out of disk space when performing this test. BUG=chromium:1298347 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I8405881c5fcde7815b983ef7497eeabc536dcf96 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3478997 Commit-Queue: François Degros Reviewed-by: Noel Gordon Cr-Commit-Position: refs/heads/main@{#973617} NOKEYCHECK=True GitOrigin-RevId: 5bebdc45142ddbb621b027f0802208743d3ea93d --- google/zip_unittest.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 2254a3d..920fe49 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -918,9 +918,10 @@ TEST_F(ZipTest, NestedZip) { // // This test is too slow with TSAN. // OS Fuchsia does not seem to support large files. -// The trybot android-asan is running out of space when performing this test. -#if defined(THREAD_SANITIZER) || defined(OS_FUCHSIA) || \ - (defined(OS_ANDROID) && defined(ADDRESS_SANITIZER)) +// Some Android waterfall and CQ try bots are running out of space when +// performing this test (android-asan, android-11-x86-rel, +// android-marshmallow-x86-rel-non-cq). +#if defined(THREAD_SANITIZER) || defined(OS_FUCHSIA) || defined(OS_ANDROID) TEST_F(ZipTest, DISABLED_BigFile) { #else TEST_F(ZipTest, BigFile) { -- cgit v1.2.3 From ad40226ebfec24d3cd46eb8545cc65607bc125d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 22 Feb 2022 04:14:12 +0000 Subject: [zip] Add default args to ZipReader's methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added sensible default arguments to: ZipReader::ExtractCurrentEntryToString() ZipReader::ExtractCurrentEntry() Updated tests. BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests TEST=autoninja -C out/Default components_unittests && out/Default/components_unittests --gtest_filter='UnzipTest.*' Change-Id: Iec8d047e5eb777774b1aaef1f3c66f38db72a593 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3469122 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#973621} NOKEYCHECK=True GitOrigin-RevId: bf5d8b34f48462995b919eb5428ef1b050ed0a00 --- google/zip.cc | 4 +--- google/zip_reader.cc | 4 ++-- google/zip_reader.h | 13 ++++++++--- google/zip_reader_unittest.cc | 50 ++++++++++++++++++++++++++----------------- 4 files changed, 43 insertions(+), 28 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index 25e4fcb..88842c0 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -4,7 +4,6 @@ #include "third_party/zlib/google/zip.h" -#include #include #include @@ -216,8 +215,7 @@ bool Unzip(const base::PlatformFile& src_file, // It's a file. std::unique_ptr writer = writer_factory.Run(entry->path); - if (!writer || !reader.ExtractCurrentEntry( - writer.get(), std::numeric_limits::max())) { + if (!writer || !reader.ExtractCurrentEntry(writer.get())) { DLOG(WARNING) << "Cannot extract " << entry->path; return false; } diff --git a/google/zip_reader.cc b/google/zip_reader.cc index b134cde..dcda7da 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -415,8 +415,8 @@ bool ZipReader::ExtractCurrentEntryToString(uint64_t max_read_bytes, // reallocations for the common case when the uncompressed size is correct. // However, we need to assume that the uncompressed size could be incorrect // therefore this function needs to read as much data as possible. - output->reserve(static_cast(std::min( - base::checked_cast(max_read_bytes), entry_.original_size))); + output->reserve(base::checked_cast(std::min( + max_read_bytes, base::checked_cast(entry_.original_size)))); StringWriterDelegate writer(output); if (!ExtractCurrentEntry(&writer, max_read_bytes)) { diff --git a/google/zip_reader.h b/google/zip_reader.h index 1f309bd..0d81c21 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -15,6 +16,7 @@ #include "base/files/file_path.h" #include "base/files/file_util.h" #include "base/memory/weak_ptr.h" +#include "base/numerics/safe_conversions.h" #include "base/time/time.h" #if defined(USE_SYSTEM_MINIZIP) @@ -60,8 +62,7 @@ class WriterDelegate { // // while (const ZipReader::entry* entry = reader.Next()) { // auto writer = CreateFilePathWriterDelegate(extract_dir, entry->path); -// if (!reader.ExtractCurrentEntry( -// writer, std::numeric_limits::max())) { +// if (!reader.ExtractCurrentEntry(writer)) { // // Cannot extract // return; // } @@ -192,7 +193,8 @@ class ZipReader { // // Precondition: Next() returned a non-null Entry. bool ExtractCurrentEntry(WriterDelegate* delegate, - uint64_t num_bytes_to_extract) const; + uint64_t num_bytes_to_extract = + std::numeric_limits::max()) const; // Asynchronously extracts the current entry to the given output file path. If // the current entry is a directory it just creates the directory @@ -230,6 +232,11 @@ class ZipReader { bool ExtractCurrentEntryToString(uint64_t max_read_bytes, std::string* output) const; + bool ExtractCurrentEntryToString(std::string* output) const { + return ExtractCurrentEntryToString( + base::checked_cast(output->max_size()), output); + } + // Returns the number of entries in the ZIP archive. // // Precondition: one of the Open() methods returned true. diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index 0680e5c..5cd878d 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -115,8 +115,7 @@ class MockWriterDelegate : public zip::WriterDelegate { bool ExtractCurrentEntryToFilePath(zip::ZipReader* reader, base::FilePath path) { zip::FilePathWriterDelegate writer(path); - return reader->ExtractCurrentEntry(&writer, - std::numeric_limits::max()); + return reader->ExtractCurrentEntry(&writer); } const zip::ZipReader::Entry* LocateAndOpenEntry( @@ -426,7 +425,7 @@ TEST_F(ZipReaderTest, EncryptedFile_WrongPassword) { EXPECT_FALSE(entry->is_directory); EXPECT_FALSE(entry->is_encrypted); std::string contents = "dummy"; - EXPECT_TRUE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_TRUE(reader.ExtractCurrentEntryToString(&contents)); EXPECT_EQ("This is not encrypted.\n", contents); } @@ -442,7 +441,7 @@ TEST_F(ZipReaderTest, EncryptedFile_WrongPassword) { EXPECT_FALSE(entry->is_directory); EXPECT_TRUE(entry->is_encrypted); std::string contents = "dummy"; - EXPECT_FALSE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_FALSE(reader.ExtractCurrentEntryToString(&contents)); EXPECT_EQ("", contents); } @@ -462,7 +461,7 @@ TEST_F(ZipReaderTest, EncryptedFile_RightPassword) { EXPECT_FALSE(entry->is_directory); EXPECT_FALSE(entry->is_encrypted); std::string contents = "dummy"; - EXPECT_TRUE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_TRUE(reader.ExtractCurrentEntryToString(&contents)); EXPECT_EQ("This is not encrypted.\n", contents); } @@ -478,7 +477,7 @@ TEST_F(ZipReaderTest, EncryptedFile_RightPassword) { EXPECT_FALSE(entry->is_directory); EXPECT_TRUE(entry->is_encrypted); std::string contents = "dummy"; - EXPECT_FALSE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_FALSE(reader.ExtractCurrentEntryToString(&contents)); EXPECT_EQ("", contents); } @@ -490,7 +489,7 @@ TEST_F(ZipReaderTest, EncryptedFile_RightPassword) { EXPECT_FALSE(entry->is_directory); EXPECT_TRUE(entry->is_encrypted); std::string contents = "dummy"; - EXPECT_TRUE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_TRUE(reader.ExtractCurrentEntryToString(&contents)); EXPECT_EQ("This is encrypted with ZipCrypto.\n", contents); } @@ -664,10 +663,7 @@ TEST_F(ZipReaderTest, ExtractToFileAsync_WrongCrc) { ASSERT_TRUE(base::ReadFileToString(target_path, &contents)); EXPECT_EQ("This file has been changed after its CRC was computed.\n", contents); - - int64_t file_size = 0; - ASSERT_TRUE(base::GetFileSize(target_path, &file_size)); - EXPECT_EQ(file_size, listener.current_progress()); + EXPECT_EQ(contents.size(), listener.current_progress()); } // Verifies that the asynchronous extraction to a file works. @@ -729,7 +725,7 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryToString) { } // More than necessary byte read limit: must pass. - EXPECT_TRUE(reader.ExtractCurrentEntryToString(16, &contents)); + EXPECT_TRUE(reader.ExtractCurrentEntryToString(&contents)); EXPECT_EQ(std::string(base::StringPiece("0123456", i)), contents); } reader.Close(); @@ -787,7 +783,7 @@ TEST_F(ZipReaderTest, ExtractPosixPermissions) { for (auto entry : {"0.txt", "1.txt", "2.txt", "3.txt"}) { ASSERT_TRUE(LocateAndOpenEntry(&reader, base::FilePath::FromASCII(entry))); FilePathWriterDelegate delegate(temp_dir.GetPath().AppendASCII(entry)); - ASSERT_TRUE(reader.ExtractCurrentEntry(&delegate, 10000)); + ASSERT_TRUE(reader.ExtractCurrentEntry(&delegate)); } reader.Close(); @@ -830,8 +826,7 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryPrepareFailure) { ASSERT_TRUE(reader.Open(test_zip_file_)); ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); - ASSERT_FALSE(reader.ExtractCurrentEntry( - &mock_writer, std::numeric_limits::max())); + ASSERT_FALSE(reader.ExtractCurrentEntry(&mock_writer)); } // Test that when WriterDelegate::WriteBytes returns false, no other methods on @@ -847,8 +842,7 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryWriteBytesFailure) { ASSERT_TRUE(reader.Open(test_zip_file_)); ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); - ASSERT_FALSE(reader.ExtractCurrentEntry( - &mock_writer, std::numeric_limits::max())); + ASSERT_FALSE(reader.ExtractCurrentEntry(&mock_writer)); } // Test that extraction succeeds when the writer delegate reports all is well. @@ -865,8 +859,7 @@ TEST_F(ZipReaderTest, ExtractCurrentEntrySuccess) { ASSERT_TRUE(reader.Open(test_zip_file_)); ASSERT_TRUE(LocateAndOpenEntry(&reader, target_path)); - ASSERT_TRUE(reader.ExtractCurrentEntry(&mock_writer, - std::numeric_limits::max())); + ASSERT_TRUE(reader.ExtractCurrentEntry(&mock_writer)); } TEST_F(ZipReaderTest, WrongCrc) { @@ -876,9 +869,26 @@ TEST_F(ZipReaderTest, WrongCrc) { const ZipReader::Entry* const entry = LocateAndOpenEntry(&reader, base::FilePath::FromASCII("Corrupted.txt")); ASSERT_TRUE(entry); + std::string contents = "dummy"; - EXPECT_FALSE(reader.ExtractCurrentEntryToString(1000, &contents)); + EXPECT_FALSE(reader.ExtractCurrentEntryToString(&contents)); EXPECT_EQ("", contents); + + contents = "dummy"; + EXPECT_FALSE( + reader.ExtractCurrentEntryToString(entry->original_size + 1, &contents)); + EXPECT_EQ("", contents); + + contents = "dummy"; + EXPECT_FALSE( + reader.ExtractCurrentEntryToString(entry->original_size, &contents)); + EXPECT_EQ("This file has been changed after its CRC was computed.\n", + contents); + + contents = "dummy"; + EXPECT_FALSE( + reader.ExtractCurrentEntryToString(entry->original_size - 1, &contents)); + EXPECT_EQ("This file has been changed after its CRC was computed.", contents); } class FileWriterDelegateTest : public ::testing::Test { -- cgit v1.2.3 From f8d70d13465e79ff7513aafe3a0f4374271fbade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 22 Feb 2022 05:56:10 +0000 Subject: [zip] Simplify ZipReader::ExtractCurrentEntryToString() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The output string is not cleared anymore in case of error. This allows to observe the extracted content even in case of error, which is useful in unit tests and consistent with the behaviour of the other Extract*() methods such as ExtractCurrentEntryToFilePathAsync(). BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I498bdd41f9920b6e5b9d62819540476f23979995 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3473545 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#973631} NOKEYCHECK=True GitOrigin-RevId: da22d8a66a885a419376778e410482678bc9941d --- google/zip_reader.cc | 13 +------------ google/zip_reader.h | 21 +++++++++------------ google/zip_reader_unittest.cc | 7 ++++--- 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index dcda7da..ab4476b 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -419,18 +419,7 @@ bool ZipReader::ExtractCurrentEntryToString(uint64_t max_read_bytes, max_read_bytes, base::checked_cast(entry_.original_size)))); StringWriterDelegate writer(output); - if (!ExtractCurrentEntry(&writer, max_read_bytes)) { - if (output->size() < max_read_bytes) { - // There was an error in extracting entry. If ExtractCurrentEntry() - // returns false, the entire file was not read - in which case - // output->size() should equal |max_read_bytes| unless an error occurred - // which caused extraction to be aborted. - output->clear(); - } - return false; - } - - return true; + return ExtractCurrentEntry(&writer, max_read_bytes); } bool ZipReader::OpenInternal() { diff --git a/google/zip_reader.h b/google/zip_reader.h index 0d81c21..67cea3c 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -212,21 +212,18 @@ class ZipReader { const ProgressCallback& progress_callback); // Extracts the current entry into memory. If the current entry is a - // directory, the |output| parameter is set to the empty string. If the - // current entry is a file, the |output| parameter is filled with its - // contents. + // directory, |*output| is set to the empty string. If the current entry is a + // file, |*output| is filled with its contents. // - // The |output| parameter can be filled with a big amount of data, avoid - // passing it around by value, but by reference or pointer. - // - // The value in Entry::original_size cannot be trusted, so the real size of + // The value in |Entry::original_size| cannot be trusted, so the real size of // the uncompressed contents can be different. |max_read_bytes| limits the - // ammount of memory used to carry the entry. + // amount of memory used to carry the entry. // - // Returns true if the entire content is read. If the entry is bigger than - // |max_read_bytes|, returns false and |output| is filled with - // |max_read_bytes| of data. If an error occurs, returns false, and |output| - // is set to the empty string. + // Returns true if the entire content is read without error. If the content is + // bigger than |max_read_bytes|, this function returns false and |*output| is + // filled with |max_read_bytes| of data. If an error occurs, this function + // returns false and |*output| contains the content extracted so far, which + // might be garbage data. // // Precondition: Next() returned a non-null Entry. bool ExtractCurrentEntryToString(uint64_t max_read_bytes, diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index 5cd878d..cb887a9 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -442,7 +442,6 @@ TEST_F(ZipReaderTest, EncryptedFile_WrongPassword) { EXPECT_TRUE(entry->is_encrypted); std::string contents = "dummy"; EXPECT_FALSE(reader.ExtractCurrentEntryToString(&contents)); - EXPECT_EQ("", contents); } EXPECT_FALSE(reader.Next()); @@ -872,12 +871,14 @@ TEST_F(ZipReaderTest, WrongCrc) { std::string contents = "dummy"; EXPECT_FALSE(reader.ExtractCurrentEntryToString(&contents)); - EXPECT_EQ("", contents); + EXPECT_EQ("This file has been changed after its CRC was computed.\n", + contents); contents = "dummy"; EXPECT_FALSE( reader.ExtractCurrentEntryToString(entry->original_size + 1, &contents)); - EXPECT_EQ("", contents); + EXPECT_EQ("This file has been changed after its CRC was computed.\n", + contents); contents = "dummy"; EXPECT_FALSE( -- cgit v1.2.3 From fb9ea7fd319d87fcb373cc80c2cc00ece06791ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 22 Feb 2022 08:05:54 +0000 Subject: [zip] Move progress callback in ZipReader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass the ProgressCallback by value to ZipReader's ExtractCurrentEntryToFilePathAsync() and ExtractChunk() and move it. This is an optimization. BUG=chromium:1295127 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I3214c0ea9d897dc53a1999b92b364f9673120edc Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3478940 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#973640} NOKEYCHECK=True GitOrigin-RevId: b6044c6f242889d31da099d7a6413dd41b1b793b --- google/zip_reader.cc | 9 +++++---- google/zip_reader.h | 13 +++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index ab4476b..e28e5d4 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -337,7 +337,7 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( const base::FilePath& output_file_path, SuccessCallback success_callback, FailureCallback failure_callback, - const ProgressCallback& progress_callback) { + ProgressCallback progress_callback) { DCHECK(zip_file_); DCHECK_LT(0, next_index_); DCHECK(ok_); @@ -391,7 +391,7 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( FROM_HERE, base::BindOnce(&ZipReader::ExtractChunk, weak_ptr_factory_.GetWeakPtr(), std::move(output_file), std::move(success_callback), - std::move(failure_callback), progress_callback, + std::move(failure_callback), std::move(progress_callback), 0 /* initial offset */)); } @@ -449,7 +449,7 @@ void ZipReader::Reset() { void ZipReader::ExtractChunk(base::File output_file, SuccessCallback success_callback, FailureCallback failure_callback, - const ProgressCallback& progress_callback, + ProgressCallback progress_callback, int64_t offset) { char buffer[internal::kZipBufSize]; @@ -489,7 +489,8 @@ void ZipReader::ExtractChunk(base::File output_file, FROM_HERE, base::BindOnce(&ZipReader::ExtractChunk, weak_ptr_factory_.GetWeakPtr(), std::move(output_file), std::move(success_callback), - std::move(failure_callback), progress_callback, offset)); + std::move(failure_callback), std::move(progress_callback), + offset)); } // FileWriterDelegate ---------------------------------------------------------- diff --git a/google/zip_reader.h b/google/zip_reader.h index 67cea3c..ff1d3ec 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -188,8 +188,9 @@ class ZipReader { bool ok() const { return ok_; } // Extracts |num_bytes_to_extract| bytes of the current entry to |delegate|, - // starting from the beginning of the entry. Return value specifies whether - // the entire file was extracted. + // starting from the beginning of the entry. + // + // Returns true if the entire file was extracted without error. // // Precondition: Next() returned a non-null Entry. bool ExtractCurrentEntry(WriterDelegate* delegate, @@ -200,8 +201,8 @@ class ZipReader { // the current entry is a directory it just creates the directory // synchronously instead. // - // success_callback will be called on success and failure_callback will be - // called on failure. progress_callback will be called at least once. + // |success_callback| will be called on success and |failure_callback| will be + // called on failure. |progress_callback| will be called at least once. // Callbacks will be posted to the current MessageLoop in-order. // // Precondition: Next() returned a non-null Entry. @@ -209,7 +210,7 @@ class ZipReader { const base::FilePath& output_file_path, SuccessCallback success_callback, FailureCallback failure_callback, - const ProgressCallback& progress_callback); + ProgressCallback progress_callback); // Extracts the current entry into memory. If the current entry is a // directory, |*output| is set to the empty string. If the current entry is a @@ -258,7 +259,7 @@ class ZipReader { void ExtractChunk(base::File target_file, SuccessCallback success_callback, FailureCallback failure_callback, - const ProgressCallback& progress_callback, + ProgressCallback progress_callback, const int64_t offset); std::string encoding_; -- cgit v1.2.3 From 6f44c22c1f003bd20011062abec283678842567c Mon Sep 17 00:00:00 2001 From: Titouan Rigoudy Date: Tue, 22 Feb 2022 12:58:14 +0000 Subject: [Sheriff] Disable ZipTest.BigFile on Mac and debug builds. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: chromium:1299736 Change-Id: I62392d8773d31342944c055b6da1070406fa563b Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3479798 Reviewed-by: Pâris Meuleman Commit-Queue: Titouan Rigoudy Owners-Override: Titouan Rigoudy Cr-Commit-Position: refs/heads/main@{#973697} NOKEYCHECK=True GitOrigin-RevId: 0c98a465156f4d58ef55dc54299482a5092b932e --- google/zip_unittest.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 920fe49..19e3b16 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -921,7 +921,10 @@ TEST_F(ZipTest, NestedZip) { // Some Android waterfall and CQ try bots are running out of space when // performing this test (android-asan, android-11-x86-rel, // android-marshmallow-x86-rel-non-cq). -#if defined(THREAD_SANITIZER) || defined(OS_FUCHSIA) || defined(OS_ANDROID) +// TODO(https://crbug.com/1299736): Re-enable on Mac and debug builds when +// timeouts are fixed. +#if defined(THREAD_SANITIZER) || BUILDFLAG(IS_FUCHSIA) || \ + BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_MAC) || !defined(NDEBUG) TEST_F(ZipTest, DISABLED_BigFile) { #else TEST_F(ZipTest, BigFile) { -- cgit v1.2.3 From 4823a8571d38fd4a15503fa6348ffd1bb8dabe58 Mon Sep 17 00:00:00 2001 From: Fabrice de Gans Date: Tue, 22 Feb 2022 17:17:22 +0000 Subject: [zlib] Enable file descriptor operations on Fuchsia This enables APIs that take file descriptor arguments on Fuchsia. Previously, these were only enabled on POSIX platforms. Since Fuchsia also uses file descriptors, these APIs are now also enabled on Fuchsia. Bug: 1290560 Change-Id: I282e712dca4b57a0551df0266f85b025556ffc71 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3479958 Reviewed-by: Adam Langley Commit-Queue: Fabrice de Gans Cr-Commit-Position: refs/heads/main@{#973781} NOKEYCHECK=True GitOrigin-RevId: 0936966bdc62b5d714ea68e4efb5324853217c0a --- google/zip.cc | 6 +++--- google/zip.h | 6 +++--- google/zip_internal.cc | 2 +- google/zip_internal.h | 2 +- google/zip_unittest.cc | 4 ++-- google/zip_writer.cc | 2 +- google/zip_writer.h | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index 88842c0..5687b7f 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -121,7 +121,7 @@ bool Zip(const ZipParams& params) { std::unique_ptr zip_writer; -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) if (params.dest_fd != base::kInvalidPlatformFile) { DCHECK(params.dest_file.empty()); zip_writer = @@ -241,7 +241,7 @@ bool Zip(const base::FilePath& src_dir, .include_hidden_files = include_hidden_files}); } -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) bool ZipFiles(const base::FilePath& src_dir, Paths src_relative_paths, int dest_fd) { @@ -250,6 +250,6 @@ bool ZipFiles(const base::FilePath& src_dir, .dest_fd = dest_fd, .src_files = src_relative_paths}); } -#endif // defined(OS_POSIX) +#endif // defined(OS_POSIX) || defined(OS_FUCHSIA) } // namespace zip diff --git a/google/zip.h b/google/zip.h index f5b8a3a..ebd7c56 100644 --- a/google/zip.h +++ b/google/zip.h @@ -99,7 +99,7 @@ struct ZipParams { // Either dest_file or dest_fd should be set, but not both. base::FilePath dest_file; -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) // Destination file passed a file descriptor. // Either dest_file or dest_fd should be set, but not both. int dest_fd = base::kInvalidPlatformFile; @@ -159,7 +159,7 @@ bool Zip(const base::FilePath& src_dir, const base::FilePath& dest_file, bool include_hidden_files); -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) // Zips files listed in |src_relative_paths| to destination specified by file // descriptor |dest_fd|, without taking ownership of |dest_fd|. The paths listed // in |src_relative_paths| are relative to the |src_dir| and will be used as the @@ -168,7 +168,7 @@ bool Zip(const base::FilePath& src_dir, bool ZipFiles(const base::FilePath& src_dir, Paths src_relative_paths, int dest_fd); -#endif // defined(OS_POSIX) +#endif // defined(OS_POSIX) || defined(OS_FUCHSIA) // Options of the Unzip function, with valid default values. struct UnzipOptions { diff --git a/google/zip_internal.cc b/google/zip_internal.cc index 00e9eef..1adf2e6 100644 --- a/google/zip_internal.cc +++ b/google/zip_internal.cc @@ -342,7 +342,7 @@ zipFile OpenForZipping(const std::string& file_name_utf8, int append_flag) { zip_func_ptrs); } -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) zipFile OpenFdForZipping(int zip_fd, int append_flag) { zlib_filefunc64_def zip_funcs; FillFdOpenFileFunc(&zip_funcs, zip_fd); diff --git a/google/zip_internal.h b/google/zip_internal.h index c7feba6..92833fa 100644 --- a/google/zip_internal.h +++ b/google/zip_internal.h @@ -54,7 +54,7 @@ unzFile PrepareMemoryForUnzipping(const std::string& data); // Windows. |append_flag| will be passed to zipOpen2(). zipFile OpenForZipping(const std::string& file_name_utf8, int append_flag); -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) // Opens the file referred to by |zip_fd| for zipping. |append_flag| will be // passed to zipOpen2(). zipFile OpenFdForZipping(int zip_fd, int append_flag); diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 19e3b16..3025d89 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -634,7 +634,7 @@ TEST_F(ZipTest, ZipTimeStamp) { TestTimeStamp("02 Jan 2038 23:59:58", VALID_YEAR); } -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) TEST_F(ZipTest, ZipFiles) { base::FilePath src_dir = GetDataDirectory().AppendASCII("test"); @@ -658,7 +658,7 @@ TEST_F(ZipTest, ZipFiles) { EXPECT_EQ(entry->path, zip_file_list_[i]); } } -#endif // defined(OS_POSIX) +#endif // defined(OS_POSIX) || defined(OS_FUCHSIA) TEST_F(ZipTest, UnzipFilesWithIncorrectSize) { // test_mismatch_size.zip contains files with names from 0.txt to 7.txt with diff --git a/google/zip_writer.cc b/google/zip_writer.cc index 201f199..0f2bc5c 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -134,7 +134,7 @@ bool ZipWriter::AddDirectoryEntry(const base::FilePath& path) { return AddDirectoryContents(path); } -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) // static std::unique_ptr ZipWriter::CreateWithFd( int zip_file_fd, diff --git a/google/zip_writer.h b/google/zip_writer.h index fcc9627..aa3c965 100644 --- a/google/zip_writer.h +++ b/google/zip_writer.h @@ -36,7 +36,7 @@ class ZipWriter { // Creates a writer that will write a ZIP file to |zip_file_fd| or |zip_file| // and which entries are relative to |file_accessor|'s source directory. // All file reads are performed using |file_accessor|. -#if defined(OS_POSIX) +#if defined(OS_POSIX) || defined(OS_FUCHSIA) static std::unique_ptr CreateWithFd(int zip_file_fd, FileAccessor* file_accessor); #endif -- cgit v1.2.3 From 27dbe48bd5b44177e48e39cea58a591c086ce9ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Thu, 24 Feb 2022 05:20:55 +0000 Subject: [zip] Disable test ZipTest.BigFile on Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A few Linux bots are timing out when performing this test. This test is still running on ChromeOS (which is essentially Linux under the hood) and Windows (including win-asan). BUG=chromium:1298347, chromium:1298347 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I9b12050a788467e616e093e792c247ab32d08d16 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3486320 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#974496} NOKEYCHECK=True GitOrigin-RevId: b0623af23cb06551d7c0030d0425bee29b0a1558 --- google/zip_unittest.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 3025d89..b46299e 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -916,15 +916,20 @@ TEST_F(ZipTest, NestedZip) { // are correctly enumerated (crbug.com/1298347), and that the big file can be // extracted. // +// Because this test is dealing with big files, it tends to take a lot of disk +// space and time (crbug.com/1299736). Therefore, it only gets run on a few bots +// (ChromeOS and Windows). +// // This test is too slow with TSAN. // OS Fuchsia does not seem to support large files. -// Some Android waterfall and CQ try bots are running out of space when +// Some 32-bit Android waterfall and CQ try bots are running out of space when // performing this test (android-asan, android-11-x86-rel, // android-marshmallow-x86-rel-non-cq). -// TODO(https://crbug.com/1299736): Re-enable on Mac and debug builds when -// timeouts are fixed. -#if defined(THREAD_SANITIZER) || BUILDFLAG(IS_FUCHSIA) || \ - BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_MAC) || !defined(NDEBUG) +// Some Mac, Linux and Debug (dbg) bots tend to time out when performing this +// test (crbug.com/1299736). +#if defined(THREAD_SANITIZER) || BUILDFLAG(IS_FUCHSIA) || \ + BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_MAC) || BUILDFLAG(IS_LINUX) || \ + !defined(NDEBUG) TEST_F(ZipTest, DISABLED_BigFile) { #else TEST_F(ZipTest, BigFile) { -- cgit v1.2.3 From c23f5e82995a13b4e17ddc7271cebfc63ca48b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Fri, 25 Feb 2022 05:00:31 +0000 Subject: [zip] Disable test ZipTest.BigFile on Lacros MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The linux-lacros-tester-rel bot tends to time out when performing this test. This test is still running on ChromeOS (which is essentially Linux under the hood) and Windows (including win-asan). BUG=chromium:1298347, chromium:1299736, chromium:1300448 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I8d461897cd3cfa5bad8039df909780a63a2a96be Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3488651 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#975006} NOKEYCHECK=True GitOrigin-RevId: 84c65473636325a66ca32d65e332f7d2aba88ca8 --- google/zip_unittest.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index b46299e..433555d 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -926,10 +926,10 @@ TEST_F(ZipTest, NestedZip) { // performing this test (android-asan, android-11-x86-rel, // android-marshmallow-x86-rel-non-cq). // Some Mac, Linux and Debug (dbg) bots tend to time out when performing this -// test (crbug.com/1299736). +// test (crbug.com/1299736, crbug.com/1300448). #if defined(THREAD_SANITIZER) || BUILDFLAG(IS_FUCHSIA) || \ BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_MAC) || BUILDFLAG(IS_LINUX) || \ - !defined(NDEBUG) + BUILDFLAG(IS_CHROMEOS_LACROS) || !defined(NDEBUG) TEST_F(ZipTest, DISABLED_BigFile) { #else TEST_F(ZipTest, BigFile) { -- cgit v1.2.3 From 5de432790d674b036289085ff6199bede1d6b795 Mon Sep 17 00:00:00 2001 From: Yuki Shiino Date: Fri, 25 Feb 2022 08:00:30 +0000 Subject: sheriff: Disable ZipTest.BigFile on CrOS due to flakiness Bug: 1300448 Change-Id: I0780ceb9150f1c59cb7b801f57431b8ead6cfe7a Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3489577 Owners-Override: Yuki Shiino Reviewed-by: Kentaro Hara Commit-Queue: Yuki Shiino Cr-Commit-Position: refs/heads/main@{#975040} NOKEYCHECK=True GitOrigin-RevId: 76d608ed096266c426e0a93ca5b8a85ab77aad72 --- google/zip_unittest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 433555d..98a5a3d 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -929,7 +929,7 @@ TEST_F(ZipTest, NestedZip) { // test (crbug.com/1299736, crbug.com/1300448). #if defined(THREAD_SANITIZER) || BUILDFLAG(IS_FUCHSIA) || \ BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_MAC) || BUILDFLAG(IS_LINUX) || \ - BUILDFLAG(IS_CHROMEOS_LACROS) || !defined(NDEBUG) + BUILDFLAG(IS_CHROMEOS) || !defined(NDEBUG) TEST_F(ZipTest, DISABLED_BigFile) { #else TEST_F(ZipTest, BigFile) { -- cgit v1.2.3 From aa5ea608a40133d8287870e189750d62cb9dc2f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Tue, 1 Mar 2022 00:39:36 +0000 Subject: [zip] Re-enable ZipTest.BigFile on ChromeOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the flakiness dashboard [1], there didn't seem to be any flakiness on the remaining ChromeOS bots that were still running this test before it got disabled. Let's try to re-enable it and monitor the situation. [1] https://test-results.appspot.com/dashboards/flakiness_dashboard.html#testType=zlib_unittests&tests=ZipTest.BigFile This reverts commit 76d608ed096266c426e0a93ca5b8a85ab77aad72. Original change's description: > sheriff: Disable ZipTest.BigFile on CrOS due to flakiness > > Bug: 1300448 > Change-Id: I0780ceb9150f1c59cb7b801f57431b8ead6cfe7a > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3489577 > Owners-Override: Yuki Shiino > Reviewed-by: Kentaro Hara > Commit-Queue: Yuki Shiino > Cr-Commit-Position: refs/heads/main@{#975040} Bug: 1300448, 1299736, 1298347 Change-Id: Iacd23467b70a77501bb439c49be96541a13b81f9 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3496442 Reviewed-by: Noel Gordon Commit-Queue: François Degros Bot-Commit: Rubber Stamper Reviewed-by: Kentaro Hara Cr-Commit-Position: refs/heads/main@{#976021} NOKEYCHECK=True GitOrigin-RevId: 68971e57405f9b06b08fc191c66dac555ee44a00 --- google/zip_unittest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 98a5a3d..433555d 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -929,7 +929,7 @@ TEST_F(ZipTest, NestedZip) { // test (crbug.com/1299736, crbug.com/1300448). #if defined(THREAD_SANITIZER) || BUILDFLAG(IS_FUCHSIA) || \ BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_MAC) || BUILDFLAG(IS_LINUX) || \ - BUILDFLAG(IS_CHROMEOS) || !defined(NDEBUG) + BUILDFLAG(IS_CHROMEOS_LACROS) || !defined(NDEBUG) TEST_F(ZipTest, DISABLED_BigFile) { #else TEST_F(ZipTest, BigFile) { -- cgit v1.2.3 From aa6909a566735820ab0ca4de6f4045eac691f904 Mon Sep 17 00:00:00 2001 From: Daniel Cheng Date: Fri, 4 Mar 2022 09:51:11 +0000 Subject: Migrate base::size() to std::size() in //third_party. Remaining uses are all in explicit customization points for third-party libraries, or in the case of //third_party/sudden_motion_sensor, libraries that are effectively forked. Bug: 1299695 Change-Id: Ic49e481775d76fb80efc7170bf9baa1534e7d1f2 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3500722 Reviewed-by: Lei Zhang Owners-Override: Lei Zhang Commit-Queue: Daniel Cheng Cr-Commit-Position: refs/heads/main@{#977575} NOKEYCHECK=True GitOrigin-RevId: 05d0147c3ebf761316fa24197a6cb3de19ec1d7f --- google/compression_utils_unittest.cc | 16 ++++++++-------- google/zip_reader_unittest.cc | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/google/compression_utils_unittest.cc b/google/compression_utils_unittest.cc index 415b9ab..76572e5 100644 --- a/google/compression_utils_unittest.cc +++ b/google/compression_utils_unittest.cc @@ -7,9 +7,9 @@ #include #include +#include #include -#include "base/cxx17_backports.h" #include "testing/gtest/include/gtest/gtest.h" namespace compression { @@ -33,24 +33,24 @@ const uint8_t kCompressedData[] = { } // namespace TEST(CompressionUtilsTest, GzipCompression) { - std::string data(reinterpret_cast(kData), base::size(kData)); + std::string data(reinterpret_cast(kData), std::size(kData)); std::string compressed_data; EXPECT_TRUE(GzipCompress(data, &compressed_data)); std::string golden_compressed_data( reinterpret_cast(kCompressedData), - base::size(kCompressedData)); + std::size(kCompressedData)); EXPECT_EQ(golden_compressed_data, compressed_data); } TEST(CompressionUtilsTest, GzipUncompression) { std::string compressed_data(reinterpret_cast(kCompressedData), - base::size(kCompressedData)); + std::size(kCompressedData)); std::string uncompressed_data; EXPECT_TRUE(GzipUncompress(compressed_data, &uncompressed_data)); std::string golden_data(reinterpret_cast(kData), - base::size(kData)); + std::size(kData)); EXPECT_EQ(golden_data, uncompressed_data); } @@ -59,7 +59,7 @@ TEST(CompressionUtilsTest, GzipUncompressionFromSpanToString) { EXPECT_TRUE(GzipUncompress(kCompressedData, &uncompressed_data)); std::string golden_data(reinterpret_cast(kData), - base::size(kData)); + std::size(kData)); EXPECT_EQ(golden_data, uncompressed_data); } @@ -84,10 +84,10 @@ TEST(CompressionUtilsTest, LargeInput) { TEST(CompressionUtilsTest, InPlace) { const std::string original_data(reinterpret_cast(kData), - base::size(kData)); + std::size(kData)); const std::string golden_compressed_data( reinterpret_cast(kCompressedData), - base::size(kCompressedData)); + std::size(kCompressedData)); std::string data(original_data); EXPECT_TRUE(GzipCompress(data, &data)); diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index cb887a9..363e302 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -8,12 +8,12 @@ #include #include +#include #include #include #include "base/bind.h" #include "base/check.h" -#include "base/cxx17_backports.h" #include "base/files/file.h" #include "base/files/file_path.h" #include "base/files/file_util.h" @@ -516,7 +516,7 @@ TEST_F(ZipReaderTest, OpenFromString) { "\x50\x75\x78\x0b\x00\x01\x04\x8e\xf0\x00\x00\x04\x88\x13\x00\x00" "\x50\x4b\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00\x4e\x00\x00\x00" "\x52\x00\x00\x00\x00\x00"; - std::string data(kTestData, base::size(kTestData)); + std::string data(kTestData, std::size(kTestData)); ZipReader reader; ASSERT_TRUE(reader.OpenFromString(data)); base::FilePath target_path(FILE_PATH_LITERAL("test.txt")); -- cgit v1.2.3 From 85bdd6400c7113651df4e00c5dcbb8e9a8ed5e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 9 Mar 2022 03:04:55 +0000 Subject: [zip] Change one of FileWriterDelegate's constructor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No functional change. Optimization only. Changed the constructor of FileWriterDelegate that takes ownership of the File. It now moves the passed File by value rather than by unique_ptr. This avoid one extra level of indirection, and dynamic memory allocation. BUG=None TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests TEST=autoninja -C out/Default components_unittests && out/Default/components_unittests --gtest_filter='UnzipTest.*' Change-Id: Idca9b1836189d20fca10681fcb4d00c6d392eae4 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3511609 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#979053} NOKEYCHECK=True GitOrigin-RevId: d915951cf63c82c7bf5ed6250c3799660cb81280 --- google/zip_reader.cc | 10 +++++++--- google/zip_reader.h | 10 +++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index e28e5d4..0ab08c0 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -495,10 +495,14 @@ void ZipReader::ExtractChunk(base::File output_file, // FileWriterDelegate ---------------------------------------------------------- -FileWriterDelegate::FileWriterDelegate(base::File* file) : file_(file) {} +FileWriterDelegate::FileWriterDelegate(base::File* file) : file_(file) { + DCHECK(file_); +} -FileWriterDelegate::FileWriterDelegate(std::unique_ptr file) - : file_(file.get()), owned_file_(std::move(file)) {} +FileWriterDelegate::FileWriterDelegate(base::File owned_file) + : owned_file_(std::move(owned_file)) { + DCHECK_EQ(file_, &owned_file_); +} FileWriterDelegate::~FileWriterDelegate() { if (!file_->SetLength(file_length_)) { diff --git a/google/zip_reader.h b/google/zip_reader.h index ff1d3ec..c1844da 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -283,7 +283,7 @@ class FileWriterDelegate : public WriterDelegate { explicit FileWriterDelegate(base::File* file); // Constructs a FileWriterDelegate that takes ownership of |file|. - explicit FileWriterDelegate(std::unique_ptr file); + explicit FileWriterDelegate(base::File owned_file); FileWriterDelegate(const FileWriterDelegate&) = delete; FileWriterDelegate& operator=(const FileWriterDelegate&) = delete; @@ -311,12 +311,12 @@ class FileWriterDelegate : public WriterDelegate { int64_t file_length() { return file_length_; } private: - // The file the delegate modifies. - base::File* file_; - // The delegate can optionally own the file it modifies, in which case // owned_file_ is set and file_ is an alias for owned_file_. - std::unique_ptr owned_file_; + base::File owned_file_; + + // The file the delegate modifies. + base::File* const file_ = &owned_file_; int64_t file_length_ = 0; }; -- cgit v1.2.3 From 5afbbe32885db54d7934ff507c9b4cc178d29da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 14 Mar 2022 03:09:35 +0000 Subject: [zip] FileWriterDelegate doesn't truncate the file anymore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed the code that used to truncate the output file in FileWriterDelegate. There is no need for that. FileWriterDelegate is only getting passed empty files to start with. The passed file is either a freshly created one, or it is a reused temporary file that just got truncated [1]. [1] https://source.chromium.org/chromium/chromium/src/+/main:chrome/common/safe_browsing/zip_analyzer.cc;l=64;drc=e2cba64c183ae17816143ee344e6f7c81451555a BUG=None TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests TEST=autoninja -C out/Default components_unittests && out/Default/components_unittests --gtest_filter='UnzipTest.*' Change-Id: Ib3551029e18d84f6e8585d8662fed9466a023207 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3512049 Reviewed-by: Noel Gordon Reviewed-by: Alex Danilo Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#980411} NOKEYCHECK=True GitOrigin-RevId: 96875c3d1aca80f2384a4004d711dbdc87df3a86 --- google/zip_reader.cc | 14 ++++++++------ google/zip_reader.h | 8 ++++---- google/zip_reader_unittest.cc | 26 +++++++------------------- 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 0ab08c0..64a2563 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -504,14 +504,16 @@ FileWriterDelegate::FileWriterDelegate(base::File owned_file) DCHECK_EQ(file_, &owned_file_); } -FileWriterDelegate::~FileWriterDelegate() { - if (!file_->SetLength(file_length_)) { - DVPLOG(1) << "Failed updating length of written file"; - } -} +FileWriterDelegate::~FileWriterDelegate() {} bool FileWriterDelegate::PrepareOutput() { - return file_->Seek(base::File::FROM_BEGIN, 0) >= 0; + DCHECK(file_); + const bool ok = file_->IsValid(); + if (ok) { + DCHECK_EQ(file_->GetLength(), 0) + << " The output file should be initially empty"; + } + return ok; } bool FileWriterDelegate::WriteBytes(const char* data, int num_bytes) { diff --git a/google/zip_reader.h b/google/zip_reader.h index c1844da..3131108 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -274,7 +274,8 @@ class ZipReader { base::WeakPtrFactory weak_ptr_factory_{this}; }; -// A writer delegate that writes to a given File. +// A writer delegate that writes to a given File. This file is expected to be +// initially empty. class FileWriterDelegate : public WriterDelegate { public: // Constructs a FileWriterDelegate that manipulates |file|. The delegate will @@ -288,12 +289,11 @@ class FileWriterDelegate : public WriterDelegate { FileWriterDelegate(const FileWriterDelegate&) = delete; FileWriterDelegate& operator=(const FileWriterDelegate&) = delete; - // Truncates the file to the number of bytes written. ~FileWriterDelegate() override; // WriterDelegate methods: - // Seeks to the beginning of the file, returning false if the seek fails. + // Returns true if the file handle passed to the constructor is valid. bool PrepareOutput() override; // Writes |num_bytes| bytes of |data| to the file, returning false on error or @@ -307,7 +307,7 @@ class FileWriterDelegate : public WriterDelegate { // executable. void SetPosixFilePermissions(int mode) override; - // Return the actual size of the file. + // Gets the number of bytes written into the file. int64_t file_length() { return file_length_; } private: diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index 363e302..53155f6 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -903,34 +903,22 @@ class FileWriterDelegateTest : public ::testing::Test { ASSERT_TRUE(file_.IsValid()); } - // Writes data to the file, leaving the current position at the end of the - // write. - void PopulateFile() { - static const char kSomeData[] = "this sure is some data."; - static const size_t kSomeDataLen = sizeof(kSomeData) - 1; - ASSERT_NE(-1LL, file_.Write(0LL, kSomeData, kSomeDataLen)); - } - base::FilePath temp_file_path_; base::File file_; }; -TEST_F(FileWriterDelegateTest, WriteToStartAndTruncate) { - // Write stuff and advance. - PopulateFile(); +TEST_F(FileWriterDelegateTest, WriteToEnd) { + const std::string payload = "This is the actualy payload data.\n"; - // This should rewind, write, then truncate. - static const char kSomeData[] = "short"; - static const int kSomeDataLen = sizeof(kSomeData) - 1; { FileWriterDelegate writer(&file_); + EXPECT_EQ(0, writer.file_length()); ASSERT_TRUE(writer.PrepareOutput()); - ASSERT_TRUE(writer.WriteBytes(kSomeData, kSomeDataLen)); + ASSERT_TRUE(writer.WriteBytes(payload.data(), payload.size())); + EXPECT_EQ(payload.size(), writer.file_length()); } - ASSERT_EQ(kSomeDataLen, file_.GetLength()); - char buf[kSomeDataLen] = {}; - ASSERT_EQ(kSomeDataLen, file_.Read(0LL, buf, kSomeDataLen)); - ASSERT_EQ(std::string(kSomeData), std::string(buf, kSomeDataLen)); + + EXPECT_EQ(payload.size(), file_.GetLength()); } } // namespace zip -- cgit v1.2.3 From f92a1884cdb88f86e2ce3bb217e017c2c906531b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 14 Mar 2022 04:25:35 +0000 Subject: [zip] Move class Redact to its own header file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows to remove duplicated code. BUG=chromium:1303983 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I4bee3b6a901122eec20eec36f0f934b841f54588 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3515262 Reviewed-by: Alex Danilo Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#980422} NOKEYCHECK=True GitOrigin-RevId: 6f64fd4363bbf96ecc7de97032c831597f3bc898 --- google/BUILD.gn | 1 + google/redact.h | 31 +++++++++++++++++++++++++++++++ google/zip_reader.cc | 10 +--------- google/zip_writer.cc | 13 +------------ 4 files changed, 34 insertions(+), 21 deletions(-) create mode 100644 google/redact.h diff --git a/google/BUILD.gn b/google/BUILD.gn index 1d5c74f..e996b16 100644 --- a/google/BUILD.gn +++ b/google/BUILD.gn @@ -7,6 +7,7 @@ import("//build_overrides/build.gni") if (build_with_chromium) { static_library("zip") { sources = [ + "redact.h", "zip.cc", "zip.h", "zip_internal.cc", diff --git a/google/redact.h b/google/redact.h new file mode 100644 index 0000000..ea7da16 --- /dev/null +++ b/google/redact.h @@ -0,0 +1,31 @@ +// Copyright (c) 2022 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +#ifndef THIRD_PARTY_ZLIB_GOOGLE_REDACT_H_ +#define THIRD_PARTY_ZLIB_GOOGLE_REDACT_H_ + +#include + +#include "base/files/file_path.h" +#include "base/logging.h" + +namespace zip { + +// Redacts file paths in log messages. +// Example: +// LOG(ERROR) << "Cannot open " << Redact(path); +class Redact { + public: + explicit Redact(const base::FilePath& path) : path_(path) {} + + friend std::ostream& operator<<(std::ostream& out, const Redact&& r) { + return LOG_IS_ON(INFO) ? out << "'" << r.path_ << "'" : out << "(redacted)"; + } + + private: + const base::FilePath& path_; +}; + +} // namespace zip + +#endif // THIRD_PARTY_ZLIB_GOOGLE_REDACT_H_ diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 64a2563..a79b8d4 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -19,6 +19,7 @@ #include "base/strings/utf_string_conversions.h" #include "base/threading/sequenced_task_runner_handle.h" #include "build/build_config.h" +#include "third_party/zlib/google/redact.h" #include "third_party/zlib/google/zip_internal.h" #if defined(USE_SYSTEM_MINIZIP) @@ -57,15 +58,6 @@ std::ostream& operator<<(std::ostream& out, UnzipError error) { #undef SWITCH_ERR } -struct Redact { - explicit Redact(const base::FilePath& path) : path(path) {} - const base::FilePath& path; -}; - -std::ostream& operator<<(std::ostream& out, Redact r) { - return LOG_IS_ON(INFO) ? out << "'" << r.path << "'" : out << "(redacted)"; -} - // A writer delegate that writes to a given string. class StringWriterDelegate : public WriterDelegate { public: diff --git a/google/zip_writer.cc b/google/zip_writer.cc index 0f2bc5c..e3f677f 100644 --- a/google/zip_writer.cc +++ b/google/zip_writer.cc @@ -10,23 +10,12 @@ #include "base/logging.h" #include "base/strings/strcat.h" #include "base/strings/string_util.h" +#include "third_party/zlib/google/redact.h" #include "third_party/zlib/google/zip_internal.h" namespace zip { namespace internal { -class Redact { - public: - explicit Redact(const base::FilePath& path) : path_(path) {} - - friend std::ostream& operator<<(std::ostream& out, const Redact&& r) { - return LOG_IS_ON(INFO) ? out << "'" << r.path_ << "'" : out << "(redacted)"; - } - - private: - const base::FilePath& path_; -}; - bool ZipWriter::ShouldContinue() { if (!progress_callback_) return true; -- cgit v1.2.3 From c0f60596ebe26977a07d6256e524e1890b3316af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 14 Mar 2022 05:59:33 +0000 Subject: [zip] Remove duplicated code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made FilePathWriterDelegate a subclass of FileWriterDelegate. This allows to reuse code. BUG=None TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests TEST=autoninja -C out/Default components_unittests && out/Default/components_unittests --gtest_filter='UnzipTest.*' Change-Id: I96bf437cddcd853850e79d820d9561cfbbd80fe2 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3511611 Reviewed-by: Alex Danilo Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#980438} NOKEYCHECK=True GitOrigin-RevId: a6c10f7a37219d77efec93bf1726d58728eca800 --- google/zip_reader.cc | 27 ++++++--------------------- google/zip_reader.h | 20 ++++---------------- 2 files changed, 10 insertions(+), 37 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index a79b8d4..01d06b0 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -527,9 +527,9 @@ void FileWriterDelegate::SetPosixFilePermissions(int mode) { // FilePathWriterDelegate ------------------------------------------------------ -FilePathWriterDelegate::FilePathWriterDelegate( - const base::FilePath& output_file_path) - : output_file_path_(output_file_path) {} +FilePathWriterDelegate::FilePathWriterDelegate(base::FilePath output_file_path) + : FileWriterDelegate(base::File()), + output_file_path_(std::move(output_file_path)) {} FilePathWriterDelegate::~FilePathWriterDelegate() {} @@ -539,24 +539,9 @@ bool FilePathWriterDelegate::PrepareOutput() { if (!base::CreateDirectory(output_file_path_.DirName())) return false; - file_.Initialize(output_file_path_, - base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE); - return file_.IsValid(); -} - -bool FilePathWriterDelegate::WriteBytes(const char* data, int num_bytes) { - return num_bytes == file_.WriteAtCurrentPos(data, num_bytes); -} - -void FilePathWriterDelegate::SetTimeModified(const base::Time& time) { - file_.Close(); - base::TouchFile(output_file_path_, base::Time::Now(), time); -} - -void FilePathWriterDelegate::SetPosixFilePermissions(int mode) { -#if defined(OS_POSIX) - zip::SetPosixFilePermissions(file_.GetPlatformFile(), mode); -#endif + owned_file_.Initialize(output_file_path_, base::File::FLAG_CREATE_ALWAYS | + base::File::FLAG_WRITE); + return FileWriterDelegate::PrepareOutput(); } } // namespace zip diff --git a/google/zip_reader.h b/google/zip_reader.h index 3131108..1289fd3 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -310,7 +310,7 @@ class FileWriterDelegate : public WriterDelegate { // Gets the number of bytes written into the file. int64_t file_length() { return file_length_; } - private: + protected: // The delegate can optionally own the file it modifies, in which case // owned_file_ is set and file_ is an alias for owned_file_. base::File owned_file_; @@ -322,9 +322,9 @@ class FileWriterDelegate : public WriterDelegate { }; // A writer delegate that writes a file at a given path. -class FilePathWriterDelegate : public WriterDelegate { +class FilePathWriterDelegate : public FileWriterDelegate { public: - explicit FilePathWriterDelegate(const base::FilePath& output_file_path); + explicit FilePathWriterDelegate(base::FilePath output_file_path); FilePathWriterDelegate(const FilePathWriterDelegate&) = delete; FilePathWriterDelegate& operator=(const FilePathWriterDelegate&) = delete; @@ -336,20 +336,8 @@ class FilePathWriterDelegate : public WriterDelegate { // Creates the output file and any necessary intermediate directories. bool PrepareOutput() override; - // Writes |num_bytes| bytes of |data| to the file, returning false if not all - // bytes could be written. - bool WriteBytes(const char* data, int num_bytes) override; - - // Sets the last-modified time of the data. - void SetTimeModified(const base::Time& time) override; - - // On POSIX systems, sets the file to be executable if the source file was - // executable. - void SetPosixFilePermissions(int mode) override; - private: - base::FilePath output_file_path_; - base::File file_; + const base::FilePath output_file_path_; }; } // namespace zip -- cgit v1.2.3 From e466446bbcd2093aa35dfbbab33ee189304e57e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 14 Mar 2022 06:34:38 +0000 Subject: [zip] Improve log messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed option UnzipOptions::log_skipped_files. It wasn't getting set anywhere. Instead, changed the "skipped entry" log level to VLOG(1). Improved log messages. Redacted file paths that could be considered PII in some circumstances. Changed some debug warning messages to error messages. BUG=chromium:1303983 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests TEST=autoninja -C out/Default components_unittests && out/Default/components_unittests --gtest_filter='UnzipTest.*' Change-Id: I2bc5fc7d3dcf5af465faee4bdd4311e244486a9e Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3520763 Reviewed-by: Alex Danilo Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#980441} NOKEYCHECK=True GitOrigin-RevId: f2cb378ae18451b14f0176de16fe4745aead0a9f --- google/zip.cc | 27 +++++++++++++++------------ google/zip.h | 4 ---- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index 5687b7f..a52f406 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -14,6 +14,7 @@ #include "base/memory/ptr_util.h" #include "base/strings/string_util.h" #include "build/build_config.h" +#include "third_party/zlib/google/redact.h" #include "third_party/zlib/google/zip_internal.h" #include "third_party/zlib/google/zip_reader.h" #include "third_party/zlib/google/zip_writer.h" @@ -55,12 +56,13 @@ class DirectFileAccessor : public FileAccessor { const base::FilePath absolute_path = src_dir_.Append(path); if (base::DirectoryExists(absolute_path)) { files->emplace_back(); - LOG(ERROR) << "Cannot open '" << path << "': It is a directory"; + LOG(ERROR) << "Cannot open " << Redact(path) << ": It is a directory"; } else { - files->emplace_back(absolute_path, - base::File::FLAG_OPEN | base::File::FLAG_READ); - LOG_IF(ERROR, !files->back().IsValid()) - << "Cannot open '" << path << "'"; + const base::File& file = files->emplace_back( + absolute_path, base::File::FLAG_OPEN | base::File::FLAG_READ); + LOG_IF(ERROR, !file.IsValid()) + << "Cannot open " << Redact(path) << ": " + << base::File::ErrorToString(file.error_details()); } } @@ -93,7 +95,7 @@ class DirectFileAccessor : public FileAccessor { base::File::Info file_info; if (!base::GetFileInfo(src_dir_.Append(path), &file_info)) { - LOG(ERROR) << "Cannot get info of '" << path << "'"; + PLOG(ERROR) << "Cannot get info of " << Redact(path); return false; } @@ -170,7 +172,8 @@ bool Unzip(const base::FilePath& src_file, UnzipOptions options) { base::File file(src_file, base::File::FLAG_OPEN | base::File::FLAG_READ); if (!file.IsValid()) { - DLOG(WARNING) << "Cannot open '" << src_file << "'"; + LOG(ERROR) << "Cannot open " << Redact(src_file) << ": " + << base::File::ErrorToString(file.error_details()); return false; } @@ -189,19 +192,18 @@ bool Unzip(const base::PlatformFile& src_file, reader.SetPassword(std::move(options.password)); if (!reader.OpenFromPlatformFile(src_file)) { - DLOG(WARNING) << "Cannot open ZIP from file handle " << src_file; + LOG(ERROR) << "Cannot open ZIP from file handle " << src_file; return false; } while (const ZipReader::Entry* const entry = reader.Next()) { if (entry->is_unsafe) { - DLOG(WARNING) << "Found unsafe entry in ZIP: " << entry->path; + LOG(ERROR) << "Found unsafe entry " << Redact(entry->path) << " in ZIP"; return false; } if (options.filter && !options.filter.Run(entry->path)) { - DLOG_IF(WARNING, options.log_skipped_files) - << "Skipped ZIP entry " << entry->path; + VLOG(1) << "Skipped ZIP entry " << Redact(entry->path); continue; } @@ -216,7 +218,8 @@ bool Unzip(const base::PlatformFile& src_file, // It's a file. std::unique_ptr writer = writer_factory.Run(entry->path); if (!writer || !reader.ExtractCurrentEntry(writer.get())) { - DLOG(WARNING) << "Cannot extract " << entry->path; + LOG(ERROR) << "Cannot extract file " << Redact(entry->path) + << " from ZIP"; return false; } } diff --git a/google/zip.h b/google/zip.h index ebd7c56..0928bbd 100644 --- a/google/zip.h +++ b/google/zip.h @@ -182,10 +182,6 @@ struct UnzipOptions { // Password to decrypt the encrypted files. std::string password; - - // If |log_skipped_files| is true, files skipped during extraction are printed - // to debug log. - bool log_skipped_files = true; }; typedef base::RepeatingCallback( -- cgit v1.2.3 From b0676a1f52484bf53a1a49d0e48ff8abc430fafe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Mon, 14 Mar 2022 07:53:16 +0000 Subject: [zip] Add WriterDelegate::OnError() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added an OnError() method allowing a WriterDelegate to do something in case of extraction error. FileWriterDelegate::OnError() empties the output file. FilePathWriterDelegate::OnError() removes the output file altogether. BUG=chromium:1304520 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests TEST=autoninja -C out/Default components_unittests && out/Default/components_unittests --gtest_filter='UnzipTest.*' Change-Id: I3c567773a8da8c8db055db04240e646f0fc94228 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3511757 Reviewed-by: Noel Gordon Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#980454} NOKEYCHECK=True GitOrigin-RevId: 1fb92cbdc817102903c7462066a8235268ede8e2 --- google/zip_reader.cc | 29 +++++++++++++++++++++++------ google/zip_reader.h | 14 ++++++++++---- google/zip_reader_unittest.cc | 23 +++++++++++++++++++++-- google/zip_unittest.cc | 24 ++++++++++++++---------- 4 files changed, 68 insertions(+), 22 deletions(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 01d06b0..33bf788 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -309,17 +309,19 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, remaining_capacity -= num_bytes_to_write; } + if (const int err = unzCloseCurrentFile(zip_file_); err != UNZ_OK) { + LOG(ERROR) << "Cannot extract file " << Redact(entry_.path) + << " from ZIP: " << UnzipError(err); + entire_file_extracted = false; + } + if (entire_file_extracted) { delegate->SetPosixFilePermissions(entry_.posix_mode); if (entry_.last_modified != base::Time::UnixEpoch()) { delegate->SetTimeModified(entry_.last_modified); } - } - - if (const int err = unzCloseCurrentFile(zip_file_); err != UNZ_OK) { - LOG(ERROR) << "Cannot extract file " << Redact(entry_.path) - << " from ZIP: " << UnzipError(err); - return false; + } else { + delegate->OnError(); } return entire_file_extracted; @@ -525,6 +527,11 @@ void FileWriterDelegate::SetPosixFilePermissions(int mode) { #endif } +void FileWriterDelegate::OnError() { + file_length_ = 0; + file_->SetLength(0); +} + // FilePathWriterDelegate ------------------------------------------------------ FilePathWriterDelegate::FilePathWriterDelegate(base::FilePath output_file_path) @@ -544,4 +551,14 @@ bool FilePathWriterDelegate::PrepareOutput() { return FileWriterDelegate::PrepareOutput(); } +void FilePathWriterDelegate::OnError() { + FileWriterDelegate::OnError(); + owned_file_.Close(); + + if (!base::DeleteFile(output_file_path_)) { + LOG(ERROR) << "Cannot delete partially extracted file " + << Redact(output_file_path_); + } +} + } // namespace zip diff --git a/google/zip_reader.h b/google/zip_reader.h index 1289fd3..6ca9cd9 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -48,6 +48,10 @@ class WriterDelegate { // may apply some of the permissions (for example, the executable bit) to the // output file. virtual void SetPosixFilePermissions(int mode) {} + + // Called if an error occurred while extracting the file. The WriterDelegate + // can then remove and clean up the partially extracted data. + virtual void OnError() {} }; // This class is used for reading ZIP archives. A typical use case of this class @@ -291,8 +295,6 @@ class FileWriterDelegate : public WriterDelegate { ~FileWriterDelegate() override; - // WriterDelegate methods: - // Returns true if the file handle passed to the constructor is valid. bool PrepareOutput() override; @@ -307,6 +309,9 @@ class FileWriterDelegate : public WriterDelegate { // executable. void SetPosixFilePermissions(int mode) override; + // Empties the file to avoid leaving garbage data in it. + void OnError() override; + // Gets the number of bytes written into the file. int64_t file_length() { return file_length_; } @@ -331,11 +336,12 @@ class FilePathWriterDelegate : public FileWriterDelegate { ~FilePathWriterDelegate() override; - // WriterDelegate methods: - // Creates the output file and any necessary intermediate directories. bool PrepareOutput() override; + // Deletes the file. + void OnError() override; + private: const base::FilePath output_file_path_; }; diff --git a/google/zip_reader_unittest.cc b/google/zip_reader_unittest.cc index 53155f6..fc80637 100644 --- a/google/zip_reader_unittest.cc +++ b/google/zip_reader_unittest.cc @@ -110,6 +110,7 @@ class MockWriterDelegate : public zip::WriterDelegate { MOCK_METHOD2(WriteBytes, bool(const char*, int)); MOCK_METHOD1(SetTimeModified, void(const base::Time&)); MOCK_METHOD1(SetPosixFilePermissions, void(int)); + MOCK_METHOD0(OnError, void()); }; bool ExtractCurrentEntryToFilePath(zip::ZipReader* reader, @@ -828,13 +829,14 @@ TEST_F(ZipReaderTest, ExtractCurrentEntryPrepareFailure) { ASSERT_FALSE(reader.ExtractCurrentEntry(&mock_writer)); } -// Test that when WriterDelegate::WriteBytes returns false, no other methods on -// the delegate are called and the extraction fails. +// Test that when WriterDelegate::WriteBytes returns false, only the OnError +// method on the delegate is called and the extraction fails. TEST_F(ZipReaderTest, ExtractCurrentEntryWriteBytesFailure) { testing::StrictMock mock_writer; EXPECT_CALL(mock_writer, PrepareOutput()).WillOnce(Return(true)); EXPECT_CALL(mock_writer, WriteBytes(_, _)).WillOnce(Return(false)); + EXPECT_CALL(mock_writer, OnError()); base::FilePath target_path(FILE_PATH_LITERAL("foo/bar/quux.txt")); ZipReader reader; @@ -921,4 +923,21 @@ TEST_F(FileWriterDelegateTest, WriteToEnd) { EXPECT_EQ(payload.size(), file_.GetLength()); } +TEST_F(FileWriterDelegateTest, EmptyOnError) { + const std::string payload = "This is the actualy payload data.\n"; + + { + FileWriterDelegate writer(&file_); + EXPECT_EQ(0, writer.file_length()); + ASSERT_TRUE(writer.PrepareOutput()); + ASSERT_TRUE(writer.WriteBytes(payload.data(), payload.size())); + EXPECT_EQ(payload.size(), writer.file_length()); + EXPECT_EQ(payload.size(), file_.GetLength()); + writer.OnError(); + EXPECT_EQ(0, writer.file_length()); + } + + EXPECT_EQ(0, file_.GetLength()); +} + } // namespace zip diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 433555d..ab86e88 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -461,11 +461,9 @@ TEST_F(ZipTest, UnzipEncryptedWithWrongPassword) { &contents)); EXPECT_EQ("This is not encrypted.\n", contents); - // This extracted file contains rubbish data. - ASSERT_TRUE(base::ReadFileToString( - test_dir_.AppendASCII("Encrypted ZipCrypto.txt"), &contents)); - EXPECT_NE("", contents); - EXPECT_NE("This is encrypted with ZipCrypto.\n", contents); + // No rubbish file should be left behind. + EXPECT_FALSE( + base::PathExists(test_dir_.AppendASCII("Encrypted ZipCrypto.txt"))); } TEST_F(ZipTest, UnzipEncryptedWithNoPassword) { @@ -483,11 +481,17 @@ TEST_F(ZipTest, UnzipEncryptedWithNoPassword) { &contents)); EXPECT_EQ("This is not encrypted.\n", contents); - // This extracted file contains rubbish data. - ASSERT_TRUE(base::ReadFileToString( - test_dir_.AppendASCII("Encrypted ZipCrypto.txt"), &contents)); - EXPECT_NE("", contents); - EXPECT_NE("This is encrypted with ZipCrypto.\n", contents); + // No rubbish file should be left behind. + EXPECT_FALSE( + base::PathExists(test_dir_.AppendASCII("Encrypted ZipCrypto.txt"))); +} + +TEST_F(ZipTest, UnzipWrongCrc) { + ASSERT_FALSE( + zip::Unzip(GetDataDirectory().AppendASCII("Wrong CRC.zip"), test_dir_)); + + // No rubbish file should be left behind. + EXPECT_FALSE(base::PathExists(test_dir_.AppendASCII("Corrupted.txt"))); } TEST_F(ZipTest, UnzipWithDelegates) { -- cgit v1.2.3 From fbff51a880d4c9e2e629c012198e79aba2694a5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Wed, 16 Mar 2022 23:42:24 +0000 Subject: [zip] Add log messages and unit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added tests to zlib_unittests asserting the current behavior of zip::Unzip(): ZipTest.UnzipCannotCreateEmptyDir ZipTest.UnzipCannotCreateParentDir ZipTest.UnzipDifferentCases ZipTest.UnzipNoSuchFile ZipTest.UnzipRepeatedDirName ZipTest.UnzipRepeatedFileName ZipTest.UnzipWindowsSpecialNames BUG=chromium:953256 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I26e7741083c03ffa33f820c797f78124c26f72dc Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3526296 Reviewed-by: Alex Danilo Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#981912} NOKEYCHECK=True GitOrigin-RevId: 1e46eed217294f71b6b79682767e9bfa93a4ddfc --- google/test/data/Empty Dir Same Name As File.zip | Bin 0 -> 218 bytes google/test/data/Parent Dir Same Name As File.zip | Bin 0 -> 241 bytes google/test/data/Repeated Dir Name.zip | Bin 0 -> 210 bytes .../Repeated File Name With Different Cases.zip | Bin 0 -> 313 bytes google/test/data/Repeated File Name.zip | Bin 0 -> 227 bytes google/test/data/Windows Special Names.zip | Bin 0 -> 565 bytes google/zip.cc | 13 +- google/zip.h | 1 + google/zip_reader.cc | 45 ++-- google/zip_unittest.cc | 250 ++++++++++++++++----- 10 files changed, 230 insertions(+), 79 deletions(-) create mode 100644 google/test/data/Empty Dir Same Name As File.zip create mode 100644 google/test/data/Parent Dir Same Name As File.zip create mode 100644 google/test/data/Repeated Dir Name.zip create mode 100644 google/test/data/Repeated File Name With Different Cases.zip create mode 100644 google/test/data/Repeated File Name.zip create mode 100644 google/test/data/Windows Special Names.zip diff --git a/google/test/data/Empty Dir Same Name As File.zip b/google/test/data/Empty Dir Same Name As File.zip new file mode 100644 index 0000000..e26c42d Binary files /dev/null and b/google/test/data/Empty Dir Same Name As File.zip differ diff --git a/google/test/data/Parent Dir Same Name As File.zip b/google/test/data/Parent Dir Same Name As File.zip new file mode 100644 index 0000000..76cfd90 Binary files /dev/null and b/google/test/data/Parent Dir Same Name As File.zip differ diff --git a/google/test/data/Repeated Dir Name.zip b/google/test/data/Repeated Dir Name.zip new file mode 100644 index 0000000..2375bc7 Binary files /dev/null and b/google/test/data/Repeated Dir Name.zip differ diff --git a/google/test/data/Repeated File Name With Different Cases.zip b/google/test/data/Repeated File Name With Different Cases.zip new file mode 100644 index 0000000..9eb44e1 Binary files /dev/null and b/google/test/data/Repeated File Name With Different Cases.zip differ diff --git a/google/test/data/Repeated File Name.zip b/google/test/data/Repeated File Name.zip new file mode 100644 index 0000000..4a3e7b1 Binary files /dev/null and b/google/test/data/Repeated File Name.zip differ diff --git a/google/test/data/Windows Special Names.zip b/google/test/data/Windows Special Names.zip new file mode 100644 index 0000000..3b8a3ab Binary files /dev/null and b/google/test/data/Windows Special Names.zip differ diff --git a/google/zip.cc b/google/zip.cc index a52f406..de9cdce 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -29,7 +29,10 @@ bool IsHiddenFile(const base::FilePath& file_path) { // Creates a directory at |extract_dir|/|entry_path|, including any parents. bool CreateDirectory(const base::FilePath& extract_dir, const base::FilePath& entry_path) { - return base::CreateDirectory(extract_dir.Append(entry_path)); + const base::FilePath dir = extract_dir.Append(entry_path); + const bool ok = base::CreateDirectory(dir); + PLOG_IF(ERROR, !ok) << "Cannot create directory " << Redact(dir); + return ok; } // Creates a WriterDelegate that can write a file at |extract_dir|/|entry_path|. @@ -172,8 +175,8 @@ bool Unzip(const base::FilePath& src_file, UnzipOptions options) { base::File file(src_file, base::File::FLAG_OPEN | base::File::FLAG_READ); if (!file.IsValid()) { - LOG(ERROR) << "Cannot open " << Redact(src_file) << ": " - << base::File::ErrorToString(file.error_details()); + PLOG(ERROR) << "Cannot open " << Redact(src_file) << ": " + << base::File::ErrorToString(file.error_details()); return false; } @@ -209,8 +212,10 @@ bool Unzip(const base::PlatformFile& src_file, if (entry->is_directory) { // It's a directory. - if (!directory_creator.Run(entry->path)) + if (!directory_creator.Run(entry->path)) { + LOG(ERROR) << "Cannot create directory " << Redact(entry->path); return false; + } continue; } diff --git a/google/zip.h b/google/zip.h index 0928bbd..6980d03 100644 --- a/google/zip.h +++ b/google/zip.h @@ -198,6 +198,7 @@ bool Unzip(const base::PlatformFile& zip_file, UnzipOptions options = {}); // Unzips the contents of |zip_file| into |dest_dir|. +// WARNING: This can overwrite existing files in |dest_dir|. bool Unzip(const base::FilePath& zip_file, const base::FilePath& dest_dir, UnzipOptions options = {}); diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 33bf788..d2f86a4 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -145,8 +145,8 @@ bool ZipReader::OpenFromString(const std::string& data) { void ZipReader::Close() { if (zip_file_) { - if (const int err = unzClose(zip_file_); err != UNZ_OK) { - LOG(ERROR) << "Error while closing ZIP archive: " << UnzipError(err); + if (const UnzipError err{unzClose(zip_file_)}; err != UNZ_OK) { + LOG(ERROR) << "Error while closing ZIP archive: " << err; } } Reset(); @@ -162,10 +162,10 @@ const ZipReader::Entry* ZipReader::Next() { // Move to the next entry if we're not trying to open the first entry. if (next_index_ > 0) { - if (const int err = unzGoToNextFile(zip_file_); err != UNZ_OK) { + if (const UnzipError err{unzGoToNextFile(zip_file_)}; err != UNZ_OK) { reached_end_ = true; if (err != UNZ_END_OF_LIST_OF_FILE) { - LOG(ERROR) << "Cannot go to next entry in ZIP: " << UnzipError(err); + LOG(ERROR) << "Cannot go to next entry in ZIP: " << err; ok_ = false; } return nullptr; @@ -189,11 +189,11 @@ bool ZipReader::OpenEntry() { // Get entry info. unz_file_info64 info = {}; char path_in_zip[internal::kZipMaxPath] = {}; - if (const int err = unzGetCurrentFileInfo64(zip_file_, &info, path_in_zip, - sizeof(path_in_zip) - 1, nullptr, - 0, nullptr, 0); + if (const UnzipError err{unzGetCurrentFileInfo64( + zip_file_, &info, path_in_zip, sizeof(path_in_zip) - 1, nullptr, 0, + nullptr, 0)}; err != UNZ_OK) { - LOG(ERROR) << "Cannot get entry from ZIP: " << UnzipError(err); + LOG(ERROR) << "Cannot get entry from ZIP: " << err; return false; } @@ -259,10 +259,10 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, // is needed, and must be nullptr. const char* const password = entry_.is_encrypted ? password_.c_str() : nullptr; - if (const int err = unzOpenCurrentFilePassword(zip_file_, password); + if (const UnzipError err{unzOpenCurrentFilePassword(zip_file_, password)}; err != UNZ_OK) { LOG(ERROR) << "Cannot open file " << Redact(entry_.path) - << " from ZIP: " << UnzipError(err); + << " from ZIP: " << err; return false; } @@ -309,9 +309,9 @@ bool ZipReader::ExtractCurrentEntry(WriterDelegate* delegate, remaining_capacity -= num_bytes_to_write; } - if (const int err = unzCloseCurrentFile(zip_file_); err != UNZ_OK) { + if (const UnzipError err{unzCloseCurrentFile(zip_file_)}; err != UNZ_OK) { LOG(ERROR) << "Cannot extract file " << Redact(entry_.path) - << " from ZIP: " << UnzipError(err); + << " from ZIP: " << err; entire_file_extracted = false; } @@ -354,10 +354,10 @@ void ZipReader::ExtractCurrentEntryToFilePathAsync( // is needed, and must be nullptr. const char* const password = entry_.is_encrypted ? password_.c_str() : nullptr; - if (const int err = unzOpenCurrentFilePassword(zip_file_, password); + if (const UnzipError err{unzOpenCurrentFilePassword(zip_file_, password)}; err != UNZ_OK) { LOG(ERROR) << "Cannot open file " << Redact(entry_.path) - << " from ZIP: " << UnzipError(err); + << " from ZIP: " << err; base::SequencedTaskRunnerHandle::Get()->PostTask( FROM_HERE, std::move(failure_callback)); return; @@ -420,8 +420,9 @@ bool ZipReader::OpenInternal() { DCHECK(zip_file_); unz_global_info zip_info = {}; // Zero-clear. - if (const int err = unzGetGlobalInfo(zip_file_, &zip_info); err != UNZ_OK) { - LOG(ERROR) << "Cannot get ZIP info: " << UnzipError(err); + if (const UnzipError err{unzGetGlobalInfo(zip_file_, &zip_info)}; + err != UNZ_OK) { + LOG(ERROR) << "Cannot get ZIP info: " << err; return false; } @@ -451,9 +452,9 @@ void ZipReader::ExtractChunk(base::File output_file, unzReadCurrentFile(zip_file_, buffer, internal::kZipBufSize); if (num_bytes_read == 0) { - if (const int err = unzCloseCurrentFile(zip_file_); err != UNZ_OK) { + if (const UnzipError err{unzCloseCurrentFile(zip_file_)}; err != UNZ_OK) { LOG(ERROR) << "Cannot extract file " << Redact(entry_.path) - << " from ZIP: " << UnzipError(err); + << " from ZIP: " << err; std::move(failure_callback).Run(); return; } @@ -543,11 +544,17 @@ FilePathWriterDelegate::~FilePathWriterDelegate() {} bool FilePathWriterDelegate::PrepareOutput() { // We can't rely on parent directory entries being specified in the // zip, so we make sure they are created. - if (!base::CreateDirectory(output_file_path_.DirName())) + if (const base::FilePath dir = output_file_path_.DirName(); + !base::CreateDirectory(dir)) { + PLOG(ERROR) << "Cannot create directory " << Redact(dir); return false; + } owned_file_.Initialize(output_file_path_, base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE); + PLOG_IF(ERROR, !owned_file_.IsValid()) + << "Cannot create file " << Redact(output_file_path_) << ": " + << base::File::ErrorToString(owned_file_.error_details()); return FileWriterDelegate::PrepareOutput(); } diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index ab86e88..935fddd 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -7,9 +7,9 @@ #include #include -#include -#include #include +#include +#include #include #include "base/bind.h" @@ -25,6 +25,7 @@ #include "base/test/bind.h" #include "base/time/time.h" #include "build/build_config.h" +#include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" #include "testing/platform_test.h" #include "third_party/zlib/google/zip.h" @@ -36,6 +37,22 @@ namespace { +using testing::UnorderedElementsAre; + +std::vector GetRelativePaths(const base::FilePath& dir, + base::FileEnumerator::FileType type) { + std::vector got_paths; + base::FileEnumerator files(dir, true, type); + for (base::FilePath path = files.Next(); !path.empty(); path = files.Next()) { + base::FilePath relative; + EXPECT_TRUE(dir.AppendRelativePath(path, &relative)); + got_paths.push_back(relative.AsUTF8Unsafe()); + } + + EXPECT_EQ(base::File::FILE_OK, files.GetError()); + return got_paths; +} + bool CreateFile(const std::string& content, base::FilePath* file_path, base::File* file) { @@ -196,8 +213,8 @@ class VirtualFileSystem : public zip::FileAccessor { std::vector files, subdirs; }; - std::map file_tree_; - std::map files_; + std::unordered_map file_tree_; + std::unordered_map files_; }; // static @@ -261,9 +278,10 @@ class ZipTest : public PlatformTest { base::FileEnumerator files( test_dir_, true, base::FileEnumerator::FILES | base::FileEnumerator::DIRECTORIES); - base::FilePath unzipped_entry_path = files.Next(); + size_t count = 0; - while (!unzipped_entry_path.empty()) { + for (base::FilePath unzipped_entry_path = files.Next(); + !unzipped_entry_path.empty(); unzipped_entry_path = files.Next()) { EXPECT_EQ(zip_contents_.count(unzipped_entry_path), 1U) << "Couldn't find " << unzipped_entry_path; count++; @@ -281,13 +299,12 @@ class ZipTest : public PlatformTest { << "Original file '" << original_path << "' and unzipped file '" << unzipped_entry_path << "' have different contents"; } - unzipped_entry_path = files.Next(); } + EXPECT_EQ(base::File::FILE_OK, files.GetError()); size_t expected_count = 0; - for (std::set::iterator iter = zip_contents_.begin(); - iter != zip_contents_.end(); ++iter) { - if (expect_hidden_files || iter->BaseName().value()[0] != '.') + for (const base::FilePath& path : zip_contents_) { + if (expect_hidden_files || path.BaseName().value()[0] != '.') ++expected_count; } @@ -353,12 +370,23 @@ class ZipTest : public PlatformTest { base::ScopedTempDir temp_dir_; // Hard-coded contents of a known zip file. - std::set zip_contents_; + std::unordered_set zip_contents_; // Hard-coded list of relative paths for a zip file created with ZipFiles. std::vector zip_file_list_; }; +TEST_F(ZipTest, UnzipNoSuchFile) { + EXPECT_FALSE(zip::Unzip(GetDataDirectory().AppendASCII("No Such File.zip"), + test_dir_)); + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre()); + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::DIRECTORIES), + UnorderedElementsAre()); +} + TEST_F(ZipTest, Unzip) { TestUnzipFile(FILE_PATH_LITERAL("test.zip"), true); } @@ -398,32 +426,13 @@ TEST_F(ZipTest, UnzipWithFilter) { }); ASSERT_TRUE(zip::Unzip(GetDataDirectory().AppendASCII("test.zip"), test_dir_, {.filter = std::move(filter)})); - // Only foo.txt should have been extracted. The following paths should not - // be extracted: - // foo/ - // foo/bar.txt - // foo/bar/ - // foo/bar/.hidden - // foo/bar/baz.txt - // foo/bar/quux.txt - ASSERT_TRUE(base::PathExists(test_dir_.AppendASCII("foo.txt"))); - base::FileEnumerator extractedFiles( - test_dir_, - false, // Do not enumerate recursively - the file must be in the root. - base::FileEnumerator::FileType::FILES); - int extracted_count = 0; - while (!extractedFiles.Next().empty()) - ++extracted_count; - ASSERT_EQ(1, extracted_count); - - base::FileEnumerator extractedDirs( - test_dir_, - false, // Do not enumerate recursively - we require zero directories. - base::FileEnumerator::FileType::DIRECTORIES); - extracted_count = 0; - while (!extractedDirs.Next().empty()) - ++extracted_count; - ASSERT_EQ(0, extracted_count); + // Only foo.txt should have been extracted. + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("foo.txt")); + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::DIRECTORIES), + UnorderedElementsAre()); } TEST_F(ZipTest, UnzipEncryptedWithRightPassword) { @@ -494,33 +503,162 @@ TEST_F(ZipTest, UnzipWrongCrc) { EXPECT_FALSE(base::PathExists(test_dir_.AppendASCII("Corrupted.txt"))); } +TEST_F(ZipTest, UnzipRepeatedDirName) { + EXPECT_TRUE(zip::Unzip( + GetDataDirectory().AppendASCII("Repeated Dir Name.zip"), test_dir_)); + + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre()); + + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::DIRECTORIES), + UnorderedElementsAre("repeated")); +} + +TEST_F(ZipTest, UnzipRepeatedFileName) { + EXPECT_TRUE(zip::Unzip( + GetDataDirectory().AppendASCII("Repeated File Name.zip"), test_dir_)); + + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("repeated")); + + std::string contents; + EXPECT_TRUE( + base::ReadFileToString(test_dir_.AppendASCII("repeated"), &contents)); + EXPECT_EQ("Second file", contents); +} + +TEST_F(ZipTest, UnzipCannotCreateEmptyDir) { + EXPECT_FALSE(zip::Unzip( + GetDataDirectory().AppendASCII("Empty Dir Same Name As File.zip"), + test_dir_)); + + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("repeated")); + + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::DIRECTORIES), + UnorderedElementsAre()); + + std::string contents; + EXPECT_TRUE( + base::ReadFileToString(test_dir_.AppendASCII("repeated"), &contents)); + EXPECT_EQ("First file", contents); +} + +TEST_F(ZipTest, UnzipCannotCreateParentDir) { + EXPECT_FALSE(zip::Unzip( + GetDataDirectory().AppendASCII("Parent Dir Same Name As File.zip"), + test_dir_)); + + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("repeated")); + + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::DIRECTORIES), + UnorderedElementsAre()); + + std::string contents; + EXPECT_TRUE( + base::ReadFileToString(test_dir_.AppendASCII("repeated"), &contents)); + EXPECT_EQ("First file", contents); +} + +TEST_F(ZipTest, UnzipWindowsSpecialNames) { + EXPECT_TRUE(zip::Unzip( + GetDataDirectory().AppendASCII("Windows Special Names.zip"), test_dir_)); + + std::string contents; + EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("Last"), &contents)); + EXPECT_EQ("Last file", contents); + +#ifdef OS_WIN + // On Windows, the NUL* files are simply missing. No error is reported. Not + // even an error message in the logs. + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("First", "Last")); +#else + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("First", "Last", "NUL", "Nul.txt", + "nul.very long extension")); + + EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("NUL"), &contents)); + EXPECT_EQ("This is: NUL", contents); + + EXPECT_TRUE( + base::ReadFileToString(test_dir_.AppendASCII("Nul.txt"), &contents)); + EXPECT_EQ("This is: Nul.txt", contents); + + EXPECT_TRUE(base::ReadFileToString( + test_dir_.AppendASCII("nul.very long extension"), &contents)); + EXPECT_EQ("This is: nul.very long extension", contents); +#endif +} + +TEST_F(ZipTest, UnzipDifferentCases) { + EXPECT_TRUE(zip::Unzip(GetDataDirectory().AppendASCII( + "Repeated File Name With Different Cases.zip"), + test_dir_)); + +#if defined(OS_WIN) || defined(OS_MAC) + // There is only one file, with the name of the first file but the contents of + // the last file. + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("Case")); + + std::string contents; + EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("Case"), &contents)); + EXPECT_EQ("Upper case 3", contents); +#else + // All the files are correctly extracted. + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("Case", "case", "CASE")); + + std::string contents; + EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("Case"), &contents)); + EXPECT_EQ("Mixed case 111", contents); + + EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("case"), &contents)); + EXPECT_EQ("Lower case 22", contents); + + EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("CASE"), &contents)); + EXPECT_EQ("Upper case 3", contents); +#endif +} + TEST_F(ZipTest, UnzipWithDelegates) { - auto dir_creator = base::BindRepeating( - [](const base::FilePath& extract_dir, const base::FilePath& entry_path) { - return base::CreateDirectory(extract_dir.Append(entry_path)); - }, - test_dir_); - auto writer = base::BindRepeating( - [](const base::FilePath& extract_dir, const base::FilePath& entry_path) - -> std::unique_ptr { + auto dir_creator = + base::BindLambdaForTesting([this](const base::FilePath& entry_path) { + return base::CreateDirectory(test_dir_.Append(entry_path)); + }); + auto writer = + base::BindLambdaForTesting([this](const base::FilePath& entry_path) + -> std::unique_ptr { return std::make_unique( - extract_dir.Append(entry_path)); - }, - test_dir_); + test_dir_.Append(entry_path)); + }); base::File file(GetDataDirectory().AppendASCII("test.zip"), base::File::Flags::FLAG_OPEN | base::File::Flags::FLAG_READ); - ASSERT_TRUE(zip::Unzip(file.GetPlatformFile(), writer, dir_creator)); + EXPECT_TRUE(zip::Unzip(file.GetPlatformFile(), writer, dir_creator)); base::FilePath dir = test_dir_; base::FilePath dir_foo = dir.AppendASCII("foo"); base::FilePath dir_foo_bar = dir_foo.AppendASCII("bar"); - ASSERT_TRUE(base::PathExists(dir.AppendASCII("foo.txt"))); - ASSERT_TRUE(base::PathExists(dir_foo)); - ASSERT_TRUE(base::PathExists(dir_foo.AppendASCII("bar.txt"))); - ASSERT_TRUE(base::PathExists(dir_foo_bar)); - ASSERT_TRUE(base::PathExists(dir_foo_bar.AppendASCII(".hidden"))); - ASSERT_TRUE(base::PathExists(dir_foo_bar.AppendASCII("baz.txt"))); - ASSERT_TRUE(base::PathExists(dir_foo_bar.AppendASCII("quux.txt"))); + EXPECT_TRUE(base::PathExists(dir.AppendASCII("foo.txt"))); + EXPECT_TRUE(base::DirectoryExists(dir_foo)); + EXPECT_TRUE(base::PathExists(dir_foo.AppendASCII("bar.txt"))); + EXPECT_TRUE(base::DirectoryExists(dir_foo_bar)); + EXPECT_TRUE(base::PathExists(dir_foo_bar.AppendASCII(".hidden"))); + EXPECT_TRUE(base::PathExists(dir_foo_bar.AppendASCII("baz.txt"))); + EXPECT_TRUE(base::PathExists(dir_foo_bar.AppendASCII("quux.txt"))); } // Tests that a ZIP archive containing SJIS-encoded file names can be correctly -- cgit v1.2.3 From 923f5eb4bfa403d0f14d9cbca2cdf7fa7416ee67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Thu, 17 Mar 2022 05:46:45 +0000 Subject: [zip] Unzip() does not overwrite files anymore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changed the way FilePathWriterDelegate creates a new file, by using FLAG_CREATE instead of FLAG_CREATE_ALWAYS. This prevents FilePathWriterDelegate and zip::Unzip() from overwriting any existing file. This allows the detection of duplicated or conflicting file names when extracting a ZIP archive. See ZipTest.UnzipRepeatedFileName. This fixes a confusing corner case on case-insensitive file systems, which could result in file contents not matching file names as stored in the ZIP archive. See ZipTest.UnzipDifferentCases. BUG=chromium:953256 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I6cf80e6d8b9e39ffc76739c00cb6f2ecf57da88f Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3529591 Reviewed-by: Alex Danilo Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#982048} NOKEYCHECK=True GitOrigin-RevId: 7ba6004ba3727e9f5339841cfe0f54e7e51dfbf2 --- google/zip.cc | 4 ++++ google/zip.h | 4 +++- google/zip_reader.cc | 4 ++-- google/zip_reader.h | 9 ++++++--- google/zip_unittest.cc | 23 +++++++++++++---------- 5 files changed, 28 insertions(+), 16 deletions(-) diff --git a/google/zip.cc b/google/zip.cc index de9cdce..7f9af5b 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -10,6 +10,7 @@ #include "base/bind.h" #include "base/files/file.h" #include "base/files/file_enumerator.h" +#include "base/files/file_util.h" #include "base/logging.h" #include "base/memory/ptr_util.h" #include "base/strings/string_util.h" @@ -180,6 +181,9 @@ bool Unzip(const base::FilePath& src_file, return false; } + DLOG_IF(WARNING, !base::IsDirectoryEmpty(dest_dir)) + << "ZIP extraction directory is not empty: " << dest_dir; + return Unzip(file.GetPlatformFile(), base::BindRepeating(&CreateFilePathWriterDelegate, dest_dir), base::BindRepeating(&CreateDirectory, dest_dir), diff --git a/google/zip.h b/google/zip.h index 6980d03..621f0d9 100644 --- a/google/zip.h +++ b/google/zip.h @@ -198,7 +198,9 @@ bool Unzip(const base::PlatformFile& zip_file, UnzipOptions options = {}); // Unzips the contents of |zip_file| into |dest_dir|. -// WARNING: This can overwrite existing files in |dest_dir|. +// This function does not overwrite any existing file. +// A filename collision will result in an error. +// Therefore, |dest_dir| should initially be an empty directory. bool Unzip(const base::FilePath& zip_file, const base::FilePath& dest_dir, UnzipOptions options = {}); diff --git a/google/zip_reader.cc b/google/zip_reader.cc index d2f86a4..93fe134 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -550,8 +550,8 @@ bool FilePathWriterDelegate::PrepareOutput() { return false; } - owned_file_.Initialize(output_file_path_, base::File::FLAG_CREATE_ALWAYS | - base::File::FLAG_WRITE); + owned_file_.Initialize(output_file_path_, + base::File::FLAG_CREATE | base::File::FLAG_WRITE); PLOG_IF(ERROR, !owned_file_.IsValid()) << "Cannot create file " << Redact(output_file_path_) << ": " << base::File::ErrorToString(owned_file_.error_details()); diff --git a/google/zip_reader.h b/google/zip_reader.h index 6ca9cd9..edf548b 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -326,7 +326,8 @@ class FileWriterDelegate : public WriterDelegate { int64_t file_length_ = 0; }; -// A writer delegate that writes a file at a given path. +// A writer delegate that creates and writes a file at a given path. This does +// not overwrite any existing file. class FilePathWriterDelegate : public FileWriterDelegate { public: explicit FilePathWriterDelegate(base::FilePath output_file_path); @@ -336,10 +337,12 @@ class FilePathWriterDelegate : public FileWriterDelegate { ~FilePathWriterDelegate() override; - // Creates the output file and any necessary intermediate directories. + // Creates the output file and any necessary intermediate directories. Does + // not overwrite any existing file, and returns false if the output file + // cannot be created because another file conflicts with it. bool PrepareOutput() override; - // Deletes the file. + // Deletes the output file. void OnError() override; private: diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index 935fddd..cf54991 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -517,7 +517,7 @@ TEST_F(ZipTest, UnzipRepeatedDirName) { } TEST_F(ZipTest, UnzipRepeatedFileName) { - EXPECT_TRUE(zip::Unzip( + EXPECT_FALSE(zip::Unzip( GetDataDirectory().AppendASCII("Repeated File Name.zip"), test_dir_)); EXPECT_THAT( @@ -527,7 +527,7 @@ TEST_F(ZipTest, UnzipRepeatedFileName) { std::string contents; EXPECT_TRUE( base::ReadFileToString(test_dir_.AppendASCII("repeated"), &contents)); - EXPECT_EQ("Second file", contents); + EXPECT_EQ("First file", contents); } TEST_F(ZipTest, UnzipCannotCreateEmptyDir) { @@ -602,22 +602,25 @@ TEST_F(ZipTest, UnzipWindowsSpecialNames) { } TEST_F(ZipTest, UnzipDifferentCases) { - EXPECT_TRUE(zip::Unzip(GetDataDirectory().AppendASCII( - "Repeated File Name With Different Cases.zip"), - test_dir_)); - #if defined(OS_WIN) || defined(OS_MAC) - // There is only one file, with the name of the first file but the contents of - // the last file. + // Only the first file (with mixed case) is extracted. + EXPECT_FALSE(zip::Unzip(GetDataDirectory().AppendASCII( + "Repeated File Name With Different Cases.zip"), + test_dir_)); + EXPECT_THAT( GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), UnorderedElementsAre("Case")); std::string contents; EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("Case"), &contents)); - EXPECT_EQ("Upper case 3", contents); + EXPECT_EQ("Mixed case 111", contents); #else - // All the files are correctly extracted. + // All the files are extracted. + EXPECT_TRUE(zip::Unzip(GetDataDirectory().AppendASCII( + "Repeated File Name With Different Cases.zip"), + test_dir_)); + EXPECT_THAT( GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), UnorderedElementsAre("Case", "case", "CASE")); -- cgit v1.2.3 From 8b4114dfa1faa320faae92a354f84883f88b46bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Degros?= Date: Fri, 18 Mar 2022 11:52:59 +0000 Subject: [zip] Add UnzipOptions::continue_on_error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added an option allowing the ZIP extraction to continue and try to extract as many files as possible even if some file extractions fail. BUG=chromium:953256 TEST=autoninja -C out/Default zlib_unittests && out/Default/zlib_unittests Change-Id: I2ddbfbdb3abe5d0cca3023d93fdff7fcdfc9baab Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3523968 Reviewed-by: Alex Danilo Commit-Queue: François Degros Cr-Commit-Position: refs/heads/main@{#982666} NOKEYCHECK=True GitOrigin-RevId: abc9bcb7f1c7757409f4f0ef6e1089c0276bb7e7 --- google/test/data/Mixed Paths.zip | Bin 0 -> 9793 bytes google/zip.cc | 10 +- google/zip.h | 3 + google/zip_unittest.cc | 200 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 205 insertions(+), 8 deletions(-) create mode 100644 google/test/data/Mixed Paths.zip diff --git a/google/test/data/Mixed Paths.zip b/google/test/data/Mixed Paths.zip new file mode 100644 index 0000000..2af418b Binary files /dev/null and b/google/test/data/Mixed Paths.zip differ diff --git a/google/zip.cc b/google/zip.cc index 7f9af5b..1a43196 100644 --- a/google/zip.cc +++ b/google/zip.cc @@ -206,7 +206,9 @@ bool Unzip(const base::PlatformFile& src_file, while (const ZipReader::Entry* const entry = reader.Next()) { if (entry->is_unsafe) { LOG(ERROR) << "Found unsafe entry " << Redact(entry->path) << " in ZIP"; - return false; + if (!options.continue_on_error) + return false; + continue; } if (options.filter && !options.filter.Run(entry->path)) { @@ -218,7 +220,8 @@ bool Unzip(const base::PlatformFile& src_file, // It's a directory. if (!directory_creator.Run(entry->path)) { LOG(ERROR) << "Cannot create directory " << Redact(entry->path); - return false; + if (!options.continue_on_error) + return false; } continue; @@ -229,7 +232,8 @@ bool Unzip(const base::PlatformFile& src_file, if (!writer || !reader.ExtractCurrentEntry(writer.get())) { LOG(ERROR) << "Cannot extract file " << Redact(entry->path) << " from ZIP"; - return false; + if (!options.continue_on_error) + return false; } } diff --git a/google/zip.h b/google/zip.h index 621f0d9..25ec655 100644 --- a/google/zip.h +++ b/google/zip.h @@ -182,6 +182,9 @@ struct UnzipOptions { // Password to decrypt the encrypted files. std::string password; + + // Should ignore errors when extracting files? + bool continue_on_error = false; }; typedef base::RepeatingCallback( diff --git a/google/zip_unittest.cc b/google/zip_unittest.cc index cf54991..8fbec32 100644 --- a/google/zip_unittest.cc +++ b/google/zip_unittest.cc @@ -471,8 +471,9 @@ TEST_F(ZipTest, UnzipEncryptedWithWrongPassword) { EXPECT_EQ("This is not encrypted.\n", contents); // No rubbish file should be left behind. - EXPECT_FALSE( - base::PathExists(test_dir_.AppendASCII("Encrypted ZipCrypto.txt"))); + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("ClearText.txt")); } TEST_F(ZipTest, UnzipEncryptedWithNoPassword) { @@ -491,8 +492,25 @@ TEST_F(ZipTest, UnzipEncryptedWithNoPassword) { EXPECT_EQ("This is not encrypted.\n", contents); // No rubbish file should be left behind. - EXPECT_FALSE( - base::PathExists(test_dir_.AppendASCII("Encrypted ZipCrypto.txt"))); + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("ClearText.txt")); +} + +TEST_F(ZipTest, UnzipEncryptedContinueOnError) { + EXPECT_TRUE( + zip::Unzip(GetDataDirectory().AppendASCII("Different Encryptions.zip"), + test_dir_, {.continue_on_error = true})); + + std::string contents; + EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("ClearText.txt"), + &contents)); + EXPECT_EQ("This is not encrypted.\n", contents); + + // No rubbish file should be left behind. + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("ClearText.txt")); } TEST_F(ZipTest, UnzipWrongCrc) { @@ -500,7 +518,9 @@ TEST_F(ZipTest, UnzipWrongCrc) { zip::Unzip(GetDataDirectory().AppendASCII("Wrong CRC.zip"), test_dir_)); // No rubbish file should be left behind. - EXPECT_FALSE(base::PathExists(test_dir_.AppendASCII("Corrupted.txt"))); + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre()); } TEST_F(ZipTest, UnzipRepeatedDirName) { @@ -637,6 +657,148 @@ TEST_F(ZipTest, UnzipDifferentCases) { #endif } +TEST_F(ZipTest, UnzipDifferentCasesContinueOnError) { + EXPECT_TRUE(zip::Unzip(GetDataDirectory().AppendASCII( + "Repeated File Name With Different Cases.zip"), + test_dir_, {.continue_on_error = true})); + + std::string contents; + +#if defined(OS_WIN) || defined(OS_MAC) + // Only the first file (with mixed case) has been extracted. + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("Case")); + + EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("Case"), &contents)); + EXPECT_EQ("Mixed case 111", contents); +#else + // All the files have been extracted. + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES), + UnorderedElementsAre("Case", "case", "CASE")); + + EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("Case"), &contents)); + EXPECT_EQ("Mixed case 111", contents); + + EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("case"), &contents)); + EXPECT_EQ("Lower case 22", contents); + + EXPECT_TRUE(base::ReadFileToString(test_dir_.AppendASCII("CASE"), &contents)); + EXPECT_EQ("Upper case 3", contents); +#endif +} + +TEST_F(ZipTest, UnzipMixedPaths) { + EXPECT_TRUE(zip::Unzip(GetDataDirectory().AppendASCII("Mixed Paths.zip"), + test_dir_, {.continue_on_error = true})); + + std::unordered_set want_paths = { +#ifdef OS_WIN + "Dot", // + "Space→", // + "a\\b", // + "u\\v\\w\\x\\y\\z", // + "←Backslash2", // +#else + " ", // Invalid on Windows + "Angle <>", // Invalid on Windows + "Backslash1→\\", // + "Backspace \x08", // Invalid on Windows + "Bell \a", // Invalid on Windows + "C:", // Invalid on Windows + "C:\\", // Absolute path on Windows + "C:\\Temp", // Absolute path on Windows + "C:\\Temp\\", // Absolute path on Windows + "C:\\Temp\\File", // Absolute path on Windows + "Carriage Return \r", // Invalid on Windows + "Colon :", // Invalid on Windows + "Dot .", // Becomes "Dot" on Windows + "Double quote \"", // Invalid on Windows + "Escape \x1B", // Invalid on Windows + "Line Feed \n", // Invalid on Windows + "NUL .txt", // Disappears on Windows + "NUL", // Disappears on Windows + "NUL..txt", // Disappears on Windows + "NUL.tar.gz", // Disappears on Windows + "NUL.txt", // Disappears on Windows + "Pipe |", // Invalid on Windows + "Question ?", // Invalid on Windows + "Space→ ", // Becomes "Space→" on Windows + "Star *", // Invalid on Windows + "Tab \t", // Invalid on Windows + "\\\\server\\share\\file", // Absolute path on Windows + "\\←Backslash2", // Becomes "←Backslash2" on Windows + "a/b", // + "u/v/w/x/y/z", // +#ifndef OS_MAC + "CASE", // + "Case", // +#endif +#endif + " NUL.txt", // + " ←Space", // + "$HOME", // + "%TMP", // + "-", // + "...Tree", // + "..Two", // + ".One", // + "Ampersand &", // + "At @", // + "Backslash3→\\←Backslash4", // + "Backtick `", // + "Caret ^", // + "Comma ,", // + "Curly {}", // + "Dash -", // + "Delete \x7F", // + "Dollar $", // + "Equal =", // + "Euro €", // + "Exclamation !", // + "FileOrDir", // + "First", // + "Hash #", // + "Last", // + "Percent %", // + "Plus +", // + "Quote '", // + "Round ()", // + "Semicolon ;", // + "Smile \U0001F642", // + "Square []", // + "String Terminator \u009C", // + "Tilde ~", // + "Underscore _", // + "case", // + "~", // + }; + + const std::vector got_paths = + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::FILES); + + for (const std::string& path : got_paths) { + EXPECT_TRUE(want_paths.erase(path)) + << "Found unexpected file: " << std::quoted(path); + } + + for (const std::string& path : want_paths) { + EXPECT_TRUE(false) << "Cannot find expected file: " << std::quoted(path); + } + + EXPECT_THAT( + GetRelativePaths(test_dir_, base::FileEnumerator::FileType::DIRECTORIES), + UnorderedElementsAre( +#ifdef OS_WIN + "Backslash3→", "Empty", "a", "u", "u\\v", "u\\v\\w", "u\\v\\w\\x", + "u\\v\\w\\x\\y" +#else + "Empty", "a", "u", "u/v", "u/v/w", "u/v/w/x", "u/v/w/x/y" +#endif + )); +} + TEST_F(ZipTest, UnzipWithDelegates) { auto dir_creator = base::BindLambdaForTesting([this](const base::FilePath& entry_path) { @@ -664,6 +826,34 @@ TEST_F(ZipTest, UnzipWithDelegates) { EXPECT_TRUE(base::PathExists(dir_foo_bar.AppendASCII("quux.txt"))); } +TEST_F(ZipTest, UnzipOnlyDirectories) { + auto dir_creator = + base::BindLambdaForTesting([this](const base::FilePath& entry_path) { + return base::CreateDirectory(test_dir_.Append(entry_path)); + }); + + // Always return a null WriterDelegate. + auto writer = + base::BindLambdaForTesting([](const base::FilePath& entry_path) { + return std::unique_ptr(); + }); + + base::File file(GetDataDirectory().AppendASCII("test.zip"), + base::File::Flags::FLAG_OPEN | base::File::Flags::FLAG_READ); + EXPECT_TRUE(zip::Unzip(file.GetPlatformFile(), writer, dir_creator, + {.continue_on_error = true})); + base::FilePath dir = test_dir_; + base::FilePath dir_foo = dir.AppendASCII("foo"); + base::FilePath dir_foo_bar = dir_foo.AppendASCII("bar"); + EXPECT_FALSE(base::PathExists(dir.AppendASCII("foo.txt"))); + EXPECT_TRUE(base::DirectoryExists(dir_foo)); + EXPECT_FALSE(base::PathExists(dir_foo.AppendASCII("bar.txt"))); + EXPECT_TRUE(base::DirectoryExists(dir_foo_bar)); + EXPECT_FALSE(base::PathExists(dir_foo_bar.AppendASCII(".hidden"))); + EXPECT_FALSE(base::PathExists(dir_foo_bar.AppendASCII("baz.txt"))); + EXPECT_FALSE(base::PathExists(dir_foo_bar.AppendASCII("quux.txt"))); +} + // Tests that a ZIP archive containing SJIS-encoded file names can be correctly // extracted if the encoding is specified. TEST_F(ZipTest, UnzipSjis) { -- cgit v1.2.3 From 16b8c02648faf08794ed0048c0fb1427909e9a46 Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Mon, 21 Mar 2022 19:44:06 +0000 Subject: Avoid including base/files/file_util.h in some headers. Rarely would anyone ever use base/files/file_util.h functions inside a header. Most code authors meant to include base/files/file_path.h or forward declare base::FilePath. Do that and then add missing file_util.h includes to fix the build. Bug: 242216 Change-Id: I6280125f310cff0db1109c401d84e01aba386376 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3518311 Reviewed-by: Nico Weber Commit-Queue: Nico Weber Owners-Override: Nico Weber Cr-Commit-Position: refs/heads/main@{#983450} NOKEYCHECK=True GitOrigin-RevId: 527964a214baed4f325c0d7751bfc527ba10bf34 --- google/zip_reader.cc | 1 + google/zip_reader.h | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/google/zip_reader.cc b/google/zip_reader.cc index 93fe134..b8143cd 100644 --- a/google/zip_reader.cc +++ b/google/zip_reader.cc @@ -10,6 +10,7 @@ #include "base/bind.h" #include "base/check.h" #include "base/files/file.h" +#include "base/files/file_util.h" #include "base/i18n/icu_string_conversions.h" #include "base/logging.h" #include "base/numerics/safe_conversions.h" diff --git a/google/zip_reader.h b/google/zip_reader.h index edf548b..df7452a 100644 --- a/google/zip_reader.h +++ b/google/zip_reader.h @@ -14,7 +14,6 @@ #include "base/callback.h" #include "base/files/file.h" #include "base/files/file_path.h" -#include "base/files/file_util.h" #include "base/memory/weak_ptr.h" #include "base/numerics/safe_conversions.h" #include "base/time/time.h" -- cgit v1.2.3 From cb89dc607cd4b85d98867f14216c3370109be64d Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Wed, 23 Mar 2022 23:16:54 +0000 Subject: Fix out-of-bounds in infcover tests. Upstream zlib pull request: https://github.com/madler/zlib/pull/602 Bug: 225069280 Change-Id: I548a836096323053af225083916893b851532980 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3546708 Reviewed-by: Noel Gordon Commit-Queue: Noel Gordon Cr-Commit-Position: refs/heads/main@{#984579} NOKEYCHECK=True GitOrigin-RevId: afa134512eec0ab985c64bc39e39b0bb5212f8f6 --- contrib/tests/infcover.cc | 4 +++- patches/0009-infcover-oob.patch | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 patches/0009-infcover-oob.patch diff --git a/contrib/tests/infcover.cc b/contrib/tests/infcover.cc index c5300a5..16dd744 100644 --- a/contrib/tests/infcover.cc +++ b/contrib/tests/infcover.cc @@ -395,7 +395,9 @@ void cover_support(void) mem_setup(&strm); strm.avail_in = 0; strm.next_in = Z_NULL; - ret = inflateInit_(&strm, ZLIB_VERSION - 1, (int)sizeof(z_stream)); + char versioncpy[] = ZLIB_VERSION; + versioncpy[0] -= 1; + ret = inflateInit_(&strm, versioncpy, (int)sizeof(z_stream)); assert(ret == Z_VERSION_ERROR); mem_done(&strm, "wrong version"); diff --git a/patches/0009-infcover-oob.patch b/patches/0009-infcover-oob.patch new file mode 100644 index 0000000..648360f --- /dev/null +++ b/patches/0009-infcover-oob.patch @@ -0,0 +1,24 @@ +From 75690b2683667be5535ac6243438115dc9c40f6a Mon Sep 17 00:00:00 2001 +From: Florian Mayer +Date: Wed, 16 Mar 2022 16:38:36 -0700 +Subject: [PATCH] Fix out of bounds in infcover.c. + +--- + test/infcover.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/test/infcover.c b/test/infcover.c +index 2be01646c..a6d83693c 100644 +--- a/test/infcover.c ++++ b/test/infcover.c +@@ -373,7 +373,9 @@ local void cover_support(void) + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; +- ret = inflateInit_(&strm, ZLIB_VERSION - 1, (int)sizeof(z_stream)); ++ char versioncpy[] = ZLIB_VERSION; ++ versioncpy[0] -= 1; ++ ret = inflateInit_(&strm, versioncpy, (int)sizeof(z_stream)); + assert(ret == Z_VERSION_ERROR); + mem_done(&strm, "wrong version"); + -- cgit v1.2.3