aboutsummaryrefslogtreecommitdiff
path: root/icing/file
diff options
context:
space:
mode:
authorJiayu Hu <hujiayu@google.com>2023-04-19 12:31:23 -0700
committerJiayu Hu <hujiayu@google.com>2023-04-19 12:33:32 -0700
commita7d57e98ea7168d66cf01ace85598e33d5e9e5db (patch)
tree6685971fd4a027f52d674ee646a79ab41d4ce24f /icing/file
parentb59049b030cc330b6c8ae1d03ea4c1a34235ac9b (diff)
downloadicing-a7d57e98ea7168d66cf01ace85598e33d5e9e5db.tar.gz
Update Icing from upstream.
Descriptions: ======================================================================== Adds join info to QueryStatsProto ======================================================================== [Join][cleanup][1/x] Add Icing benchmark for qualified id join ======================================================================== [NumericSearch][benchmark][2/x] Modify run-benchmarks.py to onboard numeric search benchmark ======================================================================== [NumericSearch][benchmark][1/x] Add numeric search icing search engine benchmark ======================================================================== Switch Iterator access type used in SnippetRetriever back to kForwardIterator. ======================================================================== Fix `DocumentStore::OptimizeInto` by passing `options_.document_store_namespace_id_fingerprint()` ======================================================================== Add file size check when initializing mmap for existing metadata file. ======================================================================== Add compression level option to Icing and propagate down to PortableFileBackedProtoLog ======================================================================== Add portable_equals_proto namespace for EqualsProto in tokenized-document_test. ======================================================================== Refactor schema to check consistency of polymorphism by dependency map ======================================================================== Support the `this.propertyWeights()` function in advanced scoring ======================================================================== Randomly set optimize_rebuild_index_threshold in monkey test ======================================================================== Change benchmark batch query latency to average ======================================================================== Performance improvements to SnippetRetriever. ======================================================================== Bug: 270102295 Bug: 260625837 Bug: 269295094 Bug: 193244409 Bug: 269513147 Bug: 276345448 Bug: 259744228 Bug: 272145329 Bug: 276349029 Change-Id: Ief1cb2c21f67aac420d6b79d9b946bbecdc39a5c
Diffstat (limited to 'icing/file')
-rw-r--r--icing/file/portable-file-backed-proto-log.h60
-rw-r--r--icing/file/portable-file-backed-proto-log_test.cc296
2 files changed, 282 insertions, 74 deletions
diff --git a/icing/file/portable-file-backed-proto-log.h b/icing/file/portable-file-backed-proto-log.h
index 48e3501..a36bd9e 100644
--- a/icing/file/portable-file-backed-proto-log.h
+++ b/icing/file/portable-file-backed-proto-log.h
@@ -106,13 +106,32 @@ class PortableFileBackedProtoLog {
// compressed size larger than max_proto_size are also not accepted.
const int32_t max_proto_size;
+ // Level of compression if enabled, NO_COMPRESSION = 0, BEST_SPEED = 1,
+ // BEST_COMPRESSION = 9
+ const int32_t compression_level;
+
// Must specify values for options.
Options() = delete;
- explicit Options(bool compress_in,
- const int32_t max_proto_size_in = kMaxProtoSize)
- : compress(compress_in), max_proto_size(max_proto_size_in) {}
+ explicit Options(
+ bool compress_in, const int32_t max_proto_size_in = kMaxProtoSize,
+ const int32_t compression_level_in = kDeflateCompressionLevel)
+ : compress(compress_in),
+ max_proto_size(max_proto_size_in),
+ compression_level(compression_level_in) {}
};
+ // Our internal max for protos.
+ //
+ // WARNING: Changing this to a larger number may invalidate our assumption
+ // that that proto size can safely be stored in the last 3 bytes of the proto
+ // header.
+ static constexpr int kMaxProtoSize = (1 << 24) - 1; // 16MiB
+ static_assert(kMaxProtoSize <= 0x00FFFFFF,
+ "kMaxProtoSize doesn't fit in 3 bytes");
+
+ // Level of compression, BEST_SPEED = 1, BEST_COMPRESSION = 9
+ static constexpr int kDeflateCompressionLevel = 3;
+
// Number of bytes we reserve for the heading at the beginning of the proto
// log. We reserve this so the header can grow without running into the
// contents of the proto log, triggering an unnecessary migration of the data.
@@ -480,7 +499,8 @@ class PortableFileBackedProtoLog {
// Object can only be instantiated via the ::Create factory.
PortableFileBackedProtoLog(const Filesystem* filesystem,
const std::string& file_path,
- std::unique_ptr<Header> header);
+ std::unique_ptr<Header> header,
+ int32_t compression_level);
// Initializes a new proto log.
//
@@ -556,18 +576,6 @@ class PortableFileBackedProtoLog {
// protos we support.
static constexpr uint8_t kProtoMagic = 0x5C;
- // Our internal max for protos.
- //
- // WARNING: Changing this to a larger number may invalidate our assumption
- // that that proto size can safely be stored in the last 3 bytes of the proto
- // header.
- static constexpr int kMaxProtoSize = (1 << 24) - 1; // 16MiB
- static_assert(kMaxProtoSize <= 0x00FFFFFF,
- "kMaxProtoSize doesn't fit in 3 bytes");
-
- // Level of compression, BEST_SPEED = 1, BEST_COMPRESSION = 9
- static constexpr int kDeflateCompressionLevel = 3;
-
// Chunks of the file to mmap at a time, so we don't mmap the entire file.
// Only used on 32-bit devices
static constexpr int kMmapChunkSize = 4 * 1024 * 1024; // 4MiB
@@ -576,15 +584,17 @@ class PortableFileBackedProtoLog {
const Filesystem* const filesystem_;
const std::string file_path_;
std::unique_ptr<Header> header_;
+ const int32_t compression_level_;
};
template <typename ProtoT>
PortableFileBackedProtoLog<ProtoT>::PortableFileBackedProtoLog(
const Filesystem* filesystem, const std::string& file_path,
- std::unique_ptr<Header> header)
+ std::unique_ptr<Header> header, int32_t compression_level)
: filesystem_(filesystem),
file_path_(file_path),
- header_(std::move(header)) {
+ header_(std::move(header)),
+ compression_level_(compression_level) {
fd_.reset(filesystem_->OpenForAppend(file_path.c_str()));
}
@@ -617,6 +627,12 @@ PortableFileBackedProtoLog<ProtoT>::Create(const Filesystem* filesystem,
options.max_proto_size));
}
+ if (options.compression_level < 0 || options.compression_level > 9) {
+ return absl_ports::InvalidArgumentError(IcingStringUtil::StringPrintf(
+ "options.compression_level must be between 0 and 9 inclusive, was %d",
+ options.compression_level));
+ }
+
if (!filesystem->FileExists(file_path.c_str())) {
return InitializeNewFile(filesystem, file_path, options);
}
@@ -660,7 +676,8 @@ PortableFileBackedProtoLog<ProtoT>::InitializeNewFile(
CreateResult create_result = {
std::unique_ptr<PortableFileBackedProtoLog<ProtoT>>(
new PortableFileBackedProtoLog<ProtoT>(filesystem, file_path,
- std::move(header))),
+ std::move(header),
+ options.compression_level)),
/*data_loss=*/DataLoss::NONE, /*recalculated_checksum=*/false};
return create_result;
@@ -788,7 +805,8 @@ PortableFileBackedProtoLog<ProtoT>::InitializeExistingFile(
CreateResult create_result = {
std::unique_ptr<PortableFileBackedProtoLog<ProtoT>>(
new PortableFileBackedProtoLog<ProtoT>(filesystem, file_path,
- std::move(header))),
+ std::move(header),
+ options.compression_level)),
data_loss, recalculated_checksum};
return create_result;
@@ -893,7 +911,7 @@ PortableFileBackedProtoLog<ProtoT>::WriteProto(const ProtoT& proto) {
if (header_->GetCompressFlag()) {
protobuf_ports::GzipOutputStream::Options options;
options.format = protobuf_ports::GzipOutputStream::ZLIB;
- options.compression_level = kDeflateCompressionLevel;
+ options.compression_level = compression_level_;
protobuf_ports::GzipOutputStream compressing_stream(&proto_stream, options);
diff --git a/icing/file/portable-file-backed-proto-log_test.cc b/icing/file/portable-file-backed-proto-log_test.cc
index af09d18..bf5e604 100644
--- a/icing/file/portable-file-backed-proto-log_test.cc
+++ b/icing/file/portable-file-backed-proto-log_test.cc
@@ -73,24 +73,18 @@ class PortableFileBackedProtoLogTest : public ::testing::Test {
const Filesystem filesystem_;
std::string file_path_;
bool compress_ = true;
+ int32_t compression_level_ =
+ PortableFileBackedProtoLog<DocumentProto>::kDeflateCompressionLevel;
int64_t max_proto_size_ = 256 * 1024; // 256 KiB
};
TEST_F(PortableFileBackedProtoLogTest, Initialize) {
- // max_proto_size must be greater than 0
- int invalid_max_proto_size = 0;
- ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create(
- &filesystem_, file_path_,
- PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, invalid_max_proto_size)),
- StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT));
-
ICING_ASSERT_OK_AND_ASSIGN(
PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
- PortableFileBackedProtoLog<DocumentProto>::Options(compress_,
- max_proto_size_)));
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ compress_, max_proto_size_, compression_level_)));
EXPECT_THAT(create_result.proto_log, NotNull());
EXPECT_FALSE(create_result.has_data_loss());
EXPECT_FALSE(create_result.recalculated_checksum);
@@ -99,7 +93,41 @@ TEST_F(PortableFileBackedProtoLogTest, Initialize) {
ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- !compress_, max_proto_size_)),
+ !compress_, max_proto_size_, compression_level_)),
+ StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT));
+}
+
+TEST_F(PortableFileBackedProtoLogTest, InitializeValidatesOptions) {
+ // max_proto_size must be greater than 0
+ int invalid_max_proto_size = 0;
+ ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create(
+ &filesystem_, file_path_,
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ compress_, invalid_max_proto_size, compression_level_)),
+ StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT));
+
+ // max_proto_size must be under 16 MiB
+ invalid_max_proto_size = 16 * 1024 * 1024;
+ ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create(
+ &filesystem_, file_path_,
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ compress_, invalid_max_proto_size, compression_level_)),
+ StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT));
+
+ // compression_level must be between 0 and 9 inclusive
+ int invalid_compression_level = -1;
+ ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create(
+ &filesystem_, file_path_,
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ compress_, max_proto_size_, invalid_compression_level)),
+ StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT));
+
+ // compression_level must be between 0 and 9 inclusive
+ invalid_compression_level = 10;
+ ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create(
+ &filesystem_, file_path_,
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ compress_, max_proto_size_, invalid_compression_level)),
StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT));
}
@@ -108,8 +136,8 @@ TEST_F(PortableFileBackedProtoLogTest, ReservedSpaceForHeader) {
PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
- PortableFileBackedProtoLog<DocumentProto>::Options(compress_,
- max_proto_size_)));
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ compress_, max_proto_size_, compression_level_)));
// With no protos written yet, the log should be minimum the size of the
// reserved header space.
@@ -123,8 +151,8 @@ TEST_F(PortableFileBackedProtoLogTest, WriteProtoTooLarge) {
PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
- PortableFileBackedProtoLog<DocumentProto>::Options(compress_,
- max_proto_size)));
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ compress_, max_proto_size, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -140,8 +168,8 @@ TEST_F(PortableFileBackedProtoLogTest, ReadProtoWrongKProtoMagic) {
PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
- PortableFileBackedProtoLog<DocumentProto>::Options(compress_,
- max_proto_size_)));
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -175,7 +203,7 @@ TEST_F(PortableFileBackedProtoLogTest, ReadWriteUncompressedProto) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- /*compress_in=*/false, max_proto_size_)));
+ /*compress_in=*/false, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -222,7 +250,7 @@ TEST_F(PortableFileBackedProtoLogTest, ReadWriteUncompressedProto) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- /*compress_in=*/false, max_proto_size_)));
+ /*compress_in=*/false, max_proto_size_, compression_level_)));
auto recreated_proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -244,7 +272,7 @@ TEST_F(PortableFileBackedProtoLogTest, ReadWriteCompressedProto) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- /*compress_in=*/true, max_proto_size_)));
+ /*compress_in=*/true, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -291,7 +319,7 @@ TEST_F(PortableFileBackedProtoLogTest, ReadWriteCompressedProto) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- /*compress_in=*/true, max_proto_size_)));
+ /*compress_in=*/true, max_proto_size_, compression_level_)));
auto recreated_proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -304,6 +332,168 @@ TEST_F(PortableFileBackedProtoLogTest, ReadWriteCompressedProto) {
}
}
+TEST_F(PortableFileBackedProtoLogTest, ReadWriteDifferentCompressionLevel) {
+ int document1_offset;
+ int document2_offset;
+ int document3_offset;
+
+ // The first proto to write that's close to the max size. Leave some room for
+ // the rest of the proto properties.
+ std::string long_str(max_proto_size_ - 1024, 'a');
+ DocumentProto document1 = DocumentBuilder()
+ .SetKey("namespace1", "uri1")
+ .AddStringProperty("long_str", long_str)
+ .Build();
+ DocumentProto document2 =
+ DocumentBuilder().SetKey("namespace2", "uri2").Build();
+ DocumentProto document3 =
+ DocumentBuilder().SetKey("namespace3", "uri3").Build();
+
+ {
+ ICING_ASSERT_OK_AND_ASSIGN(
+ PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
+ PortableFileBackedProtoLog<DocumentProto>::Create(
+ &filesystem_, file_path_,
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ /*compress_in=*/true, max_proto_size_,
+ /*compression_level_in=*/3)));
+ auto proto_log = std::move(create_result.proto_log);
+ ASSERT_FALSE(create_result.has_data_loss());
+
+ // Write the first proto
+ ICING_ASSERT_OK_AND_ASSIGN(document1_offset,
+ proto_log->WriteProto(document1));
+
+ // Check that what we read is what we wrote
+ ASSERT_THAT(proto_log->ReadProto(document1_offset),
+ IsOkAndHolds(EqualsProto(document1)));
+
+ ICING_ASSERT_OK(proto_log->PersistToDisk());
+ }
+
+ // Make a new proto_log with the same file_path but different compression
+ // level, and make sure we can still read from and write to the same
+ // underlying file.
+ {
+ ICING_ASSERT_OK_AND_ASSIGN(
+ PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
+ PortableFileBackedProtoLog<DocumentProto>::Create(
+ &filesystem_, file_path_,
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ /*compress_in=*/true, max_proto_size_,
+ /*compression_level_in=*/9)));
+ auto recreated_proto_log = std::move(create_result.proto_log);
+ ASSERT_FALSE(create_result.has_data_loss());
+
+ // Check the first proto
+ ASSERT_THAT(recreated_proto_log->ReadProto(document1_offset),
+ IsOkAndHolds(EqualsProto(document1)));
+
+ // Write a second proto
+ ICING_ASSERT_OK_AND_ASSIGN(document2_offset,
+ recreated_proto_log->WriteProto(document2));
+
+ ASSERT_GT(document2_offset, document1_offset);
+
+ // Check the second proto
+ ASSERT_THAT(recreated_proto_log->ReadProto(document2_offset),
+ IsOkAndHolds(EqualsProto(document2)));
+
+ ICING_ASSERT_OK(recreated_proto_log->PersistToDisk());
+ }
+
+ // One more time but with 0 compression level
+ {
+ ICING_ASSERT_OK_AND_ASSIGN(
+ PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
+ PortableFileBackedProtoLog<DocumentProto>::Create(
+ &filesystem_, file_path_,
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ /*compress_in=*/true, max_proto_size_,
+ /*compression_level=*/0)));
+ auto recreated_proto_log = std::move(create_result.proto_log);
+ ASSERT_FALSE(create_result.has_data_loss());
+
+ // Check the first proto
+ ASSERT_THAT(recreated_proto_log->ReadProto(document1_offset),
+ IsOkAndHolds(EqualsProto(document1)));
+
+ // Check the second proto
+ ASSERT_THAT(recreated_proto_log->ReadProto(document2_offset),
+ IsOkAndHolds(EqualsProto(document2)));
+
+ // Write a third proto
+ ICING_ASSERT_OK_AND_ASSIGN(document3_offset,
+ recreated_proto_log->WriteProto(document3));
+
+ ASSERT_GT(document3_offset, document2_offset);
+
+ // Check the third proto
+ ASSERT_THAT(recreated_proto_log->ReadProto(document3_offset),
+ IsOkAndHolds(EqualsProto(document3)));
+ }
+}
+
+TEST_F(PortableFileBackedProtoLogTest,
+ WriteDifferentCompressionLevelDifferentSizes) {
+ int document_log_size_with_compression_3;
+ int document_log_size_with_no_compression;
+
+ // The first proto to write that's close to the max size. Leave some room for
+ // the rest of the proto properties.
+ std::string long_str(max_proto_size_ - 1024, 'a');
+ DocumentProto document1 = DocumentBuilder()
+ .SetKey("namespace1", "uri1")
+ .AddStringProperty("long_str", long_str)
+ .Build();
+
+ {
+ ICING_ASSERT_OK_AND_ASSIGN(
+ PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
+ PortableFileBackedProtoLog<DocumentProto>::Create(
+ &filesystem_, file_path_,
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ /*compress_in=*/true, max_proto_size_,
+ /*compression_level_in=*/3)));
+ auto proto_log = std::move(create_result.proto_log);
+ ASSERT_FALSE(create_result.has_data_loss());
+
+ // Write the proto
+ ICING_ASSERT_OK(proto_log->WriteProto(document1));
+ ICING_ASSERT_OK(proto_log->PersistToDisk());
+
+ document_log_size_with_compression_3 =
+ filesystem_.GetFileSize(file_path_.c_str());
+ }
+
+ // Delete the proto_log so we can reuse the file_path
+ filesystem_.DeleteFile(file_path_.c_str());
+
+ {
+ ICING_ASSERT_OK_AND_ASSIGN(
+ PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
+ PortableFileBackedProtoLog<DocumentProto>::Create(
+ &filesystem_, file_path_,
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ /*compress_in=*/true, max_proto_size_,
+ /*compression_level_in=*/0)));
+ auto proto_log = std::move(create_result.proto_log);
+ ASSERT_FALSE(create_result.has_data_loss());
+
+ // Write the proto
+ ICING_ASSERT_OK(proto_log->WriteProto(document1));
+ ICING_ASSERT_OK(proto_log->PersistToDisk());
+
+ document_log_size_with_no_compression =
+ filesystem_.GetFileSize(file_path_.c_str());
+
+ // Uncompressed document file size should be larger than original compressed
+ // document file size
+ ASSERT_GT(document_log_size_with_no_compression,
+ document_log_size_with_compression_3);
+ }
+}
+
TEST_F(PortableFileBackedProtoLogTest, CorruptHeader) {
{
ICING_ASSERT_OK_AND_ASSIGN(
@@ -311,7 +501,7 @@ TEST_F(PortableFileBackedProtoLogTest, CorruptHeader) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto recreated_proto_log = std::move(create_result.proto_log);
EXPECT_FALSE(create_result.has_data_loss());
}
@@ -328,7 +518,7 @@ TEST_F(PortableFileBackedProtoLogTest, CorruptHeader) {
ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)),
+ compress_, max_proto_size_, compression_level_)),
StatusIs(libtextclassifier3::StatusCode::INTERNAL,
HasSubstr("Invalid header checksum")));
}
@@ -341,7 +531,7 @@ TEST_F(PortableFileBackedProtoLogTest, DifferentMagic) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto recreated_proto_log = std::move(create_result.proto_log);
EXPECT_FALSE(create_result.has_data_loss());
@@ -360,7 +550,7 @@ TEST_F(PortableFileBackedProtoLogTest, DifferentMagic) {
ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)),
+ compress_, max_proto_size_, compression_level_)),
StatusIs(libtextclassifier3::StatusCode::INTERNAL,
HasSubstr("Invalid header kMagic")));
}
@@ -383,7 +573,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
EXPECT_FALSE(create_result.has_data_loss());
@@ -410,7 +600,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
EXPECT_FALSE(create_result.has_data_loss());
EXPECT_THAT(create_result.data_loss, Eq(DataLoss::NONE));
@@ -432,7 +622,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -478,7 +668,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
EXPECT_TRUE(create_result.has_data_loss());
EXPECT_THAT(create_result.data_loss, Eq(DataLoss::COMPLETE));
@@ -507,7 +697,7 @@ TEST_F(PortableFileBackedProtoLogTest, DirtyBitFalseAlarmKeepsData) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -537,7 +727,7 @@ TEST_F(PortableFileBackedProtoLogTest, DirtyBitFalseAlarmKeepsData) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
EXPECT_FALSE(create_result.has_data_loss());
@@ -569,7 +759,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -615,7 +805,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_TRUE(create_result.has_data_loss());
ASSERT_THAT(create_result.data_loss, Eq(DataLoss::PARTIAL));
@@ -640,7 +830,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -663,7 +853,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
// We previously persisted to disk so everything should be in a perfect
// state.
@@ -683,7 +873,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -707,7 +897,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
// We previously persisted to disk so everything should be in a perfect
// state.
@@ -726,7 +916,7 @@ TEST_F(PortableFileBackedProtoLogTest, DirtyBitIsFalseAfterPutAndDestructor) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -751,7 +941,7 @@ TEST_F(PortableFileBackedProtoLogTest, DirtyBitIsFalseAfterPutAndDestructor) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
// We previously persisted to disk so everything should be in a perfect
// state.
@@ -771,7 +961,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -797,7 +987,7 @@ TEST_F(PortableFileBackedProtoLogTest,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
// We previously persisted to disk so everything should be in a perfect
// state.
@@ -819,8 +1009,8 @@ TEST_F(PortableFileBackedProtoLogTest, Iterator) {
PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
- PortableFileBackedProtoLog<DocumentProto>::Options(compress_,
- max_proto_size_)));
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -872,7 +1062,7 @@ TEST_F(PortableFileBackedProtoLogTest, ComputeChecksum) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -890,7 +1080,7 @@ TEST_F(PortableFileBackedProtoLogTest, ComputeChecksum) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -915,8 +1105,8 @@ TEST_F(PortableFileBackedProtoLogTest, EraseProtoShouldSetZero) {
PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
- PortableFileBackedProtoLog<DocumentProto>::Options(compress_,
- max_proto_size_)));
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -950,8 +1140,8 @@ TEST_F(PortableFileBackedProtoLogTest, EraseProtoShouldReturnNotFound) {
PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result,
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
- PortableFileBackedProtoLog<DocumentProto>::Options(compress_,
- max_proto_size_)));
+ PortableFileBackedProtoLog<DocumentProto>::Options(
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -993,7 +1183,7 @@ TEST_F(PortableFileBackedProtoLogTest, ChecksumShouldBeCorrectWithErasedProto) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -1021,7 +1211,7 @@ TEST_F(PortableFileBackedProtoLogTest, ChecksumShouldBeCorrectWithErasedProto) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -1041,7 +1231,7 @@ TEST_F(PortableFileBackedProtoLogTest, ChecksumShouldBeCorrectWithErasedProto) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
ASSERT_FALSE(create_result.has_data_loss());
@@ -1063,7 +1253,7 @@ TEST_F(PortableFileBackedProtoLogTest, ChecksumShouldBeCorrectWithErasedProto) {
PortableFileBackedProtoLog<DocumentProto>::Create(
&filesystem_, file_path_,
PortableFileBackedProtoLog<DocumentProto>::Options(
- compress_, max_proto_size_)));
+ compress_, max_proto_size_, compression_level_)));
auto proto_log = std::move(create_result.proto_log);
EXPECT_FALSE(create_result.has_data_loss());
}