diff options
author | Jiayu Hu <hujiayu@google.com> | 2023-04-19 12:31:23 -0700 |
---|---|---|
committer | Jiayu Hu <hujiayu@google.com> | 2023-04-19 12:33:32 -0700 |
commit | a7d57e98ea7168d66cf01ace85598e33d5e9e5db (patch) | |
tree | 6685971fd4a027f52d674ee646a79ab41d4ce24f /icing/file | |
parent | b59049b030cc330b6c8ae1d03ea4c1a34235ac9b (diff) | |
download | icing-a7d57e98ea7168d66cf01ace85598e33d5e9e5db.tar.gz |
Update Icing from upstream.
Descriptions:
========================================================================
Adds join info to QueryStatsProto
========================================================================
[Join][cleanup][1/x] Add Icing benchmark for qualified id join
========================================================================
[NumericSearch][benchmark][2/x] Modify run-benchmarks.py to onboard numeric search benchmark
========================================================================
[NumericSearch][benchmark][1/x] Add numeric search icing search engine benchmark
========================================================================
Switch Iterator access type used in SnippetRetriever back to kForwardIterator.
========================================================================
Fix `DocumentStore::OptimizeInto` by passing `options_.document_store_namespace_id_fingerprint()`
========================================================================
Add file size check when initializing mmap for existing metadata file.
========================================================================
Add compression level option to Icing and propagate down to PortableFileBackedProtoLog
========================================================================
Add portable_equals_proto namespace for EqualsProto in tokenized-document_test.
========================================================================
Refactor schema to check consistency of polymorphism by dependency map
========================================================================
Support the `this.propertyWeights()` function in advanced scoring
========================================================================
Randomly set optimize_rebuild_index_threshold in monkey test
========================================================================
Change benchmark batch query latency to average
========================================================================
Performance improvements to SnippetRetriever.
========================================================================
Bug: 270102295
Bug: 260625837
Bug: 269295094
Bug: 193244409
Bug: 269513147
Bug: 276345448
Bug: 259744228
Bug: 272145329
Bug: 276349029
Change-Id: Ief1cb2c21f67aac420d6b79d9b946bbecdc39a5c
Diffstat (limited to 'icing/file')
-rw-r--r-- | icing/file/portable-file-backed-proto-log.h | 60 | ||||
-rw-r--r-- | icing/file/portable-file-backed-proto-log_test.cc | 296 |
2 files changed, 282 insertions, 74 deletions
diff --git a/icing/file/portable-file-backed-proto-log.h b/icing/file/portable-file-backed-proto-log.h index 48e3501..a36bd9e 100644 --- a/icing/file/portable-file-backed-proto-log.h +++ b/icing/file/portable-file-backed-proto-log.h @@ -106,13 +106,32 @@ class PortableFileBackedProtoLog { // compressed size larger than max_proto_size are also not accepted. const int32_t max_proto_size; + // Level of compression if enabled, NO_COMPRESSION = 0, BEST_SPEED = 1, + // BEST_COMPRESSION = 9 + const int32_t compression_level; + // Must specify values for options. Options() = delete; - explicit Options(bool compress_in, - const int32_t max_proto_size_in = kMaxProtoSize) - : compress(compress_in), max_proto_size(max_proto_size_in) {} + explicit Options( + bool compress_in, const int32_t max_proto_size_in = kMaxProtoSize, + const int32_t compression_level_in = kDeflateCompressionLevel) + : compress(compress_in), + max_proto_size(max_proto_size_in), + compression_level(compression_level_in) {} }; + // Our internal max for protos. + // + // WARNING: Changing this to a larger number may invalidate our assumption + // that that proto size can safely be stored in the last 3 bytes of the proto + // header. + static constexpr int kMaxProtoSize = (1 << 24) - 1; // 16MiB + static_assert(kMaxProtoSize <= 0x00FFFFFF, + "kMaxProtoSize doesn't fit in 3 bytes"); + + // Level of compression, BEST_SPEED = 1, BEST_COMPRESSION = 9 + static constexpr int kDeflateCompressionLevel = 3; + // Number of bytes we reserve for the heading at the beginning of the proto // log. We reserve this so the header can grow without running into the // contents of the proto log, triggering an unnecessary migration of the data. @@ -480,7 +499,8 @@ class PortableFileBackedProtoLog { // Object can only be instantiated via the ::Create factory. PortableFileBackedProtoLog(const Filesystem* filesystem, const std::string& file_path, - std::unique_ptr<Header> header); + std::unique_ptr<Header> header, + int32_t compression_level); // Initializes a new proto log. // @@ -556,18 +576,6 @@ class PortableFileBackedProtoLog { // protos we support. static constexpr uint8_t kProtoMagic = 0x5C; - // Our internal max for protos. - // - // WARNING: Changing this to a larger number may invalidate our assumption - // that that proto size can safely be stored in the last 3 bytes of the proto - // header. - static constexpr int kMaxProtoSize = (1 << 24) - 1; // 16MiB - static_assert(kMaxProtoSize <= 0x00FFFFFF, - "kMaxProtoSize doesn't fit in 3 bytes"); - - // Level of compression, BEST_SPEED = 1, BEST_COMPRESSION = 9 - static constexpr int kDeflateCompressionLevel = 3; - // Chunks of the file to mmap at a time, so we don't mmap the entire file. // Only used on 32-bit devices static constexpr int kMmapChunkSize = 4 * 1024 * 1024; // 4MiB @@ -576,15 +584,17 @@ class PortableFileBackedProtoLog { const Filesystem* const filesystem_; const std::string file_path_; std::unique_ptr<Header> header_; + const int32_t compression_level_; }; template <typename ProtoT> PortableFileBackedProtoLog<ProtoT>::PortableFileBackedProtoLog( const Filesystem* filesystem, const std::string& file_path, - std::unique_ptr<Header> header) + std::unique_ptr<Header> header, int32_t compression_level) : filesystem_(filesystem), file_path_(file_path), - header_(std::move(header)) { + header_(std::move(header)), + compression_level_(compression_level) { fd_.reset(filesystem_->OpenForAppend(file_path.c_str())); } @@ -617,6 +627,12 @@ PortableFileBackedProtoLog<ProtoT>::Create(const Filesystem* filesystem, options.max_proto_size)); } + if (options.compression_level < 0 || options.compression_level > 9) { + return absl_ports::InvalidArgumentError(IcingStringUtil::StringPrintf( + "options.compression_level must be between 0 and 9 inclusive, was %d", + options.compression_level)); + } + if (!filesystem->FileExists(file_path.c_str())) { return InitializeNewFile(filesystem, file_path, options); } @@ -660,7 +676,8 @@ PortableFileBackedProtoLog<ProtoT>::InitializeNewFile( CreateResult create_result = { std::unique_ptr<PortableFileBackedProtoLog<ProtoT>>( new PortableFileBackedProtoLog<ProtoT>(filesystem, file_path, - std::move(header))), + std::move(header), + options.compression_level)), /*data_loss=*/DataLoss::NONE, /*recalculated_checksum=*/false}; return create_result; @@ -788,7 +805,8 @@ PortableFileBackedProtoLog<ProtoT>::InitializeExistingFile( CreateResult create_result = { std::unique_ptr<PortableFileBackedProtoLog<ProtoT>>( new PortableFileBackedProtoLog<ProtoT>(filesystem, file_path, - std::move(header))), + std::move(header), + options.compression_level)), data_loss, recalculated_checksum}; return create_result; @@ -893,7 +911,7 @@ PortableFileBackedProtoLog<ProtoT>::WriteProto(const ProtoT& proto) { if (header_->GetCompressFlag()) { protobuf_ports::GzipOutputStream::Options options; options.format = protobuf_ports::GzipOutputStream::ZLIB; - options.compression_level = kDeflateCompressionLevel; + options.compression_level = compression_level_; protobuf_ports::GzipOutputStream compressing_stream(&proto_stream, options); diff --git a/icing/file/portable-file-backed-proto-log_test.cc b/icing/file/portable-file-backed-proto-log_test.cc index af09d18..bf5e604 100644 --- a/icing/file/portable-file-backed-proto-log_test.cc +++ b/icing/file/portable-file-backed-proto-log_test.cc @@ -73,24 +73,18 @@ class PortableFileBackedProtoLogTest : public ::testing::Test { const Filesystem filesystem_; std::string file_path_; bool compress_ = true; + int32_t compression_level_ = + PortableFileBackedProtoLog<DocumentProto>::kDeflateCompressionLevel; int64_t max_proto_size_ = 256 * 1024; // 256 KiB }; TEST_F(PortableFileBackedProtoLogTest, Initialize) { - // max_proto_size must be greater than 0 - int invalid_max_proto_size = 0; - ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create( - &filesystem_, file_path_, - PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, invalid_max_proto_size)), - StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); - ICING_ASSERT_OK_AND_ASSIGN( PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, - PortableFileBackedProtoLog<DocumentProto>::Options(compress_, - max_proto_size_))); + PortableFileBackedProtoLog<DocumentProto>::Options( + compress_, max_proto_size_, compression_level_))); EXPECT_THAT(create_result.proto_log, NotNull()); EXPECT_FALSE(create_result.has_data_loss()); EXPECT_FALSE(create_result.recalculated_checksum); @@ -99,7 +93,41 @@ TEST_F(PortableFileBackedProtoLogTest, Initialize) { ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - !compress_, max_proto_size_)), + !compress_, max_proto_size_, compression_level_)), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + +TEST_F(PortableFileBackedProtoLogTest, InitializeValidatesOptions) { + // max_proto_size must be greater than 0 + int invalid_max_proto_size = 0; + ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create( + &filesystem_, file_path_, + PortableFileBackedProtoLog<DocumentProto>::Options( + compress_, invalid_max_proto_size, compression_level_)), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + + // max_proto_size must be under 16 MiB + invalid_max_proto_size = 16 * 1024 * 1024; + ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create( + &filesystem_, file_path_, + PortableFileBackedProtoLog<DocumentProto>::Options( + compress_, invalid_max_proto_size, compression_level_)), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + + // compression_level must be between 0 and 9 inclusive + int invalid_compression_level = -1; + ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create( + &filesystem_, file_path_, + PortableFileBackedProtoLog<DocumentProto>::Options( + compress_, max_proto_size_, invalid_compression_level)), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); + + // compression_level must be between 0 and 9 inclusive + invalid_compression_level = 10; + ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create( + &filesystem_, file_path_, + PortableFileBackedProtoLog<DocumentProto>::Options( + compress_, max_proto_size_, invalid_compression_level)), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } @@ -108,8 +136,8 @@ TEST_F(PortableFileBackedProtoLogTest, ReservedSpaceForHeader) { PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, - PortableFileBackedProtoLog<DocumentProto>::Options(compress_, - max_proto_size_))); + PortableFileBackedProtoLog<DocumentProto>::Options( + compress_, max_proto_size_, compression_level_))); // With no protos written yet, the log should be minimum the size of the // reserved header space. @@ -123,8 +151,8 @@ TEST_F(PortableFileBackedProtoLogTest, WriteProtoTooLarge) { PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, - PortableFileBackedProtoLog<DocumentProto>::Options(compress_, - max_proto_size))); + PortableFileBackedProtoLog<DocumentProto>::Options( + compress_, max_proto_size, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -140,8 +168,8 @@ TEST_F(PortableFileBackedProtoLogTest, ReadProtoWrongKProtoMagic) { PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, - PortableFileBackedProtoLog<DocumentProto>::Options(compress_, - max_proto_size_))); + PortableFileBackedProtoLog<DocumentProto>::Options( + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -175,7 +203,7 @@ TEST_F(PortableFileBackedProtoLogTest, ReadWriteUncompressedProto) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - /*compress_in=*/false, max_proto_size_))); + /*compress_in=*/false, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -222,7 +250,7 @@ TEST_F(PortableFileBackedProtoLogTest, ReadWriteUncompressedProto) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - /*compress_in=*/false, max_proto_size_))); + /*compress_in=*/false, max_proto_size_, compression_level_))); auto recreated_proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -244,7 +272,7 @@ TEST_F(PortableFileBackedProtoLogTest, ReadWriteCompressedProto) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - /*compress_in=*/true, max_proto_size_))); + /*compress_in=*/true, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -291,7 +319,7 @@ TEST_F(PortableFileBackedProtoLogTest, ReadWriteCompressedProto) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - /*compress_in=*/true, max_proto_size_))); + /*compress_in=*/true, max_proto_size_, compression_level_))); auto recreated_proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -304,6 +332,168 @@ TEST_F(PortableFileBackedProtoLogTest, ReadWriteCompressedProto) { } } +TEST_F(PortableFileBackedProtoLogTest, ReadWriteDifferentCompressionLevel) { + int document1_offset; + int document2_offset; + int document3_offset; + + // The first proto to write that's close to the max size. Leave some room for + // the rest of the proto properties. + std::string long_str(max_proto_size_ - 1024, 'a'); + DocumentProto document1 = DocumentBuilder() + .SetKey("namespace1", "uri1") + .AddStringProperty("long_str", long_str) + .Build(); + DocumentProto document2 = + DocumentBuilder().SetKey("namespace2", "uri2").Build(); + DocumentProto document3 = + DocumentBuilder().SetKey("namespace3", "uri3").Build(); + + { + ICING_ASSERT_OK_AND_ASSIGN( + PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, + PortableFileBackedProtoLog<DocumentProto>::Create( + &filesystem_, file_path_, + PortableFileBackedProtoLog<DocumentProto>::Options( + /*compress_in=*/true, max_proto_size_, + /*compression_level_in=*/3))); + auto proto_log = std::move(create_result.proto_log); + ASSERT_FALSE(create_result.has_data_loss()); + + // Write the first proto + ICING_ASSERT_OK_AND_ASSIGN(document1_offset, + proto_log->WriteProto(document1)); + + // Check that what we read is what we wrote + ASSERT_THAT(proto_log->ReadProto(document1_offset), + IsOkAndHolds(EqualsProto(document1))); + + ICING_ASSERT_OK(proto_log->PersistToDisk()); + } + + // Make a new proto_log with the same file_path but different compression + // level, and make sure we can still read from and write to the same + // underlying file. + { + ICING_ASSERT_OK_AND_ASSIGN( + PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, + PortableFileBackedProtoLog<DocumentProto>::Create( + &filesystem_, file_path_, + PortableFileBackedProtoLog<DocumentProto>::Options( + /*compress_in=*/true, max_proto_size_, + /*compression_level_in=*/9))); + auto recreated_proto_log = std::move(create_result.proto_log); + ASSERT_FALSE(create_result.has_data_loss()); + + // Check the first proto + ASSERT_THAT(recreated_proto_log->ReadProto(document1_offset), + IsOkAndHolds(EqualsProto(document1))); + + // Write a second proto + ICING_ASSERT_OK_AND_ASSIGN(document2_offset, + recreated_proto_log->WriteProto(document2)); + + ASSERT_GT(document2_offset, document1_offset); + + // Check the second proto + ASSERT_THAT(recreated_proto_log->ReadProto(document2_offset), + IsOkAndHolds(EqualsProto(document2))); + + ICING_ASSERT_OK(recreated_proto_log->PersistToDisk()); + } + + // One more time but with 0 compression level + { + ICING_ASSERT_OK_AND_ASSIGN( + PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, + PortableFileBackedProtoLog<DocumentProto>::Create( + &filesystem_, file_path_, + PortableFileBackedProtoLog<DocumentProto>::Options( + /*compress_in=*/true, max_proto_size_, + /*compression_level=*/0))); + auto recreated_proto_log = std::move(create_result.proto_log); + ASSERT_FALSE(create_result.has_data_loss()); + + // Check the first proto + ASSERT_THAT(recreated_proto_log->ReadProto(document1_offset), + IsOkAndHolds(EqualsProto(document1))); + + // Check the second proto + ASSERT_THAT(recreated_proto_log->ReadProto(document2_offset), + IsOkAndHolds(EqualsProto(document2))); + + // Write a third proto + ICING_ASSERT_OK_AND_ASSIGN(document3_offset, + recreated_proto_log->WriteProto(document3)); + + ASSERT_GT(document3_offset, document2_offset); + + // Check the third proto + ASSERT_THAT(recreated_proto_log->ReadProto(document3_offset), + IsOkAndHolds(EqualsProto(document3))); + } +} + +TEST_F(PortableFileBackedProtoLogTest, + WriteDifferentCompressionLevelDifferentSizes) { + int document_log_size_with_compression_3; + int document_log_size_with_no_compression; + + // The first proto to write that's close to the max size. Leave some room for + // the rest of the proto properties. + std::string long_str(max_proto_size_ - 1024, 'a'); + DocumentProto document1 = DocumentBuilder() + .SetKey("namespace1", "uri1") + .AddStringProperty("long_str", long_str) + .Build(); + + { + ICING_ASSERT_OK_AND_ASSIGN( + PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, + PortableFileBackedProtoLog<DocumentProto>::Create( + &filesystem_, file_path_, + PortableFileBackedProtoLog<DocumentProto>::Options( + /*compress_in=*/true, max_proto_size_, + /*compression_level_in=*/3))); + auto proto_log = std::move(create_result.proto_log); + ASSERT_FALSE(create_result.has_data_loss()); + + // Write the proto + ICING_ASSERT_OK(proto_log->WriteProto(document1)); + ICING_ASSERT_OK(proto_log->PersistToDisk()); + + document_log_size_with_compression_3 = + filesystem_.GetFileSize(file_path_.c_str()); + } + + // Delete the proto_log so we can reuse the file_path + filesystem_.DeleteFile(file_path_.c_str()); + + { + ICING_ASSERT_OK_AND_ASSIGN( + PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, + PortableFileBackedProtoLog<DocumentProto>::Create( + &filesystem_, file_path_, + PortableFileBackedProtoLog<DocumentProto>::Options( + /*compress_in=*/true, max_proto_size_, + /*compression_level_in=*/0))); + auto proto_log = std::move(create_result.proto_log); + ASSERT_FALSE(create_result.has_data_loss()); + + // Write the proto + ICING_ASSERT_OK(proto_log->WriteProto(document1)); + ICING_ASSERT_OK(proto_log->PersistToDisk()); + + document_log_size_with_no_compression = + filesystem_.GetFileSize(file_path_.c_str()); + + // Uncompressed document file size should be larger than original compressed + // document file size + ASSERT_GT(document_log_size_with_no_compression, + document_log_size_with_compression_3); + } +} + TEST_F(PortableFileBackedProtoLogTest, CorruptHeader) { { ICING_ASSERT_OK_AND_ASSIGN( @@ -311,7 +501,7 @@ TEST_F(PortableFileBackedProtoLogTest, CorruptHeader) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto recreated_proto_log = std::move(create_result.proto_log); EXPECT_FALSE(create_result.has_data_loss()); } @@ -328,7 +518,7 @@ TEST_F(PortableFileBackedProtoLogTest, CorruptHeader) { ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_)), + compress_, max_proto_size_, compression_level_)), StatusIs(libtextclassifier3::StatusCode::INTERNAL, HasSubstr("Invalid header checksum"))); } @@ -341,7 +531,7 @@ TEST_F(PortableFileBackedProtoLogTest, DifferentMagic) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto recreated_proto_log = std::move(create_result.proto_log); EXPECT_FALSE(create_result.has_data_loss()); @@ -360,7 +550,7 @@ TEST_F(PortableFileBackedProtoLogTest, DifferentMagic) { ASSERT_THAT(PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_)), + compress_, max_proto_size_, compression_level_)), StatusIs(libtextclassifier3::StatusCode::INTERNAL, HasSubstr("Invalid header kMagic"))); } @@ -383,7 +573,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); EXPECT_FALSE(create_result.has_data_loss()); @@ -410,7 +600,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); EXPECT_FALSE(create_result.has_data_loss()); EXPECT_THAT(create_result.data_loss, Eq(DataLoss::NONE)); @@ -432,7 +622,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -478,7 +668,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); EXPECT_TRUE(create_result.has_data_loss()); EXPECT_THAT(create_result.data_loss, Eq(DataLoss::COMPLETE)); @@ -507,7 +697,7 @@ TEST_F(PortableFileBackedProtoLogTest, DirtyBitFalseAlarmKeepsData) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -537,7 +727,7 @@ TEST_F(PortableFileBackedProtoLogTest, DirtyBitFalseAlarmKeepsData) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); EXPECT_FALSE(create_result.has_data_loss()); @@ -569,7 +759,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -615,7 +805,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_TRUE(create_result.has_data_loss()); ASSERT_THAT(create_result.data_loss, Eq(DataLoss::PARTIAL)); @@ -640,7 +830,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -663,7 +853,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); // We previously persisted to disk so everything should be in a perfect // state. @@ -683,7 +873,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -707,7 +897,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); // We previously persisted to disk so everything should be in a perfect // state. @@ -726,7 +916,7 @@ TEST_F(PortableFileBackedProtoLogTest, DirtyBitIsFalseAfterPutAndDestructor) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -751,7 +941,7 @@ TEST_F(PortableFileBackedProtoLogTest, DirtyBitIsFalseAfterPutAndDestructor) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); // We previously persisted to disk so everything should be in a perfect // state. @@ -771,7 +961,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -797,7 +987,7 @@ TEST_F(PortableFileBackedProtoLogTest, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); // We previously persisted to disk so everything should be in a perfect // state. @@ -819,8 +1009,8 @@ TEST_F(PortableFileBackedProtoLogTest, Iterator) { PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, - PortableFileBackedProtoLog<DocumentProto>::Options(compress_, - max_proto_size_))); + PortableFileBackedProtoLog<DocumentProto>::Options( + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -872,7 +1062,7 @@ TEST_F(PortableFileBackedProtoLogTest, ComputeChecksum) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -890,7 +1080,7 @@ TEST_F(PortableFileBackedProtoLogTest, ComputeChecksum) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -915,8 +1105,8 @@ TEST_F(PortableFileBackedProtoLogTest, EraseProtoShouldSetZero) { PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, - PortableFileBackedProtoLog<DocumentProto>::Options(compress_, - max_proto_size_))); + PortableFileBackedProtoLog<DocumentProto>::Options( + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -950,8 +1140,8 @@ TEST_F(PortableFileBackedProtoLogTest, EraseProtoShouldReturnNotFound) { PortableFileBackedProtoLog<DocumentProto>::CreateResult create_result, PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, - PortableFileBackedProtoLog<DocumentProto>::Options(compress_, - max_proto_size_))); + PortableFileBackedProtoLog<DocumentProto>::Options( + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -993,7 +1183,7 @@ TEST_F(PortableFileBackedProtoLogTest, ChecksumShouldBeCorrectWithErasedProto) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -1021,7 +1211,7 @@ TEST_F(PortableFileBackedProtoLogTest, ChecksumShouldBeCorrectWithErasedProto) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -1041,7 +1231,7 @@ TEST_F(PortableFileBackedProtoLogTest, ChecksumShouldBeCorrectWithErasedProto) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); ASSERT_FALSE(create_result.has_data_loss()); @@ -1063,7 +1253,7 @@ TEST_F(PortableFileBackedProtoLogTest, ChecksumShouldBeCorrectWithErasedProto) { PortableFileBackedProtoLog<DocumentProto>::Create( &filesystem_, file_path_, PortableFileBackedProtoLog<DocumentProto>::Options( - compress_, max_proto_size_))); + compress_, max_proto_size_, compression_level_))); auto proto_log = std::move(create_result.proto_log); EXPECT_FALSE(create_result.has_data_loss()); } |