aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndroid Chromium Automerger <chromium-automerger@android>2013-01-08 06:55:15 +0000
committerAndroid Chromium Automerger <chromium-automerger@android>2013-01-08 06:55:15 +0000
commit4b2795fc373851344a0e226f1a6a8ff92d3c713a (patch)
tree641012308909ebaf4193151d1410185e4209d32a
parentee40f8aa2dae52f623f365769583469f6c8cd18e (diff)
parent85e27d04530c0323957f28c3f47a7d9f6efeceb0 (diff)
downloadsrc-4b2795fc373851344a0e226f1a6a8ff92d3c713a.tar.gz
Merge third_party/leveldatabase/src from https://chromium.googlesource.com/external/leveldb.git at 85e27d04530c0323957f28c3f47a7d9f6efeceb0
This commit was generated by merge_from_chromium.py. Change-Id: Id796322cbd5b4bae6fa48c70e9968054d6061a52
-rw-r--r--Makefile57
-rw-r--r--build_detect_platform23
-rw-r--r--db/c.cc10
-rw-r--r--db/c_test.c3
-rw-r--r--db/db_test.cc75
-rw-r--r--db/leveldb_main.cc238
-rw-r--r--db/version_set.cc36
-rw-r--r--db/version_set.h2
-rw-r--r--doc/index.html6
-rw-r--r--doc/log_format.txt4
-rw-r--r--doc/table_format.txt12
-rw-r--r--include/leveldb/c.h6
-rw-r--r--include/leveldb/db.h2
-rw-r--r--port/port_posix.h8
14 files changed, 434 insertions, 48 deletions
diff --git a/Makefile b/Makefile
index 4dd1366..42c4952 100644
--- a/Makefile
+++ b/Makefile
@@ -2,9 +2,6 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file. See the AUTHORS file for names of contributors.
-# Inherit some settings from environment variables, if available
-INSTALL_PATH ?= $(CURDIR)
-
#-----------------------------------------------
# Uncomment exactly one of the lines labelled (A), (B), and (C) below
# to switch between compilation modes.
@@ -24,6 +21,7 @@ CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT)
LDFLAGS += $(PLATFORM_LDFLAGS)
+LIBS += $(PLATFORM_LIBS)
LIBOBJECTS = $(SOURCES:.cc=.o)
MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
@@ -52,7 +50,7 @@ TESTS = \
version_set_test \
write_batch_test
-PROGRAMS = db_bench $(TESTS)
+PROGRAMS = db_bench leveldbutil $(TESTS)
BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
LIBRARY = libleveldb.a
@@ -71,7 +69,7 @@ SHARED = $(SHARED1)
else
# Update db.h if you change these.
SHARED_MAJOR = 1
-SHARED_MINOR = 6
+SHARED_MINOR = 9
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
SHARED2 = $(SHARED1).$(SHARED_MAJOR)
SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
@@ -83,7 +81,7 @@ $(SHARED2): $(SHARED3)
endif
$(SHARED3):
- $(CXX) $(SOURCES) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(INSTALL_PATH)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -o $(SHARED3)
+ $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3) $(LIBS)
endif # PLATFORM_SHARED_EXT
@@ -101,74 +99,77 @@ $(LIBRARY): $(LIBOBJECTS)
$(AR) -rs $@ $(LIBOBJECTS)
db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL)
- $(CXX) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS)
db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL)
- $(CXX) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LDFLAGS) -lsqlite3
+ $(CXX) $(LDFLAGS) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS)
db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL)
- $(CXX) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LDFLAGS) -lkyotocabinet
+ $(CXX) $(LDFLAGS) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS)
+
+leveldbutil: db/leveldb_main.o $(LIBOBJECTS)
+ $(CXX) $(LDFLAGS) db/leveldb_main.o $(LIBOBJECTS) -o $@ $(LIBS)
arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
corruption_test: db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS)
- $(CXX) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
$(MEMENVLIBRARY) : $(MEMENVOBJECTS)
rm -f $@
$(AR) -rs $@ $(MEMENVOBJECTS)
memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS)
- $(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LDFLAGS)
+ $(CXX) $(LDFLAGS) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LIBS)
ifeq ($(PLATFORM), IOS)
# For iOS, create universal object files to be used on both the simulator and
diff --git a/build_detect_platform b/build_detect_platform
index 83bbe42..5801d20 100644
--- a/build_detect_platform
+++ b/build_detect_platform
@@ -7,8 +7,11 @@
# CC C Compiler path
# CXX C++ Compiler path
# PLATFORM_LDFLAGS Linker flags
+# PLATFORM_LIBS Libraries flags
# PLATFORM_SHARED_EXT Extension for shared libraries
# PLATFORM_SHARED_LDFLAGS Flags for building shared library
+# This flag is embedded just before the name
+# of the shared library without intervening spaces
# PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library
# PLATFORM_CCFLAGS C compiler flags
# PLATFORM_CXXFLAGS C++ compiler flags. Will contain:
@@ -51,6 +54,7 @@ CROSS_COMPILE=
PLATFORM_CCFLAGS=
PLATFORM_CXXFLAGS=
PLATFORM_LDFLAGS=
+PLATFORM_LIBS=
PLATFORM_SHARED_EXT="so"
PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl,"
PLATFORM_SHARED_CFLAGS="-fPIC"
@@ -68,7 +72,8 @@ case "$TARGET_OS" in
PLATFORM=OS_MACOSX
COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
PLATFORM_SHARED_EXT=dylib
- PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
+ [ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd`
+ PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name $INSTALL_PATH/"
PORT_FILE=port/port_posix.cc
;;
Linux)
@@ -80,19 +85,19 @@ case "$TARGET_OS" in
SunOS)
PLATFORM=OS_SOLARIS
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_SOLARIS"
- PLATFORM_LDFLAGS="-lpthread -lrt"
+ PLATFORM_LIBS="-lpthread -lrt"
PORT_FILE=port/port_posix.cc
;;
FreeBSD)
PLATFORM=OS_FREEBSD
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_FREEBSD"
- PLATFORM_LDFLAGS="-lpthread"
+ PLATFORM_LIBS="-lpthread"
PORT_FILE=port/port_posix.cc
;;
NetBSD)
PLATFORM=OS_NETBSD
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD"
- PLATFORM_LDFLAGS="-lpthread -lgcc_s"
+ PLATFORM_LIBS="-lpthread -lgcc_s"
PORT_FILE=port/port_posix.cc
;;
OpenBSD)
@@ -104,7 +109,7 @@ case "$TARGET_OS" in
DragonFly)
PLATFORM=OS_DRAGONFLYBSD
COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_DRAGONFLYBSD"
- PLATFORM_LDFLAGS="-lpthread"
+ PLATFORM_LIBS="-lpthread"
PORT_FILE=port/port_posix.cc
;;
OS_ANDROID_CROSSCOMPILE)
@@ -136,7 +141,8 @@ DIRS="$PREFIX/db $PREFIX/util $PREFIX/table"
set -f # temporarily disable globbing so that our patterns aren't expanded
PRUNE_TEST="-name *test*.cc -prune"
PRUNE_BENCH="-name *_bench.cc -prune"
-PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`
+PRUNE_TOOL="-name leveldb_main.cc -prune"
+PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`
set +f # re-enable globbing
@@ -169,7 +175,7 @@ EOF
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
+ PLATFORM_LIBS="$PLATFORM_LIBS -lsnappy"
fi
# Test whether tcmalloc is available
@@ -177,7 +183,7 @@ EOF
int main() {}
EOF
if [ "$?" = 0 ]; then
- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
+ PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc"
fi
fi
@@ -188,6 +194,7 @@ echo "CC=$CC" >> $OUTPUT
echo "CXX=$CXX" >> $OUTPUT
echo "PLATFORM=$PLATFORM" >> $OUTPUT
echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT
+echo "PLATFORM_LIBS=$PLATFORM_LIBS" >> $OUTPUT
echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT
echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT
echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
diff --git a/db/c.cc b/db/c.cc
index ed5c4cb..08ff0ad 100644
--- a/db/c.cc
+++ b/db/c.cc
@@ -24,6 +24,8 @@ using leveldb::Env;
using leveldb::FileLock;
using leveldb::FilterPolicy;
using leveldb::Iterator;
+using leveldb::kMajorVersion;
+using leveldb::kMinorVersion;
using leveldb::Logger;
using leveldb::NewBloomFilterPolicy;
using leveldb::NewLRUCache;
@@ -582,4 +584,12 @@ void leveldb_free(void* ptr) {
free(ptr);
}
+int leveldb_major_version() {
+ return kMajorVersion;
+}
+
+int leveldb_minor_version() {
+ return kMinorVersion;
+}
+
} // end extern "C"
diff --git a/db/c_test.c b/db/c_test.c
index 9d6dfec..7cd5ee0 100644
--- a/db/c_test.c
+++ b/db/c_test.c
@@ -165,6 +165,9 @@ int main(int argc, char** argv) {
char* err = NULL;
int run = -1;
+ CheckCondition(leveldb_major_version() >= 1);
+ CheckCondition(leveldb_minor_version() >= 1);
+
snprintf(dbname, sizeof(dbname),
"%s/leveldb_c_test-%d",
GetTempDir(),
diff --git a/db/db_test.cc b/db/db_test.cc
index 74abd13..684ea3b 100644
--- a/db/db_test.cc
+++ b/db/db_test.cc
@@ -59,6 +59,12 @@ class SpecialEnv : public EnvWrapper {
// Simulate non-writable file system while this pointer is non-NULL
port::AtomicPointer non_writable_;
+ // Force sync of manifest files to fail while this pointer is non-NULL
+ port::AtomicPointer manifest_sync_error_;
+
+ // Force write to manifest files to fail while this pointer is non-NULL
+ port::AtomicPointer manifest_write_error_;
+
bool count_random_reads_;
AtomicCounter random_read_counter_;
@@ -69,6 +75,8 @@ class SpecialEnv : public EnvWrapper {
no_space_.Release_Store(NULL);
non_writable_.Release_Store(NULL);
count_random_reads_ = false;
+ manifest_sync_error_.Release_Store(NULL);
+ manifest_write_error_.Release_Store(NULL);
}
Status NewWritableFile(const std::string& f, WritableFile** r) {
@@ -100,6 +108,30 @@ class SpecialEnv : public EnvWrapper {
return base_->Sync();
}
};
+ class ManifestFile : public WritableFile {
+ private:
+ SpecialEnv* env_;
+ WritableFile* base_;
+ public:
+ ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { }
+ ~ManifestFile() { delete base_; }
+ Status Append(const Slice& data) {
+ if (env_->manifest_write_error_.Acquire_Load() != NULL) {
+ return Status::IOError("simulated writer error");
+ } else {
+ return base_->Append(data);
+ }
+ }
+ Status Close() { return base_->Close(); }
+ Status Flush() { return base_->Flush(); }
+ Status Sync() {
+ if (env_->manifest_sync_error_.Acquire_Load() != NULL) {
+ return Status::IOError("simulated sync error");
+ } else {
+ return base_->Sync();
+ }
+ }
+ };
if (non_writable_.Acquire_Load() != NULL) {
return Status::IOError("simulated write error");
@@ -109,6 +141,8 @@ class SpecialEnv : public EnvWrapper {
if (s.ok()) {
if (strstr(f.c_str(), ".sst") != NULL) {
*r = new SSTableFile(this, *r);
+ } else if (strstr(f.c_str(), "MANIFEST") != NULL) {
+ *r = new ManifestFile(this, *r);
}
}
return s;
@@ -1492,6 +1526,47 @@ TEST(DBTest, NonWritableFileSystem) {
env_->non_writable_.Release_Store(NULL);
}
+TEST(DBTest, ManifestWriteError) {
+ // Test for the following problem:
+ // (a) Compaction produces file F
+ // (b) Log record containing F is written to MANIFEST file, but Sync() fails
+ // (c) GC deletes F
+ // (d) After reopening DB, reads fail since deleted F is named in log record
+
+ // We iterate twice. In the second iteration, everything is the
+ // same except the log record never makes it to the MANIFEST file.
+ for (int iter = 0; iter < 2; iter++) {
+ port::AtomicPointer* error_type = (iter == 0)
+ ? &env_->manifest_sync_error_
+ : &env_->manifest_write_error_;
+
+ // Insert foo=>bar mapping
+ Options options = CurrentOptions();
+ options.env = env_;
+ options.create_if_missing = true;
+ options.error_if_exists = false;
+ DestroyAndReopen(&options);
+ ASSERT_OK(Put("foo", "bar"));
+ ASSERT_EQ("bar", Get("foo"));
+
+ // Memtable compaction (will succeed)
+ dbfull()->TEST_CompactMemTable();
+ ASSERT_EQ("bar", Get("foo"));
+ const int last = config::kMaxMemCompactLevel;
+ ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level
+
+ // Merging compaction (will fail)
+ error_type->Release_Store(env_);
+ dbfull()->TEST_CompactRange(last, NULL, NULL); // Should fail
+ ASSERT_EQ("bar", Get("foo"));
+
+ // Recovery: should not lose data
+ error_type->Release_Store(NULL);
+ Reopen(&options);
+ ASSERT_EQ("bar", Get("foo"));
+ }
+}
+
TEST(DBTest, FilesDeletedAfterCompaction) {
ASSERT_OK(Put("foo", "v2"));
Compact("a", "z");
diff --git a/db/leveldb_main.cc b/db/leveldb_main.cc
new file mode 100644
index 0000000..995d761
--- /dev/null
+++ b/db/leveldb_main.cc
@@ -0,0 +1,238 @@
+// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include <stdio.h>
+#include "db/dbformat.h"
+#include "db/filename.h"
+#include "db/log_reader.h"
+#include "db/version_edit.h"
+#include "db/write_batch_internal.h"
+#include "leveldb/env.h"
+#include "leveldb/iterator.h"
+#include "leveldb/options.h"
+#include "leveldb/status.h"
+#include "leveldb/table.h"
+#include "leveldb/write_batch.h"
+#include "util/logging.h"
+
+namespace leveldb {
+
+namespace {
+
+bool GuessType(const std::string& fname, FileType* type) {
+ size_t pos = fname.rfind('/');
+ std::string basename;
+ if (pos == std::string::npos) {
+ basename = fname;
+ } else {
+ basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1);
+ }
+ uint64_t ignored;
+ return ParseFileName(basename, &ignored, type);
+}
+
+// Notified when log reader encounters corruption.
+class CorruptionReporter : public log::Reader::Reporter {
+ public:
+ virtual void Corruption(size_t bytes, const Status& status) {
+ printf("corruption: %d bytes; %s\n",
+ static_cast<int>(bytes),
+ status.ToString().c_str());
+ }
+};
+
+// Print contents of a log file. (*func)() is called on every record.
+bool PrintLogContents(Env* env, const std::string& fname,
+ void (*func)(Slice)) {
+ SequentialFile* file;
+ Status s = env->NewSequentialFile(fname, &file);
+ if (!s.ok()) {
+ fprintf(stderr, "%s\n", s.ToString().c_str());
+ return false;
+ }
+ CorruptionReporter reporter;
+ log::Reader reader(file, &reporter, true, 0);
+ Slice record;
+ std::string scratch;
+ while (reader.ReadRecord(&record, &scratch)) {
+ printf("--- offset %llu; ",
+ static_cast<unsigned long long>(reader.LastRecordOffset()));
+ (*func)(record);
+ }
+ delete file;
+ return true;
+}
+
+// Called on every item found in a WriteBatch.
+class WriteBatchItemPrinter : public WriteBatch::Handler {
+ public:
+ uint64_t offset_;
+ uint64_t sequence_;
+
+ virtual void Put(const Slice& key, const Slice& value) {
+ printf(" put '%s' '%s'\n",
+ EscapeString(key).c_str(),
+ EscapeString(value).c_str());
+ }
+ virtual void Delete(const Slice& key) {
+ printf(" del '%s'\n",
+ EscapeString(key).c_str());
+ }
+};
+
+
+// Called on every log record (each one of which is a WriteBatch)
+// found in a kLogFile.
+static void WriteBatchPrinter(Slice record) {
+ if (record.size() < 12) {
+ printf("log record length %d is too small\n",
+ static_cast<int>(record.size()));
+ return;
+ }
+ WriteBatch batch;
+ WriteBatchInternal::SetContents(&batch, record);
+ printf("sequence %llu\n",
+ static_cast<unsigned long long>(WriteBatchInternal::Sequence(&batch)));
+ WriteBatchItemPrinter batch_item_printer;
+ Status s = batch.Iterate(&batch_item_printer);
+ if (!s.ok()) {
+ printf(" error: %s\n", s.ToString().c_str());
+ }
+}
+
+bool DumpLog(Env* env, const std::string& fname) {
+ return PrintLogContents(env, fname, WriteBatchPrinter);
+}
+
+// Called on every log record (each one of which is a WriteBatch)
+// found in a kDescriptorFile.
+static void VersionEditPrinter(Slice record) {
+ VersionEdit edit;
+ Status s = edit.DecodeFrom(record);
+ if (!s.ok()) {
+ printf("%s\n", s.ToString().c_str());
+ return;
+ }
+ printf("%s", edit.DebugString().c_str());
+}
+
+bool DumpDescriptor(Env* env, const std::string& fname) {
+ return PrintLogContents(env, fname, VersionEditPrinter);
+}
+
+bool DumpTable(Env* env, const std::string& fname) {
+ uint64_t file_size;
+ RandomAccessFile* file = NULL;
+ Table* table = NULL;
+ Status s = env->GetFileSize(fname, &file_size);
+ if (s.ok()) {
+ s = env->NewRandomAccessFile(fname, &file);
+ }
+ if (s.ok()) {
+ // We use the default comparator, which may or may not match the
+ // comparator used in this database. However this should not cause
+ // problems since we only use Table operations that do not require
+ // any comparisons. In particular, we do not call Seek or Prev.
+ s = Table::Open(Options(), file, file_size, &table);
+ }
+ if (!s.ok()) {
+ fprintf(stderr, "%s\n", s.ToString().c_str());
+ delete table;
+ delete file;
+ return false;
+ }
+
+ ReadOptions ro;
+ ro.fill_cache = false;
+ Iterator* iter = table->NewIterator(ro);
+ for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+ ParsedInternalKey key;
+ if (!ParseInternalKey(iter->key(), &key)) {
+ printf("badkey '%s' => '%s'\n",
+ EscapeString(iter->key()).c_str(),
+ EscapeString(iter->value()).c_str());
+ } else {
+ char kbuf[20];
+ const char* type;
+ if (key.type == kTypeDeletion) {
+ type = "del";
+ } else if (key.type == kTypeValue) {
+ type = "val";
+ } else {
+ snprintf(kbuf, sizeof(kbuf), "%d", static_cast<int>(key.type));
+ type = kbuf;
+ }
+ printf("'%s' @ %8llu : %s => '%s'\n",
+ EscapeString(key.user_key).c_str(),
+ static_cast<unsigned long long>(key.sequence),
+ type,
+ EscapeString(iter->value()).c_str());
+ }
+ }
+ s = iter->status();
+ if (!s.ok()) {
+ printf("iterator error: %s\n", s.ToString().c_str());
+ }
+
+ delete iter;
+ delete table;
+ delete file;
+ return true;
+}
+
+bool DumpFile(Env* env, const std::string& fname) {
+ FileType ftype;
+ if (!GuessType(fname, &ftype)) {
+ fprintf(stderr, "%s: unknown file type\n", fname.c_str());
+ return false;
+ }
+ switch (ftype) {
+ case kLogFile: return DumpLog(env, fname);
+ case kDescriptorFile: return DumpDescriptor(env, fname);
+ case kTableFile: return DumpTable(env, fname);
+
+ default: {
+ fprintf(stderr, "%s: not a dump-able file type\n", fname.c_str());
+ break;
+ }
+ }
+ return false;
+}
+
+bool HandleDumpCommand(Env* env, char** files, int num) {
+ bool ok = true;
+ for (int i = 0; i < num; i++) {
+ ok &= DumpFile(env, files[i]);
+ }
+ return ok;
+}
+
+}
+} // namespace leveldb
+
+static void Usage() {
+ fprintf(
+ stderr,
+ "Usage: leveldbutil command...\n"
+ " dump files... -- dump contents of specified files\n"
+ );
+}
+
+int main(int argc, char** argv) {
+ leveldb::Env* env = leveldb::Env::Default();
+ bool ok = true;
+ if (argc < 2) {
+ Usage();
+ ok = false;
+ } else {
+ std::string command = argv[1];
+ if (command == "dump") {
+ ok = leveldb::HandleDumpCommand(env, argv+2, argc-2);
+ } else {
+ Usage();
+ ok = false;
+ }
+ }
+ return (ok ? 0 : 1);
+}
diff --git a/db/version_set.cc b/db/version_set.cc
index bdd4a57..7d0a5de 100644
--- a/db/version_set.cc
+++ b/db/version_set.cc
@@ -786,12 +786,23 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) {
if (s.ok()) {
s = descriptor_file_->Sync();
}
+ if (!s.ok()) {
+ Log(options_->info_log, "MANIFEST write: %s\n", s.ToString().c_str());
+ if (ManifestContains(record)) {
+ Log(options_->info_log,
+ "MANIFEST contains log record despite error; advancing to new "
+ "version to prevent mismatch between in-memory and logged state");
+ s = Status::OK();
+ }
+ }
}
// If we just created a new descriptor file, install it by writing a
// new CURRENT file that points to it.
if (s.ok() && !new_manifest_file.empty()) {
s = SetCurrentFile(env_, dbname_, manifest_file_number_);
+ // No need to double-check MANIFEST in case of error since it
+ // will be discarded below.
}
mu->Lock();
@@ -1025,6 +1036,31 @@ const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const {
return scratch->buffer;
}
+// Return true iff the manifest contains the specified record.
+bool VersionSet::ManifestContains(const std::string& record) const {
+ std::string fname = DescriptorFileName(dbname_, manifest_file_number_);
+ Log(options_->info_log, "ManifestContains: checking %s\n", fname.c_str());
+ SequentialFile* file = NULL;
+ Status s = env_->NewSequentialFile(fname, &file);
+ if (!s.ok()) {
+ Log(options_->info_log, "ManifestContains: %s\n", s.ToString().c_str());
+ return false;
+ }
+ log::Reader reader(file, NULL, true/*checksum*/, 0);
+ Slice r;
+ std::string scratch;
+ bool result = false;
+ while (reader.ReadRecord(&r, &scratch)) {
+ if (r == Slice(record)) {
+ result = true;
+ break;
+ }
+ }
+ delete file;
+ Log(options_->info_log, "ManifestContains: result = %d\n", result ? 1 : 0);
+ return result;
+}
+
uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
uint64_t result = 0;
for (int level = 0; level < config::kNumLevels; level++) {
diff --git a/db/version_set.h b/db/version_set.h
index 792899b..9d084fd 100644
--- a/db/version_set.h
+++ b/db/version_set.h
@@ -277,6 +277,8 @@ class VersionSet {
void AppendVersion(Version* v);
+ bool ManifestContains(const std::string& record) const;
+
Env* const env_;
const std::string dbname_;
const Options* const options_;
diff --git a/doc/index.html b/doc/index.html
index cd29341..3ed0ed9 100644
--- a/doc/index.html
+++ b/doc/index.html
@@ -420,7 +420,7 @@ The preceding code associates a
based filtering policy with the database. Bloom filter based
filtering relies on keeping some number of bits of data in memory per
key (in this case 10 bits per key since that is the argument we passed
-to NewBloomFilter). This filter will reduce the number of unnecessary
+to NewBloomFilterPolicy). This filter will reduce the number of unnecessary
disk reads needed for <code>Get()</code> calls by a factor of
approximately a 100. Increasing the bits per key will lead to a
larger reduction at the cost of more memory usage. We recommend that
@@ -430,7 +430,7 @@ lot of random reads set a filter policy.
If you are using a custom comparator, you should ensure that the filter
policy you are using is compatible with your comparator. For example,
consider a comparator that ignores trailing spaces when comparing keys.
-<code>NewBloomFilter</code> must not be used with such a comparator.
+<code>NewBloomFilterPolicy</code> must not be used with such a comparator.
Instead, the application should provide a custom filter policy that
also ignores trailing spaces. For example:
<pre>
@@ -438,7 +438,7 @@ also ignores trailing spaces. For example:
private:
FilterPolicy* builtin_policy_;
public:
- CustomFilterPolicy() : builtin_policy_(NewBloomFilter(10)) { }
+ CustomFilterPolicy() : builtin_policy_(NewBloomFilterPolicy(10)) { }
~CustomFilterPolicy() { delete builtin_policy_; }
const char* Name() const { return "IgnoreTrailingSpacesFilter"; }
diff --git a/doc/log_format.txt b/doc/log_format.txt
index 3a0414b..5228f62 100644
--- a/doc/log_format.txt
+++ b/doc/log_format.txt
@@ -4,8 +4,8 @@ exception is that the tail of the file may contain a partial block.
Each block consists of a sequence of records:
block := record* trailer?
record :=
- checksum: uint32 // crc32c of type and data[]
- length: uint16
+ checksum: uint32 // crc32c of type and data[] ; little-endian
+ length: uint16 // little-endian
type: uint8 // One of FULL, FIRST, MIDDLE, LAST
data: uint8[length]
diff --git a/doc/table_format.txt b/doc/table_format.txt
index d0f3065..ca8f9b4 100644
--- a/doc/table_format.txt
+++ b/doc/table_format.txt
@@ -18,6 +18,8 @@ The file contains internal pointers. Each such pointer is called
a BlockHandle and contains the following information:
offset: varint64
size: varint64
+See https://developers.google.com/protocol-buffers/docs/encoding#varints
+for an explanation of varint64 format.
(1) The sequence of key/value pairs in the file are stored in sorted
order and partitioned into a sequence of data blocks. These blocks
@@ -41,11 +43,11 @@ BlockHandle for the data block.
(6) At the very end of the file is a fixed length footer that contains
the BlockHandle of the metaindex and index blocks as well as a magic number.
- metaindex_handle: char[p]; // Block handle for metaindex
- index_handle: char[q]; // Block handle for index
- padding: char[40-p-q]; // 0 bytes to make fixed length
- // (40==2*BlockHandle::kMaxEncodedLength)
- magic: fixed64; // == 0xdb4775248b80fb57
+ metaindex_handle: char[p]; // Block handle for metaindex
+ index_handle: char[q]; // Block handle for index
+ padding: char[40-p-q]; // zeroed bytes to make fixed length
+ // (40==2*BlockHandle::kMaxEncodedLength)
+ magic: fixed64; // == 0xdb4775248b80fb57 (little-endian)
"filter" Meta Block
-------------------
diff --git a/include/leveldb/c.h b/include/leveldb/c.h
index 9cee971..1fa5886 100644
--- a/include/leveldb/c.h
+++ b/include/leveldb/c.h
@@ -278,6 +278,12 @@ extern void leveldb_env_destroy(leveldb_env_t*);
malloc()-ed memory returned by this library. */
extern void leveldb_free(void* ptr);
+/* Return the major version number for this release. */
+extern int leveldb_major_version();
+
+/* Return the minor version number for this release. */
+extern int leveldb_minor_version();
+
#ifdef __cplusplus
} /* end extern "C" */
#endif
diff --git a/include/leveldb/db.h b/include/leveldb/db.h
index a50ac69..29d3674 100644
--- a/include/leveldb/db.h
+++ b/include/leveldb/db.h
@@ -14,7 +14,7 @@ namespace leveldb {
// Update Makefile if you change these
static const int kMajorVersion = 1;
-static const int kMinorVersion = 6;
+static const int kMinorVersion = 9;
struct Options;
struct ReadOptions;
diff --git a/port/port_posix.h b/port/port_posix.h
index 6ca352e..f2b89bf 100644
--- a/port/port_posix.h
+++ b/port/port_posix.h
@@ -26,11 +26,17 @@
#include <sys/endian.h>
#define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN)
#elif defined(OS_OPENBSD) || defined(OS_NETBSD) ||\
- defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID)
+ defined(OS_DRAGONFLYBSD)
#include <sys/types.h>
#include <sys/endian.h>
#elif defined(OS_HPUX)
#define PLATFORM_IS_LITTLE_ENDIAN false
+#elif defined(OS_ANDROID)
+ // Due to a bug in the NDK x86 <sys/endian.h> definition,
+ // _BYTE_ORDER must be used instead of __BYTE_ORDER on Android.
+ // See http://code.google.com/p/android/issues/detail?id=39824
+ #include <endian.h>
+ #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN)
#else
#include <endian.h>
#endif