Skip to content

Commit

Permalink
Release leveldb 1.11
Browse files Browse the repository at this point in the history
Fixes issues
161
174
178

As well as the issue reported by [email protected] about
MissingSSTFile unit test failing on windows.
  • Loading branch information
davidsgrogan committed Jun 13, 2013
1 parent 28dad91 commit 7b094f1
Show file tree
Hide file tree
Showing 10 changed files with 172 additions and 31 deletions.
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ TESTS = \
env_test \
filename_test \
filter_block_test \
issue178_test \
log_test \
memenv_test \
skiplist_test \
Expand Down Expand Up @@ -69,7 +70,7 @@ SHARED = $(SHARED1)
else
# Update db.h if you change these.
SHARED_MAJOR = 1
SHARED_MINOR = 10
SHARED_MINOR = 11
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
SHARED2 = $(SHARED1).$(SHARED_MAJOR)
SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
Expand Down Expand Up @@ -146,6 +147,9 @@ filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS)
filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)

issue178_test: issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)

log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)

Expand Down
21 changes: 15 additions & 6 deletions db/db_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@

namespace leveldb {

const int kNumNonTableCacheFiles = 10;

// Information kept for every waiting writer
struct DBImpl::Writer {
Status status;
Expand Down Expand Up @@ -92,9 +94,9 @@ Options SanitizeOptions(const std::string& dbname,
Options result = src;
result.comparator = icmp;
result.filter_policy = (src.filter_policy != NULL) ? ipolicy : NULL;
ClipToRange(&result.max_open_files, 20, 50000);
ClipToRange(&result.write_buffer_size, 64<<10, 1<<30);
ClipToRange(&result.block_size, 1<<10, 4<<20);
ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000);
ClipToRange(&result.write_buffer_size, 64<<10, 1<<30);
ClipToRange(&result.block_size, 1<<10, 4<<20);
if (result.info_log == NULL) {
// Open a log file in the same directory as the db
src.env->CreateDir(dbname); // In case it does not exist
Expand Down Expand Up @@ -130,12 +132,13 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
log_(NULL),
tmp_batch_(new WriteBatch),
bg_compaction_scheduled_(false),
manual_compaction_(NULL) {
manual_compaction_(NULL),
consecutive_compaction_errors_(0) {
mem_->Ref();
has_imm_.Release_Store(NULL);

// Reserve ten files or so for other uses and give the rest to TableCache.
const int table_cache_size = options.max_open_files - 10;
const int table_cache_size = options.max_open_files - kNumNonTableCacheFiles;
table_cache_ = new TableCache(dbname_, &options_, table_cache_size);

versions_ = new VersionSet(dbname_, &options_, table_cache_,
Expand Down Expand Up @@ -619,6 +622,7 @@ void DBImpl::BackgroundCall() {
Status s = BackgroundCompaction();
if (s.ok()) {
// Success
consecutive_compaction_errors_ = 0;
} else if (shutting_down_.Acquire_Load()) {
// Error most likely due to shutdown; do not wait
} else {
Expand All @@ -630,7 +634,12 @@ void DBImpl::BackgroundCall() {
Log(options_.info_log, "Waiting after background compaction error: %s",
s.ToString().c_str());
mutex_.Unlock();
env_->SleepForMicroseconds(1000000);
++consecutive_compaction_errors_;
int seconds_to_sleep = 1;
for (int i = 0; i < 3 && i < consecutive_compaction_errors_ - 1; ++i) {
seconds_to_sleep *= 2;
}
env_->SleepForMicroseconds(seconds_to_sleep * 1000000);
mutex_.Lock();
}
}
Expand Down
1 change: 1 addition & 0 deletions db/db_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ class DBImpl : public DB {

// Have we encountered a background error in paranoid mode?
Status bg_error_;
int consecutive_compaction_errors_;

// Per level compaction stats. stats_[level] stores the stats for
// compactions that produced data for the specified "level".
Expand Down
54 changes: 44 additions & 10 deletions db/db_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,11 @@ class AtomicCounter {
public:
AtomicCounter() : count_(0) { }
void Increment() {
IncrementBy(1);
}
void IncrementBy(int count) {
MutexLock l(&mu_);
count_++;
count_ += count;
}
int Read() {
MutexLock l(&mu_);
Expand All @@ -45,6 +48,10 @@ class AtomicCounter {
count_ = 0;
}
};

void DelayMilliseconds(int millis) {
Env::Default()->SleepForMicroseconds(millis * 1000);
}
}

// Special Env used to delay background operations
Expand All @@ -69,6 +76,7 @@ class SpecialEnv : public EnvWrapper {
AtomicCounter random_read_counter_;

AtomicCounter sleep_counter_;
AtomicCounter sleep_time_counter_;

explicit SpecialEnv(Env* base) : EnvWrapper(base) {
delay_sstable_sync_.Release_Store(NULL);
Expand Down Expand Up @@ -103,7 +111,7 @@ class SpecialEnv : public EnvWrapper {
Status Flush() { return base_->Flush(); }
Status Sync() {
while (env_->delay_sstable_sync_.Acquire_Load() != NULL) {
env_->SleepForMicroseconds(100000);
DelayMilliseconds(100);
}
return base_->Sync();
}
Expand Down Expand Up @@ -174,8 +182,9 @@ class SpecialEnv : public EnvWrapper {

virtual void SleepForMicroseconds(int micros) {
sleep_counter_.Increment();
target()->SleepForMicroseconds(micros);
sleep_time_counter_.IncrementBy(micros);
}

};

class DBTest {
Expand Down Expand Up @@ -625,7 +634,7 @@ TEST(DBTest, GetEncountersEmptyLevel) {
}

// Step 4: Wait for compaction to finish
env_->SleepForMicroseconds(1000000);
DelayMilliseconds(1000);

ASSERT_EQ(NumTableFilesAtLevel(0), 0);
} while (ChangeOptions());
Expand Down Expand Up @@ -1309,7 +1318,7 @@ TEST(DBTest, L0_CompactionBug_Issue44_a) {
Reopen();
Reopen();
ASSERT_EQ("(a->v)", Contents());
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
DelayMilliseconds(1000); // Wait for compaction to finish
ASSERT_EQ("(a->v)", Contents());
}

Expand All @@ -1325,7 +1334,7 @@ TEST(DBTest, L0_CompactionBug_Issue44_b) {
Put("","");
Reopen();
Put("","");
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
DelayMilliseconds(1000); // Wait for compaction to finish
Reopen();
Put("d","dv");
Reopen();
Expand All @@ -1335,7 +1344,7 @@ TEST(DBTest, L0_CompactionBug_Issue44_b) {
Delete("b");
Reopen();
ASSERT_EQ("(->)(c->cv)", Contents());
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
DelayMilliseconds(1000); // Wait for compaction to finish
ASSERT_EQ("(->)(c->cv)", Contents());
}

Expand Down Expand Up @@ -1520,6 +1529,30 @@ TEST(DBTest, NoSpace) {
ASSERT_GE(env_->sleep_counter_.Read(), 5);
}

TEST(DBTest, ExponentialBackoff) {
Options options = CurrentOptions();
options.env = env_;
Reopen(&options);

ASSERT_OK(Put("foo", "v1"));
ASSERT_EQ("v1", Get("foo"));
Compact("a", "z");
env_->non_writable_.Release_Store(env_); // Force errors for new files
env_->sleep_counter_.Reset();
env_->sleep_time_counter_.Reset();
for (int i = 0; i < 5; i++) {
dbfull()->TEST_CompactRange(2, NULL, NULL);
}
env_->non_writable_.Release_Store(NULL);

// Wait for compaction to finish
DelayMilliseconds(1000);

ASSERT_GE(env_->sleep_counter_.Read(), 5);
ASSERT_LT(env_->sleep_counter_.Read(), 10);
ASSERT_GE(env_->sleep_time_counter_.Read(), 10e6);
}

TEST(DBTest, NonWritableFileSystem) {
Options options = CurrentOptions();
options.write_buffer_size = 1000;
Expand All @@ -1533,7 +1566,7 @@ TEST(DBTest, NonWritableFileSystem) {
fprintf(stderr, "iter %d; errors %d\n", i, errors);
if (!Put("foo", big).ok()) {
errors++;
env_->SleepForMicroseconds(100000);
DelayMilliseconds(100);
}
}
ASSERT_GT(errors, 0);
Expand Down Expand Up @@ -1589,6 +1622,7 @@ TEST(DBTest, MissingSSTFile) {
dbfull()->TEST_CompactMemTable();
ASSERT_EQ("bar", Get("foo"));

Close();
ASSERT_TRUE(DeleteAnSSTFile());
Options options = CurrentOptions();
options.paranoid_checks = true;
Expand Down Expand Up @@ -1742,13 +1776,13 @@ TEST(DBTest, MultiThreaded) {
}

// Let them run for a while
env_->SleepForMicroseconds(kTestSeconds * 1000000);
DelayMilliseconds(kTestSeconds * 1000);

// Stop the threads and wait for them to finish
mt.stop.Release_Store(&mt);
for (int id = 0; id < kNumThreads; id++) {
while (mt.thread_done[id].Acquire_Load() == NULL) {
env_->SleepForMicroseconds(100000);
DelayMilliseconds(100);
}
}
} while (ChangeOptions());
Expand Down
2 changes: 1 addition & 1 deletion db/filename_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ TEST(FileNameTest, Parse) {
for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
std::string f = errors[i];
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
};
}
}

TEST(FileNameTest, Construction) {
Expand Down
21 changes: 13 additions & 8 deletions db/version_set.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1331,14 +1331,19 @@ Compaction* VersionSet::CompactRange(
}

// Avoid compacting too much in one shot in case the range is large.
const uint64_t limit = MaxFileSizeForLevel(level);
uint64_t total = 0;
for (size_t i = 0; i < inputs.size(); i++) {
uint64_t s = inputs[i]->file_size;
total += s;
if (total >= limit) {
inputs.resize(i + 1);
break;
// But we cannot do this for level-0 since level-0 files can overlap
// and we must not pick one file and drop another older file if the
// two files overlap.
if (level > 0) {
const uint64_t limit = MaxFileSizeForLevel(level);
uint64_t total = 0;
for (size_t i = 0; i < inputs.size(); i++) {
uint64_t s = inputs[i]->file_size;
total += s;
if (total >= limit) {
inputs.resize(i + 1);
break;
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion include/leveldb/db.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ namespace leveldb {

// Update Makefile if you change these
static const int kMajorVersion = 1;
static const int kMinorVersion = 10;
static const int kMinorVersion = 11;

struct Options;
struct ReadOptions;
Expand Down
88 changes: 88 additions & 0 deletions issues/issue178_test.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// Test for issue 178: a manual compaction causes deleted data to reappear.
#include <iostream>
#include <sstream>
#include <cstdlib>

#include "leveldb/db.h"
#include "leveldb/write_batch.h"
#include "util/testharness.h"

namespace {

const int kNumKeys = 1100000;

std::string Key1(int i) {
char buf[100];
snprintf(buf, sizeof(buf), "my_key_%d", i);
return buf;
}

std::string Key2(int i) {
return Key1(i) + "_xxx";
}

class Issue178 { };

TEST(Issue178, Test) {
// Get rid of any state from an old run.
std::string dbpath = leveldb::test::TmpDir() + "/leveldb_cbug_test";
DestroyDB(dbpath, leveldb::Options());

// Open database. Disable compression since it affects the creation
// of layers and the code below is trying to test against a very
// specific scenario.
leveldb::DB* db;
leveldb::Options db_options;
db_options.create_if_missing = true;
db_options.compression = leveldb::kNoCompression;
ASSERT_OK(leveldb::DB::Open(db_options, dbpath, &db));

// create first key range
leveldb::WriteBatch batch;
for (size_t i = 0; i < kNumKeys; i++) {
batch.Put(Key1(i), "value for range 1 key");
}
ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));

// create second key range
batch.Clear();
for (size_t i = 0; i < kNumKeys; i++) {
batch.Put(Key2(i), "value for range 2 key");
}
ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));

// delete second key range
batch.Clear();
for (size_t i = 0; i < kNumKeys; i++) {
batch.Delete(Key2(i));
}
ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));

// compact database
std::string start_key = Key1(0);
std::string end_key = Key1(kNumKeys - 1);
leveldb::Slice least(start_key.data(), start_key.size());
leveldb::Slice greatest(end_key.data(), end_key.size());

// commenting out the line below causes the example to work correctly
db->CompactRange(&least, &greatest);

// count the keys
leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions());
size_t num_keys = 0;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
num_keys++;
}
delete iter;
ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys";

// close database
delete db;
DestroyDB(dbpath, leveldb::Options());
}

} // anonymous namespace

int main(int argc, char** argv) {
return leveldb::test::RunAllTests();
}
2 changes: 1 addition & 1 deletion util/coding_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ TEST(Coding, Varint64) {
values.push_back(power);
values.push_back(power-1);
values.push_back(power+1);
};
}

std::string s;
for (int i = 0; i < values.size(); i++) {
Expand Down
Loading

0 comments on commit 7b094f1

Please sign in to comment.