Non-locking, read-only opens (hypermodeinc#432)

* Read-only opens

This introduces a read-only option on the DB struct. The main effect
is to force all transactions to be read-only. It also skips anything
that might write to disk.

It will put a shared lock on the directory when it's opened
read-only. This will allow other read-only opens but will deny
read-write opens until all readers have closed. If the database is
already open read-write and an attempt is made to open it
again (read-only or read-write), an error occurs.

It also:

 - Fails if the manifest doesn't exist on read-only open
 - Does not attempt to truncate the manifest
 - Skips compactors and memtable
 - All vlogs are opened read-only

* Avoid truncating vlog in read-only mode

There shouldn't be any need anyways.

* ReadOnly opens fail on corrupt DB

If there is data in a vlog to be replayed, opening the DB with
ReadOnly set will return an error.

* Unplumb the readonly args

The vlog structure already carries the opts from it's Open()

* Use flags instead of bools for OpenExistingSyncedFile

* Comment changes per @manishrjain

* OpenExistingSyncFile -> OpenExistingFile

To reflect that it's not just for synced files.

* Add a placeholder for Windows read-only mode

* Hack tests to pass through Windows failures

These tests are useless on Windows until the placeholder in
dir_windows.go is removed.

* Changes requested by @manishrjain

Arg cleanups + house preferences.

db.go

@@ -173,6 +173,9 @@ func Open(opt Options) (db *DB, err error) {
 			return nil, y.Wrapf(err, "Invalid Dir: %q", path)
 		}
 		if !dirExists {
+			if opt.ReadOnly {
+				return nil, y.Wrapf(err, "Cannot find Dir for read-only open: %q", path)
+			}
 			// Try to create the directory
 			err = os.Mkdir(path, 0700)
 			if err != nil {
@@ -188,8 +191,8 @@ func Open(opt Options) (db *DB, err error) {
 	if err != nil {
 		return nil, err
 	}
-
-	dirLockGuard, err := acquireDirectoryLock(opt.Dir, lockFile)
+	var dirLockGuard, valueDirLockGuard *directoryLockGuard
+	dirLockGuard, err = acquireDirectoryLock(opt.Dir, lockFile, opt.ReadOnly)
 	if err != nil {
 		return nil, err
 	}
@@ -198,9 +201,8 @@ func Open(opt Options) (db *DB, err error) {
 			_ = dirLockGuard.release()
 		}
 	}()
-	var valueDirLockGuard *directoryLockGuard
 	if absValueDir != absDir {
-		valueDirLockGuard, err = acquireDirectoryLock(opt.ValueDir, lockFile)
+		valueDirLockGuard, err = acquireDirectoryLock(opt.ValueDir, lockFile, opt.ReadOnly)
 		if err != nil {
 			return nil, err
 		}
@@ -217,7 +219,7 @@ func Open(opt Options) (db *DB, err error) {
 		opt.ValueLogLoadingMode == options.MemoryMap) {
 		return nil, ErrInvalidLoadingMode
 	}
-	manifestFile, manifest, err := openOrCreateManifestFile(opt.Dir)
+	manifestFile, manifest, err := openOrCreateManifestFile(opt.Dir, opt.ReadOnly)
 	if err != nil {
 		return nil, err
 	}
@@ -256,11 +258,13 @@ func Open(opt Options) (db *DB, err error) {
 		return nil, err
 	}
 
-	db.closers.compactors = y.NewCloser(1)
-	db.lc.startCompact(db.closers.compactors)
+	if !opt.ReadOnly {
+		db.closers.compactors = y.NewCloser(1)
+		db.lc.startCompact(db.closers.compactors)
 
-	db.closers.memtable = y.NewCloser(1)
-	go db.flushMemtable(db.closers.memtable) // Need levels controller to be up.
+		db.closers.memtable = y.NewCloser(1)
+		go db.flushMemtable(db.closers.memtable) // Need levels controller to be up.
+	}
 
 	if err = db.vlog.Open(db, opt); err != nil {
 		return nil, err
@@ -363,11 +367,14 @@ func (db *DB) Close() (err error) {
 	}
 	db.flushChan <- flushTask{nil, valuePointer{}} // Tell flusher to quit.
 
-	db.closers.memtable.Wait()
-	db.elog.Printf("Memtable flushed")
-
-	db.closers.compactors.SignalAndWait()
-	db.elog.Printf("Compaction finished")
+	if db.closers.memtable != nil {
+		db.closers.memtable.Wait()
+		db.elog.Printf("Memtable flushed")
+	}
+	if db.closers.compactors != nil {
+		db.closers.compactors.SignalAndWait()
+		db.elog.Printf("Compaction finished")
+	}
 
 	if lcErr := db.lc.close(); err == nil {
 		err = errors.Wrap(lcErr, "DB.Close")
@@ -377,8 +384,10 @@ func (db *DB) Close() (err error) {
 
 	db.elog.Finish()
 
-	if guardErr := db.dirLockGuard.release(); err == nil {
-		err = errors.Wrap(guardErr, "DB.Close")
+	if db.dirLockGuard != nil {
+		if guardErr := db.dirLockGuard.release(); err == nil {
+			err = errors.Wrap(guardErr, "DB.Close")
+		}
 	}
 	if db.valueDirGuard != nil {
 		if guardErr := db.valueDirGuard.release(); err == nil {

db_test.go

@@ -1506,6 +1506,75 @@ func TestMergeOperatorGetAfterStop(t *testing.T) {
 	})
 }
 
+func TestReadOnly(t *testing.T) {
+	dir, err := ioutil.TempDir("", "badger")
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+	opts := getTestOptions(dir)
+
+	// Create the DB
+	db, err := Open(opts)
+	require.NoError(t, err)
+	for i := 0; i < 10000; i++ {
+		txnSet(t, db, []byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("value%d", i)), 0x00)
+	}
+
+	// Attempt a read-only open while it's open read-write.
+	opts.ReadOnly = true
+	_, err = Open(opts)
+	require.Error(t, err)
+	if err == ErrWindowsNotSupported {
+		return
+	}
+	require.Contains(t, err.Error(), "Another process is using this Badger database")
+	db.Close()
+
+	// Open one read-only
+	opts.ReadOnly = true
+	kv1, err := Open(opts)
+	require.NoError(t, err)
+	defer kv1.Close()
+
+	// Open another read-only
+	kv2, err := Open(opts)
+	require.NoError(t, err)
+	defer kv2.Close()
+
+	// Attempt a read-write open while it's open for read-only
+	opts.ReadOnly = false
+	_, err = Open(opts)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "Another process is using this Badger database")
+
+	// Get a thing from the DB
+	txn1 := kv1.NewTransaction(true)
+	v1, err := txn1.Get([]byte("key1"))
+	require.NoError(t, err)
+	b1, err := v1.Value()
+	require.NoError(t, err)
+	require.Equal(t, b1, []byte("value1"))
+	err = txn1.Commit(nil)
+	require.NoError(t, err)
+
+	// Get a thing from the DB via the other connection
+	txn2 := kv2.NewTransaction(true)
+	v2, err := txn2.Get([]byte("key2000"))
+	require.NoError(t, err)
+	b2, err := v2.Value()
+	require.NoError(t, err)
+	require.Equal(t, b2, []byte("value2000"))
+	err = txn2.Commit(nil)
+	require.NoError(t, err)
+
+	// Attempt to set a value on a read-only connection
+	txn := kv1.NewTransaction(true)
+	err = txn.SetWithMeta([]byte("key"), []byte("value"), 0x00)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "No sets or deletes are allowed in a read-only transaction")
+	err = txn.Commit(nil)
+	require.NoError(t, err)
+}
+
 func ExampleOpen() {
 	dir, err := ioutil.TempDir("", "badger")
 	if err != nil {

dir_unix.go

@@ -35,11 +35,14 @@ type directoryLockGuard struct {
 	f *os.File
 	// The absolute path to our pid file.
 	path string
+	// Was this a shared lock for a read-only database?
+	readOnly bool
 }
 
-// acquireDirectoryLock gets an exclusive lock on the directory (using flock).  It writes our pid
-// to dirPath/pidFileName for convenience.
-func acquireDirectoryLock(dirPath string, pidFileName string) (*directoryLockGuard, error) {
+// acquireDirectoryLock gets a lock on the directory (using flock). If
+// this is not read-only, it will also write our pid to
+// dirPath/pidFileName for convenience.
+func acquireDirectoryLock(dirPath string, pidFileName string, readOnly bool) (*directoryLockGuard, error) {
 	// Convert to absolute path so that Release still works even if we do an unbalanced
 	// chdir in the meantime.
 	absPidFilePath, err := filepath.Abs(filepath.Join(dirPath, pidFileName))
@@ -50,30 +53,40 @@ func acquireDirectoryLock(dirPath string, pidFileName string) (*directoryLockGua
 	if err != nil {
 		return nil, errors.Wrapf(err, "cannot open directory %q", dirPath)
 	}
-	err = unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB)
+	opts := unix.LOCK_EX | unix.LOCK_NB
+	if readOnly {
+		opts = unix.LOCK_SH | unix.LOCK_NB
+	}
+
+	err = unix.Flock(int(f.Fd()), opts)
 	if err != nil {
 		f.Close()
 		return nil, errors.Wrapf(err,
 			"Cannot acquire directory lock on %q.  Another process is using this Badger database.",
 			dirPath)
 	}
 
-	// Yes, we happily overwrite a pre-existing pid file.  We're the only badger process using this
-	// directory.
-	err = ioutil.WriteFile(absPidFilePath, []byte(fmt.Sprintf("%d\n", os.Getpid())), 0666)
-	if err != nil {
-		f.Close()
-		return nil, errors.Wrapf(err,
-			"Cannot write pid file %q", absPidFilePath)
+	if !readOnly {
+		// Yes, we happily overwrite a pre-existing pid file.  We're the
+		// only read-write badger process using this directory.
+		err = ioutil.WriteFile(absPidFilePath, []byte(fmt.Sprintf("%d\n", os.Getpid())), 0666)
+		if err != nil {
+			f.Close()
+			return nil, errors.Wrapf(err,
+				"Cannot write pid file %q", absPidFilePath)
+		}
 	}
-
-	return &directoryLockGuard{f, absPidFilePath}, nil
+	return &directoryLockGuard{f, absPidFilePath, readOnly}, nil
 }
 
 // Release deletes the pid file and releases our lock on the directory.
 func (guard *directoryLockGuard) release() error {
-	// It's important that we remove the pid file first.
-	err := os.Remove(guard.path)
+	var err error
+	if !guard.readOnly {
+		// It's important that we remove the pid file first.
+		err = os.Remove(guard.path)
+	}
+
 	if closeErr := guard.f.Close(); err == nil {
 		err = closeErr
 	}

dir_windows.go

@@ -57,7 +57,11 @@ type directoryLockGuard struct {
 }
 
 // AcquireDirectoryLock acquires exclusive access to a directory.
-func acquireDirectoryLock(dirPath string, pidFileName string) (*directoryLockGuard, error) {
+func acquireDirectoryLock(dirPath string, pidFileName string, readOnly bool) (*directoryLockGuard, error) {
+	if readOnly {
+		return nil, ErrWindowsNotSupported
+	}
+
 	// Convert to absolute path so that Release still works even if we do an unbalanced
 	// chdir in the meantime.
 	absLockFilePath, err := filepath.Abs(filepath.Join(dirPath, pidFileName))

errors.go

@@ -81,6 +81,13 @@ var (
 	// ErrInvalidLoadingMode is returned when opt.ValueLogLoadingMode option is not
 	// within the valid range
 	ErrInvalidLoadingMode = errors.New("Invalid ValueLogLoadingMode, must be FileIO or MemoryMap")
+
+	// ErrCorruptDatabase is returned when opt.ReadOnly is set but the
+	// database requires a value log replay.
+	ErrCorruptDatabase = errors.New("database was not properly closed, cannot open read-only")
+
+	// ErrWindowsNotSupported is returned when opt.ReadOnly is used on Windows
+	ErrWindowsNotSupported = errors.New("Read-only mode is not supported on Windows")
 )
 
 // Key length can't be more than uint16, as determined by table::header.

levels.go

@@ -104,7 +104,11 @@ func newLevelsController(kv *DB, mf *Manifest) (*levelsController, error) {
 	var maxFileID uint64
 	for fileID, tableManifest := range mf.Tables {
 		fname := table.NewFilename(fileID, kv.opt.Dir)
-		fd, err := y.OpenExistingSyncedFile(fname, true)
+		var flags uint32 = y.Sync
+		if kv.opt.ReadOnly {
+			flags |= y.ReadOnly
+		}
+		fd, err := y.OpenExistingFile(fname, flags)
 		if err != nil {
 			closeAllTables(tables)
 			return nil, errors.Wrapf(err, "Opening file: %q", fname)

manifest.go

@@ -112,17 +112,24 @@ func (m *Manifest) clone() Manifest {
 
 // openOrCreateManifestFile opens a Badger manifest file if it exists, or creates on if
 // one doesn’t.
-func openOrCreateManifestFile(dir string) (ret *manifestFile, result Manifest, err error) {
-	return helpOpenOrCreateManifestFile(dir, manifestDeletionsRewriteThreshold)
+func openOrCreateManifestFile(dir string, readOnly bool) (ret *manifestFile, result Manifest, err error) {
+	return helpOpenOrCreateManifestFile(dir, readOnly, manifestDeletionsRewriteThreshold)
 }
 
-func helpOpenOrCreateManifestFile(dir string, deletionsThreshold int) (ret *manifestFile, result Manifest, err error) {
+func helpOpenOrCreateManifestFile(dir string, readOnly bool, deletionsThreshold int) (ret *manifestFile, result Manifest, err error) {
 	path := filepath.Join(dir, ManifestFilename)
-	fp, err := y.OpenExistingSyncedFile(path, false) // We explicitly sync in addChanges, outside the lock.
+	var flags uint32
+	if readOnly {
+		flags |= y.ReadOnly
+	}
+	fp, err := y.OpenExistingFile(path, flags) // We explicitly sync in addChanges, outside the lock.
 	if err != nil {
 		if !os.IsNotExist(err) {
 			return nil, Manifest{}, err
 		}
+		if readOnly {
+			return nil, Manifest{}, fmt.Errorf("no manifest found, required for read-only db")
+		}
 		m := createManifest()
 		fp, netCreations, err := helpRewrite(dir, &m)
 		if err != nil {
@@ -144,12 +151,13 @@ func helpOpenOrCreateManifestFile(dir string, deletionsThreshold int) (ret *mani
 		return nil, Manifest{}, err
 	}
 
-	// Truncate file so we don't have a half-written entry at the end.
-	if err := fp.Truncate(truncOffset); err != nil {
-		_ = fp.Close()
-		return nil, Manifest{}, err
+	if !readOnly {
+		// Truncate file so we don't have a half-written entry at the end.
+		if err := fp.Truncate(truncOffset); err != nil {
+			_ = fp.Close()
+			return nil, Manifest{}, err
+		}
 	}
-
 	if _, err = fp.Seek(0, io.SeekEnd); err != nil {
 		_ = fp.Close()
 		return nil, Manifest{}, err
@@ -256,7 +264,7 @@ func helpRewrite(dir string, m *Manifest) (*os.File, int, error) {
 	if err := os.Rename(rewritePath, manifestPath); err != nil {
 		return nil, 0, err
 	}
-	fp, err = y.OpenExistingSyncedFile(manifestPath, false)
+	fp, err = y.OpenExistingFile(manifestPath, 0)
 	if err != nil {
 		return nil, 0, err
 	}

manifest_test.go

@@ -210,7 +210,7 @@ func TestManifestRewrite(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(dir)
 	deletionsThreshold := 10
-	mf, m, err := helpOpenOrCreateManifestFile(dir, deletionsThreshold)
+	mf, m, err := helpOpenOrCreateManifestFile(dir, false, deletionsThreshold)
 	defer func() {
 		if mf != nil {
 			mf.close()
@@ -236,7 +236,7 @@ func TestManifestRewrite(t *testing.T) {
 	err = mf.close()
 	require.NoError(t, err)
 	mf = nil
-	mf, m, err = helpOpenOrCreateManifestFile(dir, deletionsThreshold)
+	mf, m, err = helpOpenOrCreateManifestFile(dir, false, deletionsThreshold)
 	require.NoError(t, err)
 	require.Equal(t, map[uint64]tableManifest{
 		uint64(deletionsThreshold * 3): {Level: 0},

options.go

@@ -86,6 +86,12 @@ type Options struct {
 
 	maxBatchCount int64 // max entries in batch
 	maxBatchSize  int64 // max batch size in bytes
+
+	// Open the DB as read-only. With this set, multiple processes can
+	// open the same Badger DB. Note: if the DB being opened had crashed
+	// before and has vlog data to be replayed, ReadOnly will cause Open
+	// to fail with an appropriate message.
+	ReadOnly bool
 }
 
 // DefaultOptions sets a list of recommended options for good performance.

transaction.go

@@ -511,6 +511,11 @@ func (txn *Txn) Commit(callback func(error)) error {
 //  defer txn.Discard()
 //  // Call various APIs.
 func (db *DB) NewTransaction(update bool) *Txn {
+	if db.opt.ReadOnly && update {
+		// DB is read-only, force read-only transaction.
+		update = false
+	}
+
 	txn := &Txn{
 		update: update,
 		db:     db,