Generalize LMDB usage and add support for multiple backing stores

Signed-off-by: Victor Porof <[email protected]>

examples/iterator.rs

@@ -7,6 +7,16 @@
 //!
 //!     cargo run --example iterator
 
+use std::fs;
+use std::str;
+
+use tempfile::Builder;
+
+use rkv::backend::{
+    Lmdb,
+    LmdbDatabase,
+    LmdbEnvironment,
+};
 use rkv::{
     Manager,
     Rkv,
@@ -15,17 +25,13 @@ use rkv::{
     StoreOptions,
     Value,
 };
-use tempfile::Builder;
-
-use std::fs;
-use std::str;
 
 fn main() {
     let root = Builder::new().prefix("iterator").tempdir().unwrap();
     fs::create_dir_all(root.path()).unwrap();
     let p = root.path();
 
-    let created_arc = Manager::singleton().write().unwrap().get_or_create(p, Rkv::new).unwrap();
+    let created_arc = Manager::singleton().write().unwrap().get_or_create(p, Rkv::new::<Lmdb>).unwrap();
     let k = created_arc.read().unwrap();
     let store = k.open_single("store", StoreOptions::create()).unwrap();
 
@@ -58,7 +64,7 @@ fn main() {
     }
 }
 
-fn populate_store(k: &Rkv, store: SingleStore) -> Result<(), StoreError> {
+fn populate_store(k: &Rkv<LmdbEnvironment>, store: SingleStore<LmdbDatabase>) -> Result<(), StoreError> {
     let mut writer = k.write()?;
     for (country, city) in vec![
         ("Canada", Value::Str("Ottawa")),

examples/simple-store.rs

@@ -7,17 +7,25 @@
 //!
 //!     cargo run --example simple-store
 
+use std::fs;
+
+use tempfile::Builder;
+
+use rkv::backend::{
+    BackendStat,
+    Lmdb,
+    LmdbDatabase,
+    LmdbRwTransaction,
+};
 use rkv::{
     Manager,
-    MultiStore,
     Rkv,
     StoreOptions,
     Value,
-    Writer,
 };
-use tempfile::Builder;
 
-use std::fs;
+type MultiStore = rkv::MultiStore<LmdbDatabase>;
+type Writer<'env> = rkv::Writer<LmdbRwTransaction<'env>>;
 
 fn getput<'env, 's>(store: MultiStore, writer: &'env mut Writer, ids: &'s mut Vec<String>) {
     let keys = vec!["str1", "str2", "str3"];
@@ -53,12 +61,11 @@ fn main() {
     let p = root.path();
 
     // The manager enforces that each process opens the same lmdb environment at most once
-    let created_arc = Manager::singleton().write().unwrap().get_or_create(p, Rkv::new).unwrap();
+    let created_arc = Manager::singleton().write().unwrap().get_or_create(p, Rkv::new::<Lmdb>).unwrap();
     let k = created_arc.read().unwrap();
 
     // Creates a store called "store"
     let store = k.open_single("store", StoreOptions::create()).unwrap();
-
     let multistore = k.open_multi("multistore", StoreOptions::create()).unwrap();
 
     println!("Inserting data...");
@@ -95,6 +102,7 @@ fn main() {
         delete(multistore, &mut writer);
         writer.commit().unwrap();
     }
+
     println!("Looking up keys...");
     {
         // Use a reader to query the store
@@ -179,5 +187,6 @@ fn main() {
         println!("Get from store value: {:?}", store.get(&reader, "foo").unwrap());
         println!("Get from another store value: {:?}", another_store.get(&reader, "foo").unwrap());
     }
+
     println!("Environment statistics: btree depth = {}", k.stat().unwrap().depth());
 }

src/backend.rs

@@ -0,0 +1,24 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+mod common;
+mod impl_lmdb;
+mod traits;
+
+pub use common::*;
+pub use traits::*;
+
+pub use impl_lmdb::DatabaseImpl as LmdbDatabase;
+pub use impl_lmdb::EnvironmentBuilderImpl as Lmdb;
+pub use impl_lmdb::EnvironmentImpl as LmdbEnvironment;
+pub use impl_lmdb::ErrorImpl as LmdbError;
+pub use impl_lmdb::RoCursorImpl as LmdbRoCursor;
+pub use impl_lmdb::RoTransactionImpl as LmdbRoTransaction;
+pub use impl_lmdb::RwTransactionImpl as LmdbRwTransaction;

src/backend/common.rs

@@ -0,0 +1,41 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#![allow(non_camel_case_types)]
+
+pub enum EnvironmentFlags {
+    FIXED_MAP,
+    NO_SUB_DIR,
+    WRITE_MAP,
+    READ_ONLY,
+    NO_META_SYNC,
+    NO_SYNC,
+    MAP_ASYNC,
+    NO_TLS,
+    NO_LOCK,
+    NO_READAHEAD,
+    NO_MEM_INIT,
+}
+
+pub enum DatabaseFlags {
+    REVERSE_KEY,
+    DUP_SORT,
+    INTEGER_KEY,
+    DUP_FIXED,
+    INTEGER_DUP,
+    REVERSE_DUP,
+}
+
+pub enum WriteFlags {
+    NO_OVERWRITE,
+    NO_DUP_DATA,
+    CURRENT,
+    APPEND,
+    APPEND_DUP,
+}

src/backend/impl_lmdb.rs

@@ -0,0 +1,42 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+mod cursor;
+mod database;
+mod environment;
+mod error;
+mod flags;
+mod info;
+mod iter;
+mod stat;
+mod transaction;
+
+pub use cursor::{
+    RoCursorImpl,
+    RwCursorImpl,
+};
+pub use database::DatabaseImpl;
+pub use environment::{
+    EnvironmentBuilderImpl,
+    EnvironmentImpl,
+};
+pub use error::ErrorImpl;
+pub use flags::{
+    DatabaseFlagsImpl,
+    EnvironmentFlagsImpl,
+    WriteFlagsImpl,
+};
+pub use info::InfoImpl;
+pub use iter::IterImpl;
+pub use stat::StatImpl;
+pub use transaction::{
+    RoTransactionImpl,
+    RwTransactionImpl,
+};

src/backend/impl_lmdb/cursor.rs

@@ -0,0 +1,64 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use lmdb::Cursor;
+
+use super::IterImpl;
+use crate::backend::traits::BackendRoCursor;
+
+#[derive(Debug)]
+pub struct RoCursorImpl<'env>(pub(crate) lmdb::RoCursor<'env>);
+
+impl<'env> BackendRoCursor<'env> for RoCursorImpl<'env> {
+    type Iter = IterImpl<'env>;
+
+    fn iter(&mut self) -> Self::Iter {
+        IterImpl(self.0.iter())
+    }
+
+    fn iter_from<K>(&mut self, key: K) -> Self::Iter
+    where
+        K: AsRef<[u8]>,
+    {
+        IterImpl(self.0.iter_from(key))
+    }
+
+    fn iter_dup_of<K>(&mut self, key: K) -> Self::Iter
+    where
+        K: AsRef<[u8]>,
+    {
+        IterImpl(self.0.iter_dup_of(key))
+    }
+}
+
+#[derive(Debug)]
+pub struct RwCursorImpl<'env>(pub(crate) lmdb::RwCursor<'env>);
+
+impl<'env> BackendRoCursor<'env> for RwCursorImpl<'env> {
+    type Iter = IterImpl<'env>;
+
+    fn iter(&mut self) -> Self::Iter {
+        IterImpl(self.0.iter())
+    }
+
+    fn iter_from<K>(&mut self, key: K) -> Self::Iter
+    where
+        K: AsRef<[u8]>,
+    {
+        IterImpl(self.0.iter_from(key))
+    }
+
+    fn iter_dup_of<K>(&mut self, key: K) -> Self::Iter
+    where
+        K: AsRef<[u8]>,
+    {
+        IterImpl(self.0.iter_dup_of(key))
+    }
+}

src/backend/impl_lmdb/database.rs

@@ -0,0 +1,16 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use crate::backend::traits::BackendDatabase;
+
+#[derive(Debug, Eq, PartialEq, Copy, Clone)]
+pub struct DatabaseImpl(pub(crate) lmdb::Database);
+
+impl BackendDatabase for DatabaseImpl {}