Add bulk insert support

jesse-r-s-hines · Mar 28, 2022 · eaf3c42 · eaf3c42
1 parent 0c7b6ee
commit eaf3c42
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 3 deletions.
diff --git a/src/stores.cpp b/src/stores.cpp
@@ -1,6 +1,8 @@
 #include <memory>
 #include <filesystem>
 #include <fstream>
+#include <vector>
+#include <utility>
 
 #include "stores.h"
 
@@ -9,7 +11,7 @@ namespace stores {
     using fs::path;
     using std::ofstream, std::ifstream;
     using namespace std::string_literals;
-    using std::string, std::to_string, std::tuple, std::function, std::unique_ptr, std::make_unique;
+    using std::string, std::to_string, std::vector, std::pair, std::tuple, std::function, std::unique_ptr, std::make_unique;
     using uint = unsigned int;
 
 
@@ -29,6 +31,16 @@ namespace stores {
         _count--;
     };
 
+    void Store::_bulkInsert(const vector<pair<string, string>>& items) {
+        for (auto& [key, value] : items)
+            this->_insert(key, value);
+    }
+
+    void Store::bulkInsert(const vector<pair<string, string>>& items) {
+        this->_bulkInsert(items);
+        _count += items.size();
+    }
+
 
 
     SQLite3Store::SQLite3Store(const path& filepath, int flags) : Store(filepath) {

diff --git a/src/stores.h b/src/stores.h
@@ -6,6 +6,8 @@
 #include <memory>
 #include <map>
 #include <filesystem>
+#include <vector>
+#include <utility>
 
 #include <sqlite3.h>
 #include "rocksdb/db.h"
@@ -26,7 +28,8 @@ namespace stores {
         virtual void _update(const std::string& key, const std::string& value) = 0;
         virtual std::string _get(const std::string& key) = 0;
         virtual void _remove(const std::string& key) = 0;
-
+
+        virtual void _bulkInsert(const std::vector<std::pair<std::string, std::string>>& items);
     public:
         const std::filesystem::path filepath;
 
@@ -40,6 +43,9 @@ namespace stores {
         void update(const std::string& key, const std::string& value);
         std::string get(const std::string& key);
         void remove(const std::string& key);
+
+        /** A potentially more efficient bulk insert. All items should be unique. */
+        void bulkInsert(const std::vector<std::pair<std::string, std::string>>& items);
     };
 
     /**

diff --git a/src/tests.cpp b/src/tests.cpp
@@ -17,7 +17,7 @@ namespace tests {
     namespace fs = std::filesystem;
     using fs::path;
     using namespace std::string_literals;
-    using std::string, std::vector, std::map, std::function, std::unique_ptr, std::make_unique;
+    using std::string, std::vector, std::map, std::pair, std::function, std::unique_ptr, std::make_unique;
     using stores::Store;
 
     const string filepath = path("out") / "tests" / "store";
@@ -108,4 +108,19 @@ namespace tests {
             }
         }
     }
+
+    TEST_CASE("Test bulk insert") {
+        // Make a bunch of records. Run this last so we can examine the db manually as well.
+        fs::remove_all("out/tests");
+        fs::create_directories("out/tests/");
+
+        for (auto& storeFactory : storeFactories) {
+            auto store = storeFactory();
+            string a = utils::randHash(32), b = utils::randHash(32), c = utils::randHash(32);
+            vector<pair<string, string>> data{ {a, "1"}, {b, "2"}, {c, "3"} };
+            store->bulkInsert(data);
+            REQUIRE(store->get(b) == "2");
+            REQUIRE(store->count() == 3);
+        }
+    }
 }