Skip to content

Commit

Permalink
added mmap load
Browse files Browse the repository at this point in the history
  • Loading branch information
sunkaicheng committed Aug 25, 2019
1 parent 0866192 commit 34ecd92
Show file tree
Hide file tree
Showing 6 changed files with 795 additions and 236 deletions.
77 changes: 75 additions & 2 deletions examples/dump_load.cc
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#include <iostream>
#include <string>
#include <parallel_hashmap/phmap.h>
#include <parallel_hashmap/phmap_dump.h>

using phmap::flat_hash_map;
using phmap::parallel_flat_hash_map;
Expand All @@ -21,7 +21,7 @@ void dump_load_uint64_uint32() {
phmap::BinaryInputArchive ar_in("./dump.data");
mp2.load(ar_in);
// Iterate and print keys and values g|++
for (const auto& n : mp2)
for (const auto& n : mp2)
std::cout << n.first << "'s value is: " << n.second << "\n";
}

Expand Down Expand Up @@ -49,10 +49,83 @@ void dump_load_parallel_flat_hash_map() {
std::cout << "key: " << n.first << ", value: " << n.second << "\n";
}

#if defined(__linux__)
void mmap_load_uint64_uint32() {
using MapType = flat_hash_map<uint64_t, uint32_t,
phmap::container_internal::hash_default_hash<uint64_t>,
phmap::container_internal::hash_default_eq<uint64_t>,
phmap::MmapAllocator<
phmap::container_internal::Pair<const uint64_t, uint32_t>>>;
MapType mp1;
mp1.reserve(100);
phmap::MmapOutputArchive ar_out("./dump.data");
// Add a new entry
mp1[100] = 99;
mp1[300] = 299;

// Iterate and print keys and values
for (const auto& n : mp1)
std::cout << n.first << "'s value is: " << n.second << "\n";

mp1.mmap_dump(ar_out);
MapType mp2;

phmap::MmapInputArchive ar_in("./dump.data");
mp2.mmap_load(ar_in);
mp2[849242] = 141;
mp2[11] = 1111;
// Iterate and print keys and values g|++
for (const auto& n : mp2)
std::cout << n.first << "'s value is: " << n.second << "\n";
}

void mmap_load_parallel_flat_hash_map() {
using MapType = parallel_flat_hash_map<uint64_t, uint32_t,
phmap::container_internal::hash_default_hash<uint64_t>,
phmap::container_internal::hash_default_eq<uint64_t>,
phmap::MmapAllocator<
phmap::container_internal::Pair<const uint64_t, uint32_t>>,
4,
phmap::NullMutex>;

MapType mp1;
phmap::MmapOutputArchive ar_out("./dump.data");

// Add a new entry
mp1[100] = 99;
mp1[300] = 299;
mp1[101] = 992;
mp1[1300] = 2991;
mp1[1130] = 299;
mp1[2130] = 1299;
// Iterate and print
for (const auto& n : mp1)
std::cout << "key: " << n.first << ", value: " << n.second << "\n";

mp1.mmap_dump(ar_out);
MapType mp2;
phmap::MmapInputArchive ar_in("./dump.data");

mp2.mmap_load(ar_in);
std::cout << "[debug] map capacity: " << mp2.capacity() << ", size: " << mp2.size() << std::endl;
for (size_t i = 0; i < 100; i ++) {
mp2[6771 + i] = i;
}
std::cout << "[debug] map capacity: " << mp2.capacity() << ", size: " << mp2.size() << std::endl;
for (const auto& n : mp2)
std::cout << "key: " << n.first << ", value: " << n.second << "\n";
}
#endif

int main()
{
dump_load_uint64_uint32();
dump_load_parallel_flat_hash_map();

#if defined(__linux__)
mmap_load_uint64_uint32();
mmap_load_parallel_flat_hash_map();
#endif
return 0;
}

115 changes: 17 additions & 98 deletions parallel_hashmap/phmap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,6 @@
#include <utility>
#include <array>
#include <cassert>
#include <iostream>
#include <fstream>
#include <sstream>

#include "phmap_utils.h"
#include "phmap_base.h"
Expand Down Expand Up @@ -1537,62 +1534,17 @@ class raw_hash_set
}
}

template<typename OutputArchive, typename V = value_type>
typename std::enable_if<type_traits_internal::IsDumpableType<V>::value, bool>::type
dump(OutputArchive& ar) {
typename OutputArchive::Guard guard(&ar);
if (!ar.dump(size_)) {
std::cerr << "Failed to dump size_" << std::endl;
return false;
}
if (size_ == 0) {
return true;
}
if (!ar.dump(capacity_)) {
std::cerr << "Failed to dump capacity_" << std::endl;
return false;
}
if (!ar.dump(reinterpret_cast<char*>(ctrl_),
sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) {
template<typename OutputArchive>
bool dump(OutputArchive&);

std::cerr << "Failed to dump ctrl_" << std::endl;
return false;
}
if (!ar.dump(reinterpret_cast<char*>(slots_), sizeof(slot_type) * capacity_)) {
std::cerr << "Failed to dump slot_" << std::endl;
return false;
}
return true;
}
template<typename InputArchive>
bool load(InputArchive&);

template<typename InputArchive, typename V = value_type>
typename std::enable_if<type_traits_internal::IsDumpableType<V>::value, bool>::type
load(InputArchive& ar) {
typename InputArchive::Guard guard(&ar);
if (!ar.load(&size_)){
std::cerr << "Failed to load size_" << std::endl;
return false;
}
if (size_ == 0) {
return true;
}
if (!ar.load(&capacity_)) {
std::cerr << "Failed to load capacity_" << std::endl;
return false;
}
// allocate memory for ctrl_ and slots_
initialize_slots();
if (!ar.load(reinterpret_cast<char*>(ctrl_),
sizeof(ctrl_t) * (capacity_ + Group::kWidth - 1) / Group::kWidth * Group::kWidth)) {
std::cerr << "Failed to load ctrl" << std::endl;
return false;
}
if (!ar.load(reinterpret_cast<char*>(slots_), sizeof(slot_type) * capacity_)) {
std::cerr << "Failed to load slot" << std::endl;
return false;
}
return true;
}
template<typename OutputArchive>
bool mmap_dump(OutputArchive&);

template<typename MmapInputArchive>
bool mmap_load(MmapInputArchive&);

void rehash(size_t n) {
if (n == 0 && capacity_ == 0) return;
Expand Down Expand Up @@ -3199,50 +3151,17 @@ class parallel_hash_set
a.swap(b);
}

template<typename OutputArchive, typename V = value_type>
typename std::enable_if<type_traits_internal::IsDumpableType<V>::value, bool>::type
dump(OutputArchive& ar) {
typename OutputArchive::Guard guard(&ar);
if (! ar.dump(subcnt())) {
std::cerr << "Failed to dump meta!" << std::endl;
return false;
}
for (size_t i = 0; i < sets_.size(); ++i) {
auto& inner = sets_[i];
typename Lockable::UniqueLock m(const_cast<Inner&>(inner));
if (!inner.set_.dump(ar)) {
std::cerr << "Failed to dump submap " << i << std::endl;
return false;
}
}
return true;
}
template<typename OutputArchive>
bool dump(OutputArchive& ar);

template<typename InputArchive, typename V = value_type>
typename std::enable_if<type_traits_internal::IsDumpableType<V>::value, bool>::type
load(InputArchive& ar) {
typename InputArchive::Guard guard(&ar);
size_t submap_count = 0;
if (!ar.load(&submap_count)) {
std::cerr << "Failed to load submap count!" << std::endl;
return false;
}
template<typename InputArchive>
bool load(InputArchive& ar);

if (submap_count != subcnt()) {
std::cerr << "submap count(" << submap_count << ") != N(" << N << ")" << std::endl;
return false;
}
template<typename OutputArchive>
bool mmap_dump(OutputArchive& ar);

for (size_t i = 0; i < submap_count; ++i) {
auto& inner = sets_[i];
typename Lockable::UniqueLock m(const_cast<Inner&>(inner));
if (!inner.set_.load(ar)) {
std::cerr << "Failed to load submap " << i << std::endl;
return false;
}
}
return true;
}
template<typename InputArchive>
bool mmap_load(InputArchive& ar);

private:
template <class Container, typename Enabler>
Expand Down
30 changes: 1 addition & 29 deletions parallel_hashmap/phmap_base.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// ---------------------------------------------------------------------------

#include <algorithm>
#include <cassert>
#include <cstddef>
Expand Down Expand Up @@ -70,35 +71,6 @@ struct EqualTo

namespace type_traits_internal {

template<typename T>
struct PairTrait : public std::false_type {
using first_type = typename std::remove_cv<T>::type;
using second_type = typename std::remove_cv<T>::type;
};

template<typename T1, typename T2>
struct PairTrait<std::pair<T1, T2>>: public std::true_type {
using first_type = T1;
using second_type = T2;
};

template<typename T>
#if defined(__GLIBCXX__) && __GLIBCXX__ < 20150801
struct IsTriviallyCopyable : public std::integral_constant<bool, __has_trivial_copy(T)> {
};
#else
struct IsTriviallyCopyable : public std::is_trivially_copyable<T> {
};
#endif

template<typename V>
struct IsDumpableType {
static constexpr bool value = IsTriviallyCopyable<V>::value
|| (PairTrait<V>::value
&& IsTriviallyCopyable<typename PairTrait<V>::first_type>::value
&& IsTriviallyCopyable<typename PairTrait<V>::second_type>::value);
};

template <typename... Ts>
struct VoidTImpl {
using type = void;
Expand Down
Loading

0 comments on commit 34ecd92

Please sign in to comment.