Merge pull request ceph#2201 from dachary/wip-8496-erasure-code-base-…

…class

erasure code base class

Reviewed-by: Andreas-Joachim Peters <[email protected]>

src/erasure-code/ErasureCode.cc

@@ -0,0 +1,205 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2014 Cloudwatt <[email protected]>
+ *
+ * Author: Loic Dachary <[email protected]>
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include <errno.h>
+#include <vector>
+#include <algorithm>
+
+#include "common/strtol.h"
+#include "ErasureCode.h"
+
+int ErasureCode::minimum_to_decode(const set<int> &want_to_read,
+                                   const set<int> &available_chunks,
+                                   set<int> *minimum)
+{
+  if (includes(available_chunks.begin(), available_chunks.end(),
+	       want_to_read.begin(), want_to_read.end())) {
+    *minimum = want_to_read;
+  } else {
+    unsigned int k = get_data_chunk_count();
+    if (available_chunks.size() < (unsigned)k)
+      return -EIO;
+    set<int>::iterator i;
+    unsigned j;
+    for (i = available_chunks.begin(), j = 0; j < (unsigned)k; ++i, j++)
+      minimum->insert(*i);
+  }
+  return 0;
+}
+
+int ErasureCode::minimum_to_decode_with_cost(const set<int> &want_to_read,
+                                             const map<int, int> &available,
+                                             set<int> *minimum)
+{
+  set <int> available_chunks;
+  for (map<int, int>::const_iterator i = available.begin();
+       i != available.end();
+       ++i)
+    available_chunks.insert(i->first);
+  return minimum_to_decode(want_to_read, available_chunks, minimum);
+}
+
+int ErasureCode::encode_prepare(const bufferlist &raw,
+                                bufferlist *prepared) const
+{
+  unsigned int k = get_data_chunk_count();
+  unsigned int m = get_chunk_count() - k;
+  unsigned blocksize = get_chunk_size(raw.length());
+  unsigned padded_length = blocksize * k;
+  *prepared = raw;
+  if (padded_length - raw.length() > 0) {
+    bufferptr pad(padded_length - raw.length());
+    pad.zero();
+    prepared->push_back(pad);
+  }
+  unsigned coding_length = blocksize * m;
+  bufferptr coding(buffer::create_page_aligned(coding_length));
+  prepared->push_back(coding);
+  prepared->rebuild_page_aligned();
+  return 0;
+}
+
+int ErasureCode::encode(const set<int> &want_to_encode,
+                        const bufferlist &in,
+                        map<int, bufferlist> *encoded)
+{
+  unsigned int k = get_data_chunk_count();
+  unsigned int m = get_chunk_count() - k;
+  bufferlist out;
+  int err = encode_prepare(in, &out);
+  if (err)
+    return err;
+  unsigned blocksize = get_chunk_size(in.length());
+  for (unsigned int i = 0; i < k + m; i++) {
+    bufferlist &chunk = (*encoded)[i];
+    chunk.substr_of(out, i * blocksize, blocksize);
+  }
+  encode_chunks(want_to_encode, encoded);
+  for (unsigned int i = 0; i < k + m; i++) {
+    if (want_to_encode.count(i) == 0)
+      encoded->erase(i);
+  }
+  return 0;
+}
+
+int ErasureCode::encode_chunks(const set<int> &want_to_encode,
+                               map<int, bufferlist> *encoded)
+{
+  assert("ErasureCode::encode_chunks not implemented" == 0);
+}
+
+int ErasureCode::decode(const set<int> &want_to_read,
+                        const map<int, bufferlist> &chunks,
+                        map<int, bufferlist> *decoded)
+{
+  vector<int> have;
+  have.reserve(chunks.size());
+  for (map<int, bufferlist>::const_iterator i = chunks.begin();
+       i != chunks.end();
+       ++i) {
+    have.push_back(i->first);
+  }
+  if (includes(
+	have.begin(), have.end(), want_to_read.begin(), want_to_read.end())) {
+    for (set<int>::iterator i = want_to_read.begin();
+	 i != want_to_read.end();
+	 ++i) {
+      (*decoded)[*i] = chunks.find(*i)->second;
+    }
+    return 0;
+  }
+  unsigned int k = get_data_chunk_count();
+  unsigned int m = get_chunk_count() - k;
+  unsigned blocksize = (*chunks.begin()).second.length();
+  for (unsigned int i =  0; i < k + m; i++) {
+    if (chunks.find(i) == chunks.end()) {
+      bufferptr ptr(buffer::create_page_aligned(blocksize));
+      (*decoded)[i].push_front(ptr);
+    } else {
+      (*decoded)[i] = chunks.find(i)->second;
+      (*decoded)[i].rebuild_page_aligned();
+    }
+  }
+  return decode_chunks(want_to_read, chunks, decoded);
+}
+
+int ErasureCode::decode_chunks(const set<int> &want_to_read,
+                               const map<int, bufferlist> &chunks,
+                               map<int, bufferlist> *decoded)
+{
+  assert("ErasureCode::decode_chunks not implemented" == 0);
+}
+
+int ErasureCode::parse(const map<std::string,std::string> &parameters,
+ ostream *ss)
+{
+  return 0;
+}
+
+int ErasureCode::to_int(const std::string &name,
+			const map<std::string,std::string> &parameters,
+			int *value,
+			int default_value,
+			ostream *ss)
+{
+  if (parameters.find(name) == parameters.end() ||
+      parameters.find(name)->second.size() == 0) {
+    *value = default_value;
+    return 0;
+  }
+  std::string p = parameters.find(name)->second;
+  std::string err;
+  int r = strict_strtol(p.c_str(), 10, &err);
+  if (!err.empty()) {
+    *ss << "could not convert " << name << "=" << p
+	<< " to int because " << err
+	<< ", set to default " << default_value << std::endl;
+    *value = default_value;
+    return -EINVAL;
+  }
+  *value = r;
+  return 0;
+}
+
+int ErasureCode::to_bool(const std::string &name,
+			 const map<std::string,std::string> &parameters,
+			 bool *value,
+			 bool default_value,
+			 ostream *ss)
+{
+  if (parameters.find(name) == parameters.end() ||
+      parameters.find(name)->second.size() == 0) {
+    *value = default_value;
+    return 0;
+  }
+  const std::string p = parameters.find(name)->second;
+  *value = (p == "yes") || (p == "true");
+  return 0;
+}
+
+int ErasureCode::decode_concat(const map<int, bufferlist> &chunks,
+			       bufferlist *decoded)
+{
+  set<int> want_to_read;
+  for (unsigned int i = 0; i < get_data_chunk_count(); i++)
+    want_to_read.insert(i);
+  map<int, bufferlist> decoded_map;
+  int r = decode(want_to_read, chunks, &decoded_map);
+  if (r == 0)
+    for (unsigned int i = 0; i < get_data_chunk_count(); i++)
+      decoded->claim_append(decoded_map[i]);
+  return r;
+}

src/erasure-code/ErasureCode.h

@@ -0,0 +1,78 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph distributed storage system
+ *
+ * Copyright (C) 2014 Cloudwatt <[email protected]>
+ *
+ * Author: Loic Dachary <[email protected]>
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#ifndef CEPH_ERASURE_CODE_H
+#define CEPH_ERASURE_CODE_H
+
+/*! @file ErasureCode.h
+    @brief Base class for erasure code plugins implementors
+
+ */ 
+
+#include "ErasureCodeInterface.h"
+
+namespace ceph {
+
+  class ErasureCode : public ErasureCodeInterface {
+  public:
+    virtual ~ErasureCode() {}
+
+    virtual int minimum_to_decode(const set<int> &want_to_read,
+                                  const set<int> &available_chunks,
+                                  set<int> *minimum);
+
+    virtual int minimum_to_decode_with_cost(const set<int> &want_to_read,
+                                            const map<int, int> &available,
+                                            set<int> *minimum);
+
+    int encode_prepare(const bufferlist &raw, bufferlist *prepared) const;
+
+    virtual int encode(const set<int> &want_to_encode,
+                       const bufferlist &in,
+                       map<int, bufferlist> *encoded);
+
+    virtual int encode_chunks(const set<int> &want_to_encode,
+                              map<int, bufferlist> *encoded);
+
+    virtual int decode(const set<int> &want_to_read,
+                       const map<int, bufferlist> &chunks,
+                       map<int, bufferlist> *decoded);
+
+    virtual int decode_chunks(const set<int> &want_to_read,
+                              const map<int, bufferlist> &chunks,
+                              map<int, bufferlist> *decoded);
+
+    virtual int parse(const map<std::string,std::string> &parameters,
+		      ostream *ss);
+
+    static int to_int(const std::string &name,
+		      const map<std::string,std::string> &parameters,
+		      int *value,
+		      int default_value,
+		      ostream *ss);
+
+    static int to_bool(const std::string &name,
+		       const map<std::string,std::string> &parameters,
+		       bool *value,
+		       bool default_value,
+		       ostream *ss);
+
+    virtual int decode_concat(const map<int, bufferlist> &chunks,
+			      bufferlist *decoded);
+  };
+}
+
+#endif

src/erasure-code/ErasureCodeInterface.h

@@ -302,6 +302,10 @@ namespace ceph {
                        const bufferlist &in,
                        map<int, bufferlist> *encoded) = 0;
 
+
+    virtual int encode_chunks(const set<int> &want_to_encode,
+                              map<int, bufferlist> *encoded) = 0;
+
     /**
      * Decode the **chunks** and store at least **want_to_read**
      * chunks in **decoded**.
@@ -339,6 +343,10 @@ namespace ceph {
                        const map<int, bufferlist> &chunks,
                        map<int, bufferlist> *decoded) = 0;
 
+    virtual int decode_chunks(const set<int> &want_to_read,
+                              const map<int, bufferlist> &chunks,
+                              map<int, bufferlist> *decoded) = 0;
+
     /**
      * Decode the first **get_data_chunk_count()** **chunks** and
      * concatenate them them into **decoded**.
@@ -350,17 +358,7 @@ namespace ceph {
      * @return **0** on success or a negative errno on error.
      */
     virtual int decode_concat(const map<int, bufferlist> &chunks,
-			      bufferlist *decoded) {
-      set<int> want_to_read;
-      for (unsigned int i = 0; i < get_data_chunk_count(); i++)
-	want_to_read.insert(i);
-      map<int, bufferlist> decoded_map;
-      int r = decode(want_to_read, chunks, &decoded_map);
-      if (r == 0)
-	for (unsigned int i = 0; i < get_data_chunk_count(); i++)
-	  decoded->claim_append(decoded_map[i]);
-      return r;
-    }
+			      bufferlist *decoded) = 0;
   };
 
   typedef ceph::shared_ptr<ErasureCodeInterface> ErasureCodeInterfaceRef;

src/erasure-code/Makefile.am

@@ -11,11 +11,13 @@ endif # WITH_BETTER_YASM_ELF64
 
 liberasure_code_la_SOURCES = \
 	erasure-code/ErasureCodePlugin.cc
+
 if LINUX
 liberasure_code_la_LIBADD = -ldl
 endif # LINUX
 noinst_LTLIBRARIES += liberasure_code.la
 
 noinst_HEADERS += \
+	erasure-code/ErasureCode.h \
 	erasure-code/ErasureCodeInterface.h \
 	erasure-code/ErasureCodePlugin.h