Skip to content

Commit

Permalink
refactor: refact type (#1398)
Browse files Browse the repository at this point in the history 
* refact: refact type

* fix: fix hybridse cmake

* fix: fix cmake include dir

* refactor: refact mempool

* fix: fix mem_pool_test
  • Loading branch information
@dl239
dl239 authored Mar 25, 2022
1 parent a2cca35 commit 5b21ef9
Show file tree
Hide file tree
Showing 39 changed files with 829 additions and 751 deletions.
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ configure_file(
"${PROJECT_SOURCE_DIR}/src/config.h")

include_directories(
${CMAKE_SOURCE_DIR}/include
${CMAKE_SOURCE_DIR}/hybridse/include
${CMAKE_BINARY_DIR}/hybridse/src
${LLVM_INCLUDE_DIRS}
Expand Down
1 change: 1 addition & 0 deletions hybridse/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ find_package(SWIG REQUIRED)
include(UseSWIG)

include_directories(
${PROJECT_SOURCE_DIR}/../include
${CMAKE_CURRENT_SOURCE_DIR}/include
${LLVM_INCLUDE_DIRS}
${Boost_INCLUDE_DIRS}
Expand Down
3 changes: 3 additions & 0 deletions hybridse/include/codec/list_iterator_codec.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,16 @@
#include "base/fe_object.h"
#include "base/fe_slice.h"
#include "base/iterator.h"
#include "base/string_ref.h"
#include "codec/row.h"
#include "codec/row_list.h"
#include "codec/type_codec.h"
#include "glog/logging.h"
namespace hybridse {
namespace codec {

using openmldb::base::StringRef;

template <class V>
class ArrayListIterator;

Expand Down
203 changes: 12 additions & 191 deletions hybridse/include/codec/type_codec.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,193 +23,14 @@
#include <vector>
#include "base/fe_hash.h"
#include "base/mem_pool.h"
#include "base/string_ref.h"
#include "base/type.h"
#include "glog/logging.h"

namespace hybridse {
namespace codec {

static const uint32_t SEED = 0xe17a1465;

struct StringRef {
StringRef() : size_(0), data_(nullptr) {}
explicit StringRef(const char* str)
: size_(nullptr == str ? 0 : strlen(str)), data_(str) {}
explicit StringRef(uint32_t size, const char* data) : size_(size), data_(data) {}

explicit StringRef(const std::string& str)
: size_(str.size()), data_(str.data()) {}

~StringRef() {}

const inline bool IsNull() const { return nullptr == data_; }
const std::string ToString() const {
return size_ == 0 ? "" : std::string(data_, size_);
}

static int compare(const StringRef& a, const StringRef& b) {
const size_t min_len = (a.size_ < b.size_) ? a.size_ : b.size_;
int r = memcmp(a.data_, b.data_, min_len);
if (r == 0) {
if (a.size_ < b.size_) {
r = -1;
} else if (a.size_ > b.size_) {
r = +1;
}
}
return r;
}

uint32_t size_;
const char* data_;
};

__attribute__((unused)) static const StringRef operator+(const StringRef& a,
const StringRef& b) {
StringRef str;
str.size_ = a.size_ + b.size_;
char* buffer = static_cast<char*>(malloc(str.size_ + 1));
str.data_ = buffer;
if (a.size_ > 0) {
memcpy(buffer, a.data_, a.size_);
}
if (b.size_ > 0) {
memcpy(buffer + a.size_, b.data_, b.size_);
}
buffer[str.size_] = '\0';
return str;
}
__attribute__((unused)) static std::ostream& operator<<(std::ostream& os,
const StringRef& a) {
os << a.ToString();
return os;
}
__attribute__((unused)) static bool operator==(const StringRef& a,
const StringRef& b) {
return 0 == StringRef::compare(a, b);
}
__attribute__((unused)) static bool operator!=(const StringRef& a,
const StringRef& b) {
return 0 != StringRef::compare(a, b);
}
__attribute__((unused)) static bool operator>=(const StringRef& a,
const StringRef& b) {
return StringRef::compare(a, b) >= 0;
}
__attribute__((unused)) static bool operator>(const StringRef& a,
const StringRef& b) {
return StringRef::compare(a, b) > 0;
}
__attribute__((unused)) static bool operator<=(const StringRef& a,
const StringRef& b) {
return StringRef::compare(a, b) <= 0;
}
__attribute__((unused)) static bool operator<(const StringRef& a,
const StringRef& b) {
return StringRef::compare(a, b) < 0;
}

struct Timestamp {
Timestamp() : ts_(0) {}
explicit Timestamp(int64_t ts) : ts_(ts < 0 ? 0 : ts) {}
Timestamp& operator+=(const Timestamp& t1) {
ts_ += t1.ts_;
return *this;
}
Timestamp& operator-=(const Timestamp& t1) {
ts_ -= t1.ts_;
return *this;
}
int64_t ts_;
};

__attribute__((unused)) static const Timestamp operator+(const Timestamp& a,
const Timestamp& b) {
return Timestamp(a.ts_ + b.ts_);
}
__attribute__((unused)) static const Timestamp operator-(const Timestamp& a,
const Timestamp& b) {
return Timestamp(a.ts_ - b.ts_);
}
__attribute__((unused)) static const Timestamp operator/(const Timestamp& a,
const int64_t b) {
return Timestamp(static_cast<int64_t>(a.ts_ / b));
}
__attribute__((unused)) static bool operator>(const Timestamp& a,
const Timestamp& b) {
return a.ts_ > b.ts_;
}
__attribute__((unused)) static bool operator<(const Timestamp& a,
const Timestamp& b) {
return a.ts_ < b.ts_;
}
__attribute__((unused)) static bool operator>=(const Timestamp& a,
const Timestamp& b) {
return a.ts_ >= b.ts_;
}
__attribute__((unused)) static bool operator<=(const Timestamp& a,
const Timestamp& b) {
return a.ts_ <= b.ts_;
}
__attribute__((unused)) static bool operator==(const Timestamp& a,
const Timestamp& b) {
return a.ts_ == b.ts_;
}
__attribute__((unused)) static bool operator!=(const Timestamp& a,
const Timestamp& b) {
return a.ts_ != b.ts_;
}

struct Date {
Date() : date_(0) {}
explicit Date(int32_t date) : date_(date < 0 ? 0 : date) {}
Date(int32_t year, int32_t month, int32_t day) : date_(0) {
if (year < 1900 || year > 9999) {
return;
}
if (month < 1 || month > 12) {
return;
}
if (day < 1 || day > 31) {
return;
}
int32_t data = (year - 1900) << 16;
data = data | ((month - 1) << 8);
data = data | day;
date_ = data;
}
static bool Decode(int32_t date, int32_t* year, int32_t* month,
int32_t* day) {
if (date < 0) {
return false;
}
*day = date & 0x0000000FF;
date = date >> 8;
*month = 1 + (date & 0x0000FF);
*year = 1900 + (date >> 8);
return true;
}
int32_t date_;
};

__attribute__((unused)) static bool operator>(const Date& a, const Date& b) {
return a.date_ > b.date_;
}
__attribute__((unused)) static bool operator<(const Date& a, const Date& b) {
return a.date_ < b.date_;
}
__attribute__((unused)) static bool operator>=(const Date& a, const Date& b) {
return a.date_ >= b.date_;
}
__attribute__((unused)) static bool operator<=(const Date& a, const Date& b) {
return a.date_ <= b.date_;
}
__attribute__((unused)) static bool operator==(const Date& a, const Date& b) {
return a.date_ == b.date_;
}
__attribute__((unused)) static bool operator!=(const Date& a, const Date& b) {
return a.date_ != b.date_;
}

template <typename V = void>
struct ListRef {
int8_t* list;
Expand Down Expand Up @@ -395,15 +216,15 @@ inline float GetFloatField(const int8_t* row, uint32_t idx, uint32_t offset,
}
}

inline Timestamp GetTimestampFieldUnsafe(const int8_t* row, uint32_t offset) {
return Timestamp(*(reinterpret_cast<const int64_t*>(row + offset)));
inline openmldb::base::Timestamp GetTimestampFieldUnsafe(const int8_t* row, uint32_t offset) {
return openmldb::base::Timestamp(*(reinterpret_cast<const int64_t*>(row + offset)));
}

inline Timestamp GetTimestampField(const int8_t* row, uint32_t idx,
inline openmldb::base::Timestamp GetTimestampField(const int8_t* row, uint32_t idx,
uint32_t offset, int8_t* is_null) {
if (row == nullptr || IsNullAt(row, idx)) {
*is_null = true;
return Timestamp();
return openmldb::base::Timestamp();
} else {
*is_null = false;
return GetTimestampFieldUnsafe(row, offset);
Expand Down Expand Up @@ -456,22 +277,22 @@ int32_t GetStrCol(int8_t* input, int32_t row_idx, uint32_t col_idx,
// custom specialization of std::hash for timestamp, date and string
namespace std {
template <>
struct hash<hybridse::codec::Timestamp> {
std::size_t operator()(const hybridse::codec::Timestamp& t) const {
struct hash<openmldb::base::Timestamp> {
std::size_t operator()(const openmldb::base::Timestamp& t) const {
return std::hash<int64_t>()(t.ts_);
}
};

template <>
struct hash<hybridse::codec::Date> {
std::size_t operator()(const hybridse::codec::Date& t) const {
struct hash<openmldb::base::Date> {
std::size_t operator()(const openmldb::base::Date& t) const {
return std::hash<int32_t>()(t.date_);
}
};

template <>
struct hash<hybridse::codec::StringRef> {
std::size_t operator()(const hybridse::codec::StringRef& t) const {
struct hash<openmldb::base::StringRef> {
std::size_t operator()(const openmldb::base::StringRef& t) const {
return hybridse::base::hash(t.data_, t.size_, hybridse::codec::SEED);
}
};
Expand Down
4 changes: 2 additions & 2 deletions hybridse/src/base/mem_pool_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@ class MemPoolTest : public ::testing::Test {

TEST_F(MemPoolTest, ByteMemoryPoolTest) {
{
ByteMemoryPool mem_pool;
::openmldb::base::ByteMemoryPool mem_pool;
char* s1 = mem_pool.Alloc(10);
memcpy(s1, "helloworld", 10);
ASSERT_EQ("helloworld", std::string(s1, 10));
}

{
ByteMemoryPool mem_pool;
::openmldb::base::ByteMemoryPool mem_pool;
const char src1[] = "helloworld";
char* s1 = mem_pool.Alloc(10);
memcpy(s1, src1, strlen(src1));
Expand Down
2 changes: 1 addition & 1 deletion hybridse/src/benchmark/udf_bm_case.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ int32_t RunByteMemPoolAlloc1000(size_t request_size) {
return 1;
}
int32_t RunNewFree1000(size_t request_size) {
hybridse::base::ByteMemoryPool pool;
openmldb::base::ByteMemoryPool pool;
std::vector<char*> chucks;
for (int i = 0; i < 1000; i++) {
chucks.push_back(new char[request_size]);
Expand Down
2 changes: 1 addition & 1 deletion hybridse/src/codec/fe_row_codec.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -621,7 +621,7 @@ int32_t RowView::GetDate(uint32_t idx, int32_t* year, int32_t* month,
return 1;
}
int32_t date = GetDateUnsafe(idx);
codec::Date::Decode(date, year, month, day);
openmldb::base::Date::Decode(date, year, month, day);
return 0;
}
int32_t RowView::GetInteger(const int8_t* row, uint32_t idx,
Expand Down
4 changes: 2 additions & 2 deletions hybridse/src/codec/type_codec.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,11 +305,11 @@ int32_t GetCol(int8_t* input, int32_t row_idx, uint32_t col_idx, int32_t offset,
}
case hybridse::type::kTimestamp: {
new (data)
ColumnImpl<codec::Timestamp>(w, row_idx, col_idx, offset);
ColumnImpl<openmldb::base::Timestamp>(w, row_idx, col_idx, offset);
break;
}
case hybridse::type::kDate: {
new (data) ColumnImpl<codec::Date>(w, row_idx, col_idx, offset);
new (data) ColumnImpl<openmldb::base::Date>(w, row_idx, col_idx, offset);
break;
}
case hybridse::type::kBool: {
Expand Down
23 changes: 12 additions & 11 deletions hybridse/src/codegen/arithmetic_expr_ir_builder_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ ExitOnError ExitOnErr;
namespace hybridse {
namespace codegen {

using hybridse::codec::Timestamp;
using openmldb::base::Timestamp;
using openmldb::base::Date;
using hybridse::udf::Nullable;

class ArithmeticIRBuilderTest : public ::testing::Test {
Expand Down Expand Up @@ -168,11 +169,11 @@ TEST_F(ArithmeticIRBuilderTest, TestAddNull) {
::hybridse::node::kFnOpAdd);
BinaryArithmeticExprCheck<Nullable<Timestamp>, Nullable<Timestamp>,
Nullable<Timestamp>>(
codec::Timestamp(1590115420000L), nullptr, nullptr,
Timestamp(1590115420000L), nullptr, nullptr,
::hybridse::node::kFnOpAdd);
BinaryArithmeticExprCheck<Nullable<Timestamp>, Nullable<Timestamp>,
Nullable<Timestamp>>(
nullptr, codec::Timestamp(1590115420000L), nullptr,
nullptr, Timestamp(1590115420000L), nullptr,
::hybridse::node::kFnOpAdd);
}
TEST_F(ArithmeticIRBuilderTest, TestSubNull) {
Expand Down Expand Up @@ -500,8 +501,8 @@ TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp4) {
::hybridse::node::kFnOpMinus);
}
TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp5) {
BinaryArithmeticErrorCheck<Nullable<codec::Date>, Nullable<codec::Date>,
Nullable<codec::Date>>(
BinaryArithmeticErrorCheck<Nullable<Date>, Nullable<Date>,
Nullable<Date>>(
::hybridse::node::kFnOpMinus);
}
TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp6) {
Expand All @@ -516,8 +517,8 @@ TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp7) {
::hybridse::node::kFnOpMulti);
}
TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp8) {
BinaryArithmeticErrorCheck<Nullable<codec::Date>, Nullable<codec::Date>,
Nullable<codec::Date>>(
BinaryArithmeticErrorCheck<Nullable<Date>, Nullable<Date>,
Nullable<Date>>(
::hybridse::node::kFnOpMulti);
}
TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp9) {
Expand All @@ -532,8 +533,8 @@ TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp10) {
::hybridse::node::kFnOpFDiv);
}
TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp11) {
BinaryArithmeticErrorCheck<Nullable<codec::Date>, Nullable<codec::Date>,
Nullable<codec::Date>>(
BinaryArithmeticErrorCheck<Nullable<Date>, Nullable<Date>,
Nullable<Date>>(
::hybridse::node::kFnOpFDiv);
}
TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp12) {
Expand All @@ -547,8 +548,8 @@ TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp13) {
::hybridse::node::kFnOpDiv);
}
TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp14) {
BinaryArithmeticErrorCheck<Nullable<codec::Date>, Nullable<codec::Date>,
Nullable<codec::Date>>(
BinaryArithmeticErrorCheck<Nullable<Date>, Nullable<Date>,
Nullable<Date>>(
::hybridse::node::kFnOpDiv);
}
TEST_F(ArithmeticIRBuilderTest, TestErrorExprOp15) {
Expand Down
Loading

0 comments on commit 5b21ef9

Please sign in to comment.