Merge pull request #1 from 4paradigm/main

refactor(4paradigm#2168): reimpl `dayofyear` udf (4paradigm#2175)

hybridse/src/codegen/udf_ir_builder_test.cc

@@ -115,7 +115,7 @@ TEST_F(UdfIRBuilderTest, dayofweek_date_udf_test) {
  Date date(2020, 05, 22);
  CheckUdf<int32_t, Date>("dayofweek", 6, date);
 }
-TEST_F(UdfIRBuilderTest, dayofyear_date_udf_test) {
+TEST_F(UdfIRBuilderTest, DayofyearDateUdfTest) {
  {
  Date date(2020, 05, 22);
  CheckUdf<int32_t, Date>("dayofyear", 143, date);
@@ -134,31 +134,31 @@ TEST_F(UdfIRBuilderTest, dayofyear_date_udf_test) {
  }
  {
  Date date(2021, 13, 31);
- CheckUdf<int32_t, Date>("dayofyear", 0, date);
+ CheckUdf<Nullable<int32_t>, Date>("dayofyear", nullptr, date);
  }
  {
  Date date(2021, 0, 31);
- CheckUdf<int32_t, Date>("dayofyear", 0, date);
+ CheckUdf<Nullable<int32_t>, Date>("dayofyear", nullptr, date);
  }
  {
  Date date(2021, -1, 31);
- CheckUdf<int32_t, Date>("dayofyear", 0, date);
+ CheckUdf<Nullable<int32_t>, Date>("dayofyear", nullptr, date);
  }
  {
  Date date(2021, 12, 32);
- CheckUdf<int32_t, Date>("dayofyear", 0, date);
+ CheckUdf<Nullable<int32_t>, Date>("dayofyear", nullptr, date);
  }
  {
  Date date(2021, 12, 0);
- CheckUdf<int32_t, Date>("dayofyear", 0, date);
+ CheckUdf<Nullable<int32_t>, Date>("dayofyear", nullptr, date);
  }
  {
  Date date(2021, 12, -10);
- CheckUdf<int32_t, Date>("dayofyear", 0, date);
+ CheckUdf<Nullable<int32_t>, Date>("dayofyear", nullptr, date);
  }
  {
  Date date(2021, 2, 29);
- CheckUdf<int32_t, Date>("dayofyear", 0, date);
+ CheckUdf<Nullable<int32_t>, Date>("dayofyear", nullptr, date);
  }
 }
 TEST_F(UdfIRBuilderTest, weekofyear_date_udf_test) {
@@ -225,7 +225,7 @@ TEST_F(UdfIRBuilderTest, dayofweek_timestamp_udf_test) {
  Timestamp time(1590115420000L);
  CheckUdf<int32_t, Timestamp>("dayofweek", 6, time);
 }
-TEST_F(UdfIRBuilderTest, dayofyear_timestamp_udf_test) {
+TEST_F(UdfIRBuilderTest, DayofyearTimestampUdfTest) {
  Timestamp time(1590115420000L);
  CheckUdf<int32_t, Timestamp>("dayofyear", 143, time);
 }
@@ -272,11 +272,13 @@ TEST_F(UdfIRBuilderTest, dayofweek_int64_udf_test) {
  CheckUdf<int32_t, int64_t>("dayofweek", 1, 1590115420000L + 2 * 86400000L);
  CheckUdf<int32_t, int64_t>("dayofweek", 2, 1590115420000L + 3 * 86400000L);
 }
-TEST_F(UdfIRBuilderTest, dayofyear_int64_udf_test) {
+TEST_F(UdfIRBuilderTest, DayofyearInt64UdfTest) {
  CheckUdf<int32_t, int64_t>("dayofyear", 143, 1590115420000L);
  CheckUdf<int32_t, int64_t>("dayofyear", 144, 1590115420000L + 86400000L);
  CheckUdf<int32_t, int64_t>("dayofyear", 145, 1590115420000L + 2 * 86400000L);
  CheckUdf<int32_t, int64_t>("dayofyear", 146, 1590115420000L + 3 * 86400000L);
+
+ CheckUdf<Nullable<int32_t>, int64_t>("dayofyear", nullptr, -1);
 }
 TEST_F(UdfIRBuilderTest, weekofyear_int64_udf_test) {
  CheckUdf<int32_t, int64_t>("weekofyear", 21, 1590115420000L);

hybridse/src/udf/default_udf_library.cc

@@ -2019,11 +2019,8 @@ void DefaultUdfLibrary::InitTimeAndDateUdf() {
  @since 0.4.0
  )");
 
- RegisterExternal("dayofyear")
- .args<int64_t>(static_cast<int32_t (*)(int64_t)>(v1::dayofyear))
- .args<Timestamp>(static_cast<int32_t (*)(Timestamp*)>(v1::dayofyear))
- .args<Date>(static_cast<int32_t (*)(Date*)>(v1::dayofyear))
- .doc(R"(
+ const std::string dayofyear_doc =
+ R"(
  @brief Return the day of year for a timestamp or date. Returns 0 given an invalid date.
 
  Example:
@@ -2041,7 +2038,25 @@ void DefaultUdfLibrary::InitTimeAndDateUdf() {
  -- output 0
  @endcode
  @since 0.1.0
- )");
+ )";
+
+ RegisterExternal("dayofyear")
+ .args<int64_t>(reinterpret_cast<void*>(static_cast<void (*)(int64_t, int32_t*, bool*)>(v1::dayofyear)))
+ .return_by_arg(true)
+ .returns<Nullable<int32_t>>()
+ .doc(dayofyear_doc);
+
+ RegisterExternal("dayofyear")
+ .args<Timestamp>(reinterpret_cast<void*>(static_cast<void (*)(Timestamp*, int32_t*, bool*)>(v1::dayofyear)))
+ .return_by_arg(true)
+ .returns<Nullable<int32_t>>()
+ .doc(dayofyear_doc);
+
+ RegisterExternal("dayofyear")
+ .args<Date>(reinterpret_cast<void*>(static_cast<void (*)(Date*, int32_t*, bool*)>(v1::dayofyear)))
+ .return_by_arg(true)
+ .returns<Nullable<int32_t>>()
+ .doc(dayofyear_doc);
 
  RegisterExternal("weekofyear")
  .args<int64_t>(static_cast<int32_t (*)(int64_t)>(v1::weekofyear))

hybridse/src/udf/literal_traits.h

@@ -67,6 +67,15 @@ struct Nullable {
  bool is_null() const { return is_null_; }
  T* ptr() { return &data_; }
 
+ // teach gtest print values
+ friend std::ostream& operator<<(std::ostream& os, const Nullable<T>& val) {
+ if (val.is_null_) {
+ return os << "Nullable{null, type=" << DataTypeTrait<T>::to_string() << "}";
+ }
+
+ return os << "Nullable{value=" << val.data_ << ", type=" << DataTypeTrait<T>::to_string() << "}";
+ }
+
  T data_;
  bool is_null_;
 };
@@ -81,6 +90,14 @@ struct Nullable<StringRef> {
  bool is_null() const { return is_null_; }
  StringRef* ptr() { return &data_; }
 
+ friend std::ostream& operator<<(std::ostream& os, const Nullable<StringRef>& val) {
+ if (val.is_null_) {
+ return os << "Nullable{null, type=StringRef}";
+ }
+
+ return os << "Nullable{value=" << val.data_.DebugString() << ", type=StringRef}";
+ }
+
  StringRef data_;
  bool is_null_;
 };

hybridse/src/udf/udf.cc

@@ -22,6 +22,7 @@
 #include <utility>
 #include "absl/strings/ascii.h"
 #include "absl/strings/str_replace.h"
+#include "absl/time/civil_time.h"
 #include "base/iterator.h"
 #include "boost/date_time.hpp"
 #include "boost/date_time/gregorian/parsers.hpp"
@@ -57,11 +58,20 @@ bthread_key_t B_THREAD_LOCAL_MEM_POOL_KEY;
 
 void trivial_fun() {}
 
-int32_t dayofyear(int64_t ts) {
+void dayofyear(int64_t ts, int32_t* out, bool* is_null) {
+ if (ts < 0) {
+ *is_null = true;
+ *out = 0;
+ return;
+ }
+
  time_t time = (ts + TZ_OFFSET) / 1000;
  struct tm t;
+ memset(&t, 0, sizeof(struct tm));
  gmtime_r(&time, &t);
- return t.tm_yday + 1;
+
+ *out = t.tm_yday + 1;
+ *is_null = false;
 }
 int32_t dayofmonth(int64_t ts) {
  time_t time = (ts + TZ_OFFSET) / 1000;
@@ -99,24 +109,27 @@ int32_t year(int64_t ts) {
  return t.tm_year + 1900;
 }
 
-int32_t dayofyear(Timestamp *ts) { return dayofyear(ts->ts_); }
-int32_t dayofyear(Date *date) {
+void dayofyear(Timestamp *ts, int32_t *out, bool *is_null) { dayofyear(ts->ts_, out, is_null); }
+void dayofyear(Date *date, int32_t* out, bool* is_null) {
  int32_t day, month, year;
  if (!Date::Decode(date->date_, &year, &month, &day)) {
- return 0;
+ *out = 0;
+ *is_null = true;
+ return;
  }
- try {
- if (month <= 0 || month > 12) {
- return 0;
- } else if (day <= 0 || day > 31) {
- return 0;
- }
- boost::gregorian::date d(year, month, day);
- return d.day_of_year();
- } catch (...) {
- return 0;
+
+ absl::CivilDay civil_day(year, month, day);
+ if (civil_day.year() != year || civil_day.month() != month || civil_day.day() != day) {
+ // CivilTime normalize it because of invalid input
+ *out = 0;
+ *is_null = true;
+ return;
  }
+
+ *out = absl::GetYearDay(civil_day);
+ *is_null = false;
 }
+
 int32_t dayofmonth(Timestamp *ts) { return dayofmonth(ts->ts_); }
 int32_t weekofyear(Timestamp *ts) { return weekofyear(ts->ts_); }
 int32_t month(Timestamp *ts) { return month(ts->ts_); }

hybridse/src/udf/udf.h

@@ -209,9 +209,9 @@ int32_t month(Timestamp *ts);
 int32_t year(int64_t ts);
 int32_t year(Timestamp *ts);
 
-int32_t dayofyear(int64_t ts);
-int32_t dayofyear(Timestamp *ts);
-int32_t dayofyear(Date *ts);
+void dayofyear(int64_t ts, int32_t* out, bool* is_null);
+void dayofyear(Timestamp *ts, int32_t* out, bool* is_null);
+void dayofyear(Date *ts, int32_t* out, bool* is_null);
 
 int32_t dayofmonth(int64_t ts);
 int32_t dayofmonth(Timestamp *ts);

include/base/type.h

@@ -21,6 +21,7 @@
 #include <cstddef>
 #include <string>
 #include <vector>
+#include <ostream>
 
 namespace openmldb {
 namespace base {
@@ -37,6 +38,8 @@ struct Timestamp {
  return *this;
  }
  int64_t ts_;
+
+ friend std::ostream& operator<<(std::ostream& os, const Timestamp& ts) { return os << ts.ts_; }
 };
 
 __attribute__((unused)) static const Timestamp operator+(const Timestamp& a,
@@ -106,6 +109,10 @@ struct Date {
  return true;
  }
  int32_t date_;
+
+ friend std::ostream& operator<<(std::ostream& os, const Date& date) {
+ return os << date.date_;
+ }
 };
 
 __attribute__((unused)) static bool operator>(const Date& a, const Date& b) {