Extend time type

Author: rui-mo

velox/common/fuzzer/Utils.cpp

@@ -120,7 +120,7 @@ int32_t randDate(FuzzerGenerator& rng) {
 }
 
 int32_t randTime(FuzzerGenerator& rng) {
-  return rand<int64_t>(rng, TimeType::kMin, TimeType::kMax);
+  return rand<int64_t>(rng, TIME()->getMin(), TIME()->getMax());
 }
 
 /// Unicode character ranges. Ensure the vector indexes match the UTF8CharList

velox/docs/develop/types.rst

@@ -131,8 +131,9 @@ upto 38 precision, with a range of :math:`[-10^{38} + 1, +10^{38} - 1]`.
 All the three values, precision, scale, unscaled value are required to represent a
 decimal value.
 
-TIME type represents time in milliseconds from midnight UTC. Thus min/max value can  range from UTC-14:00 at 00:00:00 to UTC+14:00 at 23:59:59.999 modulo 24 hours.
-TIME type is backed by BIGINT physical type.
+TIME type represents time in milliseconds from midnight UTC. Thus min/max value can range from UTC-14:00 at 00:00:00 to UTC+14:00 at 23:59:59.999 modulo 24 hours.
+TIME_MICRO type represents time in microseconds from midnight. Thus min/max value can range from 00:00:00.000000 to 23:59:59.999999 modulo 24 hours.
+TIME and TIME_MICRO types are backed by BIGINT physical type.
 
 Custom Types
 ~~~~~~~~~~~~
@@ -292,7 +293,7 @@ Presto. These differences require us to implement the same functions
 separately for each system in Velox, such as min, max and collect_set. The
 key differences are listed below.
 
-* Spark operates on timestamps with "microsecond" precision while Presto with
+* Spark operates on TIMESTAMP and TIME types with "microsecond" precision while Presto with
   "millisecond" precision.
   Example::
 
@@ -304,6 +305,8 @@ key differences are listed below.
       ) AS t(ts);
       -- 2014-03-08 09:00:00.012345
 
+      SELECT cast(time'12:30:45.123456' as bigint);  -- 45,045,123,456
+
 * In function comparisons, nested null values are handled as values.
   Example::
 

velox/expression/CastExpr.cpp

@@ -237,7 +237,7 @@ VectorPtr CastExpr::castFromTime(
       auto* resultFlatVector = castResult->as<FlatVector<StringView>>();
 
       Buffer* buffer = resultFlatVector->getBufferWithSpace(
-          rows.countSelected() * TimeType::kTimeToVarcharRowSize,
+          rows.countSelected() * TIME()->getTimeToVarcharRowSize(),
           true /*exactSize*/);
       char* rawBuffer = buffer->asMutable<char>() + buffer->size();
 

velox/expression/UdfTypeResolver.h

@@ -157,6 +157,13 @@ struct resolver<Time> {
   using out_type = int64_t;
 };
 
+template <>
+struct resolver<TimeMicro> {
+  using in_type = int64_t;
+  using null_free_in_type = in_type;
+  using out_type = int64_t;
+};
+
 template <typename T>
 struct resolver<std::shared_ptr<T>> {
   using in_type = std::shared_ptr<T>;

velox/expression/tests/SimpleFunctionTest.cpp

@@ -1657,12 +1657,16 @@ TEST_F(SimpleFunctionTest, toDebugString) {
       "Priority: 999997\nDefaultNullBehavior: true");
 }
 
-template <typename T>
+template <typename T, typename TimeType>
 struct TimePlusOneFunction {
   VELOX_DEFINE_FUNCTION_TYPES(T);
 
-  void call(out_type<Time>& out, const arg_type<Time>& input) {
-    out = input + 1;
+  void call(out_type<TimeType>& out, const arg_type<TimeType>& input) {
+    if constexpr (std::is_same_v<TimeType, Time>) {
+      out = input + 1;
+    } else if constexpr (std::is_same_v<TimeType, TimeMicro>) {
+      out = input / 1000 + 1;
+    }
   }
 };
 
@@ -1677,16 +1681,23 @@ struct ArrayTimeFunction {
       }
     }
   }
+
+  void call(
+      out_type<Array<TimeMicro>>& out,
+      const arg_type<Array<TimeMicro>>& input) {}
 };
 
 TEST_F(SimpleFunctionTest, timeTypeTest) {
-  registerFunction<TimePlusOneFunction, Time, Time>({"time_plus_one"});
-  auto data = makeRowVector({
-      makeFlatVector<int64_t>({1, 2, 3}, TIME()),
-  });
-  auto result = evaluate("time_plus_one(c0)", data);
-  auto expected = makeFlatVector<int64_t>({2, 3, 4}, TIME());
-  assertEqualVectors(expected, result);
+  {
+    registerFunction<ParameterBinder<TimePlusOneFunction, Time>, Time, Time>(
+        {"time_plus_one"});
+    auto data = makeRowVector({
+        makeFlatVector<int64_t>({1000, 2000, 3000}, TIME()),
+    });
+    auto result = evaluate("time_plus_one(c0)", data);
+    auto expected = makeFlatVector<int64_t>({1001, 2001, 3001}, TIME());
+    assertEqualVectors(expected, result);
+  }
 
   // Test out Time in complex type.
   {
@@ -1703,4 +1714,33 @@ TEST_F(SimpleFunctionTest, timeTypeTest) {
   }
 }
 
+TEST_F(SimpleFunctionTest, timeMicroTypeTest) {
+  {
+    registerFunction<
+        ParameterBinder<TimePlusOneFunction, TimeMicro>,
+        TimeMicro,
+        TimeMicro>({"time_micro_plus_one"});
+    auto data = makeRowVector({
+        makeFlatVector<int64_t>({1000, 2000, 3000}, TIME_MICRO()),
+    });
+    auto result = evaluate("time_micro_plus_one(c0)", data);
+    auto expected = makeFlatVector<int64_t>({2, 3, 4}, TIME_MICRO());
+    assertEqualVectors(expected, result);
+  }
+
+  // Test out TimeMicro in complex type.
+  {
+    registerFunction<ArrayTimeFunction, Array<TimeMicro>, Array<TimeMicro>>(
+        {"array_time_micro"});
+
+    auto data = makeRowVector({
+        makeArrayVector<int64_t>({{1, 2, 3}, {4, 5, 6}}, TIME_MICRO()),
+    });
+
+    auto result = evaluate("array_time_micro(c0)", data);
+    auto expected = makeArrayVector<int64_t>({{}, {}}, TIME_MICRO());
+    assertEqualVectors(expected, result);
+  }
+}
+
 } // namespace

velox/functions/prestosql/tests/TypeOfTest.cpp

@@ -63,6 +63,7 @@ TEST_F(TypeOfTest, basic) {
   EXPECT_EQ("timestamp", typeOf(TIMESTAMP()));
   EXPECT_EQ("date", typeOf(DATE()));
   EXPECT_EQ("time", typeOf(TIME()));
+  EXPECT_EQ("time micro", typeOf(TIME_MICRO()));
 
   EXPECT_EQ("unknown", typeOf(UNKNOWN()));
 

velox/functions/prestosql/types/parser/tests/TypeParserTest.cpp

@@ -112,6 +112,7 @@ TEST_F(TypeParserTest, varbinary) {
 
 TEST_F(TypeParserTest, time) {
   ASSERT_EQ(*parseType("time"), *TIME());
+  ASSERT_EQ(*parseType("time micro"), *TIME_MICRO());
 }
 
 TEST_F(TypeParserTest, timeWithTimeZoneType) {

velox/type/CppToType.h

@@ -92,7 +92,18 @@ template <>
 struct CppToType<Timestamp> : public CppToTypeBase<TypeKind::TIMESTAMP> {};
 
 template <>
-struct CppToType<Time> : public CppToTypeBase<TypeKind::BIGINT> {};
+struct CppToType<Time> : public CppToTypeBase<TypeKind::BIGINT> {
+  static auto create() {
+    return TIME();
+  }
+};
+
+template <>
+struct CppToType<TimeMicro> : public CppToTypeBase<TypeKind::BIGINT> {
+  static auto create() {
+    return TIME_MICRO();
+  }
+};
 
 // TODO: maybe do something smarter than just matching any shared_ptr, e.g. we
 // can declare "registered" types explicitly

velox/type/SimpleFunctionApi.h

@@ -374,6 +374,11 @@ struct SimpleTypeTrait<Time> : public TypeTraits<TypeKind::BIGINT> {
   static constexpr const char* name = "TIME";
 };
 
+template <>
+struct SimpleTypeTrait<TimeMicro> : public TypeTraits<TypeKind::BIGINT> {
+  static constexpr const char* name = "TIME MICRO";
+};
+
 template <typename T, bool comparable, bool orderable>
 struct SimpleTypeTrait<Generic<T, comparable, orderable>> {
   static constexpr TypeKind typeKind = TypeKind::INVALID;

velox/type/Type.cpp

@@ -265,7 +265,10 @@ void Type::registerSerDe() {
   registry.Register(
       "IntervalYearMonthType", IntervalYearMonthType::deserialize);
   registry.Register("DateType", DateType::deserialize);
-  registry.Register("TimeType", TimeType::deserialize);
+
+  // The deserilization handles all supported precisions. No extra registration
+  // needed for 'TimeMicroPrecisionType'.
+  registry.Register("TimeType", TimeMilliPrecisionType::deserialize);
 }
 
 std::string ArrayType::toString() const {
@@ -1358,6 +1361,7 @@ const SingletonTypeMap& singletonBuiltInTypes() {
       {"INTERVAL YEAR TO MONTH", INTERVAL_YEAR_MONTH()},
       {"DATE", DATE()},
       {"TIME", TIME()},
+      {"TIME MICRO", TIME_MICRO()},
       {"UNKNOWN", UNKNOWN()},
   };
   return kTypes;
@@ -1508,14 +1512,20 @@ std::string getOpaqueAliasForTypeId(std::type_index typeIndex) {
   return it->second;
 }
 
-folly::dynamic TimeType::serialize() const {
-  folly::dynamic obj = folly::dynamic::object;
-  obj["name"] = "TimeType";
-  obj["type"] = name();
-  return obj;
+template <TimePrecision PRECISION>
+TypePtr TimeType<PRECISION>::deserialize(const folly::dynamic& obj) {
+  if (obj.count("precision")) {
+    auto precision = obj["precision"].asInt();
+    if (precision == static_cast<int>(TimePrecision::kMicroseconds)) {
+      return TIME_MICRO();
+    }
+  }
+  return TIME();
 }
 
-StringView TimeType::valueToString(int64_t value, char* const startPos) const {
+StringView TimeMilliPrecisionType::valueToString(
+    int64_t value,
+    char* const startPos) const {
   // Ensure the value is within valid TIME range
   VELOX_USER_CHECK(
       !(value < 0 || value >= 86400000),
@@ -1531,25 +1541,24 @@ StringView TimeType::valueToString(int64_t value, char* const startPos) const {
 
   // TIME is represented as milliseconds since midnight
   // Convert to HH:mm:ss.SSS format
-
   fmt::format_to_n(
       startPos,
-      kTimeToVarcharRowSize,
+      getTimeToVarcharRowSize(),
       "{:02d}:{:02d}:{:02d}.{:03d}",
       hours,
       minutes,
       seconds,
       millis);
-  return StringView{startPos, kTimeToVarcharRowSize};
+  return StringView{startPos, getTimeToVarcharRowSize()};
 }
 
-int64_t TimeType::valueToTime(const StringView& timeStr) const {
+int64_t TimeMilliPrecisionType::valueToTime(const StringView& timeStr) const {
   return util::fromTimeString(timeStr).thenOrThrow(
       folly::identity,
       [&](const Status& status) { VELOX_USER_FAIL("{}", status.message()); });
 }
 
-int64_t TimeType::valueToTime(
+int64_t TimeMilliPrecisionType::valueToTime(
     const StringView& timeStr,
     const tz::TimeZone* timeZone,
     int64_t sessionStartTimeMs) const {
@@ -1603,6 +1612,23 @@ int64_t TimeType::valueToTime(
   return adjustedTime;
 }
 
+StringView TimeMicroPrecisionType::valueToString(
+    int64_t value,
+    char* const startPos) const {
+  VELOX_NYI("The valueToString not implemented yet");
+}
+
+int64_t TimeMicroPrecisionType::valueToTime(const StringView& timeStr) const {
+  VELOX_NYI("The valueToTime not implemented yet");
+}
+
+int64_t TimeMicroPrecisionType::valueToTime(
+    const StringView& timeStr,
+    const tz::TimeZone* timeZone,
+    int64_t sessionStartTimeMs) const {
+  VELOX_NYI("The valueToTime not implemented yet");
+}
+
 std::string stringifyTruncatedElementList(
     size_t size,
     const std::function<void(std::stringstream&, size_t)>& stringifyElement,