diff --git a/e2e_test/batch/types/timestamp_ns.slt.part b/e2e_test/batch/types/timestamp_ns.slt.part index 6eb727f678380..e0c969496c256 100644 --- a/e2e_test/batch/types/timestamp_ns.slt.part +++ b/e2e_test/batch/types/timestamp_ns.slt.part @@ -2,7 +2,7 @@ statement ok SET RW_IMPLICIT_FLUSH TO true; statement ok -create table t1(v1 int, v2 timestamp); +create table t1(v1 int, v2 timestamp_ns); statement ok insert into t1 values(1,'2013-01-01 01:01:01.123456789'),(2,'2012-01-01 01:01:01.123456'),(3,'0000-01-01 01:01:01.123456789'),(4,'2213-01-01 01:01:01.123456789'),(5,null),(6,'2013-01-01 01:01:01.123456789'); @@ -62,7 +62,7 @@ select * from t1 where v2 >= '2012-01-01 01:01:01.123456'; 6 2013-01-01 01:01:01.123456789 query T rowsort -select v1, cast(v2 as date) as date_v2, cast(v2 as timestamp with time zone) as timestamptz_v2 from t1; +select v1, cast(v2 as date) as date_v2, cast(v2 as timestamp_ns) as timestamptz_v2 from t1; ---- 1 2013-01-01 2013-01-01 01:01:01.123456+00:00 2 2012-01-01 2012-01-01 01:01:01.123456+00:00 @@ -102,7 +102,7 @@ select v1, to_char(v2, 'YYYY-MM-DD HH24:MI:SS.NS') as formatted_v2 from t1; 6 2013-01-01 01:01:01.123456789 query T rowsort -select generate_series('2013-01-01 01:01:01.123456789'::timestamp,'2013-01-01 01:01:05.123456790'::timestamp, '1 s'); +select generate_series('2013-01-01 01:01:01.123456789'::timestamp_ns,'2013-01-01 01:01:05.123456790'::timestamp_ns, '1 s'); ---- 2013-01-01 01:01:01.123456789 2013-01-01 01:01:02.123456789 @@ -194,7 +194,7 @@ select v1, extract(nanosecond from v2) from t1; 6 1123456789 query T rowsort -select make_timestamp(2013, 01, 01, 01, 01, 1.123456789); +select make_timestamp_ns(2013, 01, 01, 01, 01, 1.123456789); ---- 2013-01-01 01:01:01.123456789 diff --git a/proto/data.proto b/proto/data.proto index 9bb15ebcc8d62..478fc3e522c8f 100644 --- a/proto/data.proto +++ b/proto/data.proto @@ -53,6 +53,7 @@ message DataType { SERIAL = 19; INT256 = 20; MAP = 21; + TIMESTAMP_NANOSECOND = 22; } TypeName type_name = 1; // Data length for char. @@ -104,6 +105,7 @@ enum ArrayType { SERIAL = 17; INT256 = 18; MAP = 20; + TIMESTAMP_NANOSECOND = 21; } message Array { diff --git a/proto/expr.proto b/proto/expr.proto index 5bd2809f7d314..6edc2db1f747e 100644 --- a/proto/expr.proto +++ b/proto/expr.proto @@ -56,6 +56,7 @@ message ExprNode { MAKE_DATE = 113; MAKE_TIME = 114; MAKE_TIMESTAMP = 115; + MAKE_TIMESTAMP_NS = 116; // From f64 to timestamp. // e.g. `select to_timestamp(1672044740.0)` SEC_TO_TIMESTAMPTZ = 104; diff --git a/src/batch/executors/src/executor/postgres_query.rs b/src/batch/executors/src/executor/postgres_query.rs index e7ce698e1e841..9a265a195f054 100644 --- a/src/batch/executors/src/executor/postgres_query.rs +++ b/src/batch/executors/src/executor/postgres_query.rs @@ -85,6 +85,7 @@ fn postgres_cell_to_scalar_impl( | DataType::Date | DataType::Time | DataType::Timestamp + | DataType::TimestampNanosecond | DataType::Timestamptz | DataType::Jsonb | DataType::Interval diff --git a/src/common/src/array/arrow/arrow_iceberg.rs b/src/common/src/array/arrow/arrow_iceberg.rs index 6882ec6f63224..10f0041e24eaa 100644 --- a/src/common/src/array/arrow/arrow_iceberg.rs +++ b/src/common/src/array/arrow/arrow_iceberg.rs @@ -97,6 +97,7 @@ impl ToArrow for IcebergArrowConvert { DataType::Date => self.date_type_to_arrow(), DataType::Time => self.time_type_to_arrow(), DataType::Timestamp => self.timestamp_type_to_arrow(), + DataType::TimestampNanosecond => self.timestampns_type_to_arrow(), DataType::Timestamptz => self.timestamptz_type_to_arrow(), DataType::Interval => self.interval_type_to_arrow(), DataType::Varchar => self.varchar_type_to_arrow(), @@ -246,6 +247,7 @@ impl ToArrow for IcebergCreateTableArrowConvert { DataType::Date => self.date_type_to_arrow(), DataType::Time => self.time_type_to_arrow(), DataType::Timestamp => self.timestamp_type_to_arrow(), + DataType::TimestampNanosecond => self.timestampns_type_to_arrow(), DataType::Timestamptz => self.timestamptz_type_to_arrow(), DataType::Interval => self.interval_type_to_arrow(), DataType::Varchar => self.varchar_type_to_arrow(), diff --git a/src/common/src/array/arrow/arrow_impl.rs b/src/common/src/array/arrow/arrow_impl.rs index edf83f0a6f54c..01d8bab6489e2 100644 --- a/src/common/src/array/arrow/arrow_impl.rs +++ b/src/common/src/array/arrow/arrow_impl.rs @@ -107,6 +107,7 @@ pub trait ToArrow { ArrayImpl::Date(array) => self.date_to_arrow(array), ArrayImpl::Time(array) => self.time_to_arrow(array), ArrayImpl::Timestamp(array) => self.timestamp_to_arrow(array), + ArrayImpl::TimestampNanosecond(array) => self.timestampns_to_arrow(array), ArrayImpl::Timestamptz(array) => self.timestamptz_to_arrow(array), ArrayImpl::Interval(array) => self.interval_to_arrow(array), ArrayImpl::Utf8(array) => self.utf8_to_arrow(array), @@ -180,6 +181,14 @@ pub trait ToArrow { ))) } + #[inline] + fn timestampns_to_arrow( + &self, + array: &TimestampNanosecondArray, + ) -> Result { + Ok(Arc::new(arrow_array::TimestampNanosecondArray::from(array))) + } + #[inline] fn timestamptz_to_arrow( &self, @@ -319,6 +328,7 @@ pub trait ToArrow { DataType::Date => self.date_type_to_arrow(), DataType::Time => self.time_type_to_arrow(), DataType::Timestamp => self.timestamp_type_to_arrow(), + DataType::TimestampNanosecond => self.timestampns_type_to_arrow(), DataType::Timestamptz => self.timestamptz_type_to_arrow(), DataType::Interval => self.interval_type_to_arrow(), DataType::Varchar => self.varchar_type_to_arrow(), @@ -382,6 +392,10 @@ pub trait ToArrow { fn timestamp_type_to_arrow(&self) -> arrow_schema::DataType { arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Microsecond, None) } + #[inline] + fn timestampns_type_to_arrow(&self) -> arrow_schema::DataType { + arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Nanosecond, None) + } #[inline] fn timestamptz_type_to_arrow(&self) -> arrow_schema::DataType { @@ -1046,6 +1060,11 @@ converts_with_timeunit!(TimestamptzArray, arrow_array::TimestampMillisecondArray converts_with_timeunit!(TimestamptzArray, arrow_array::TimestampMicrosecondArray, TimeUnit::Microsecond, @map); converts_with_timeunit!(TimestamptzArray, arrow_array::TimestampNanosecondArray, TimeUnit::Nanosecond, @map); +converts_with_timeunit!(TimestampNanosecondArray, arrow_array::TimestampSecondArray, TimeUnit::Second, @map); +converts_with_timeunit!(TimestampNanosecondArray, arrow_array::TimestampMillisecondArray, TimeUnit::Millisecond, @map); +converts_with_timeunit!(TimestampNanosecondArray, arrow_array::TimestampMicrosecondArray, TimeUnit::Microsecond, @map); +converts_with_timeunit!(TimestampNanosecondArray, arrow_array::TimestampNanosecondArray, TimeUnit::Nanosecond, @map); + /// Converts RisingWave value from and into Arrow value. trait FromIntoArrow { /// The corresponding element type in the Arrow array. @@ -1162,6 +1181,37 @@ impl FromIntoArrowWithUnit for Timestamp { } } +impl FromIntoArrowWithUnit for TimestampNanosecond { + type ArrowType = i64; + type TimestampType = TimeUnit; + + fn from_arrow_with_unit(value: Self::ArrowType, time_unit: Self::TimestampType) -> Self { + match time_unit { + TimeUnit::Second => { + TimestampNanosecond(DateTime::from_timestamp(value as _, 0).unwrap().naive_utc()) + } + TimeUnit::Millisecond => { + TimestampNanosecond(DateTime::from_timestamp_millis(value).unwrap().naive_utc()) + } + TimeUnit::Microsecond => { + TimestampNanosecond(DateTime::from_timestamp_micros(value).unwrap().naive_utc()) + } + TimeUnit::Nanosecond => { + TimestampNanosecond(DateTime::from_timestamp_nanos(value).naive_utc()) + } + } + } + + fn into_arrow_with_unit(self, time_unit: Self::TimestampType) -> Self::ArrowType { + match time_unit { + TimeUnit::Second => self.0.and_utc().timestamp(), + TimeUnit::Millisecond => self.0.and_utc().timestamp_millis(), + TimeUnit::Microsecond => self.0.and_utc().timestamp_micros(), + TimeUnit::Nanosecond => self.0.and_utc().timestamp_nanos_opt().unwrap(), + } + } +} + impl FromIntoArrowWithUnit for Timestamptz { type ArrowType = i64; type TimestampType = TimeUnit; diff --git a/src/common/src/array/chrono_array.rs b/src/common/src/array/chrono_array.rs index 0e6c9f523d9f7..91fb84d4feb0b 100644 --- a/src/common/src/array/chrono_array.rs +++ b/src/common/src/array/chrono_array.rs @@ -12,17 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{PrimitiveArray, PrimitiveArrayBuilder}; +use super::{PrimitiveArray, PrimitiveArrayBuilder, TimestampNanosecond}; use crate::types::{Date, Time, Timestamp, Timestamptz}; pub type DateArray = PrimitiveArray; pub type TimeArray = PrimitiveArray