From 16dac749b39963d4109676408c6716cb94929fb1 Mon Sep 17 00:00:00 2001 From: gokselk Date: Tue, 6 Jan 2026 02:20:21 +0300 Subject: [PATCH 01/10] Add support for additional numeric types in to_timestamp functions --- .../functions/src/datetime/to_timestamp.rs | 69 +- .../test_files/datetime/timestamps.slt | 610 ++++++++++++++++++ 2 files changed, 670 insertions(+), 9 deletions(-) diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index 58077694b07a0..318d3b52886cb 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -380,13 +380,14 @@ impl ScalarUDFImpl for ToTimestampFunc { let tz = self.timezone.clone(); match args[0].data_type() { - Int32 | Int64 => args[0] + Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 => args[0] .cast_to(&Timestamp(Second, None), None)? .cast_to(&Timestamp(Nanosecond, tz), None), Null | Timestamp(_, _) => args[0].cast_to(&Timestamp(Nanosecond, tz), None), - Float64 => { + Float32 | Float64 => { + let arg = args[0].cast_to(&Float64, None)?; let rescaled = arrow::compute::kernels::numeric::mul( - &args[0].to_array(1)?, + &arg.to_array(1)?, &arrow::array::Scalar::new(Float64Array::from(vec![ 1_000_000_000f64, ])), @@ -473,9 +474,20 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc { let tz = self.timezone.clone(); match args[0].data_type() { - Null | Int32 | Int64 | Timestamp(_, _) | Decimal128(_, _) => { - args[0].cast_to(&Timestamp(Second, tz), None) - } + Null + | Int8 + | Int16 + | Int32 + | Int64 + | UInt8 + | UInt16 + | UInt32 + | UInt64 + | Timestamp(_, _) + | Decimal128(_, _) => args[0].cast_to(&Timestamp(Second, tz), None), + Float32 | Float64 => args[0] + .cast_to(&Int64, None)? + .cast_to(&Timestamp(Second, tz), None), Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( &args, "to_timestamp_seconds", @@ -533,9 +545,22 @@ impl ScalarUDFImpl for ToTimestampMillisFunc { } match args[0].data_type() { - Null | Int32 | Int64 | Timestamp(_, _) => { + Null + | Int8 + | Int16 + | Int32 + | Int64 + | UInt8 + | UInt16 + | UInt32 + | UInt64 + | Timestamp(_, _) + | Decimal128(_, _) => { args[0].cast_to(&Timestamp(Millisecond, self.timezone.clone()), None) } + Float32 | Float64 => args[0] + .cast_to(&Int64, None)? + .cast_to(&Timestamp(Millisecond, self.timezone.clone()), None), Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( &args, "to_timestamp_millis", @@ -593,9 +618,22 @@ impl ScalarUDFImpl for ToTimestampMicrosFunc { } match args[0].data_type() { - Null | Int32 | Int64 | Timestamp(_, _) => { + Null + | Int8 + | Int16 + | Int32 + | Int64 + | UInt8 + | UInt16 + | UInt32 + | UInt64 + | Timestamp(_, _) + | Decimal128(_, _) => { args[0].cast_to(&Timestamp(Microsecond, self.timezone.clone()), None) } + Float32 | Float64 => args[0] + .cast_to(&Int64, None)? + .cast_to(&Timestamp(Microsecond, self.timezone.clone()), None), Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( &args, "to_timestamp_micros", @@ -653,9 +691,22 @@ impl ScalarUDFImpl for ToTimestampNanosFunc { } match args[0].data_type() { - Null | Int32 | Int64 | Timestamp(_, _) => { + Null + | Int8 + | Int16 + | Int32 + | Int64 + | UInt8 + | UInt16 + | UInt32 + | UInt64 + | Timestamp(_, _) + | Decimal128(_, _) => { args[0].cast_to(&Timestamp(Nanosecond, self.timezone.clone()), None) } + Float32 | Float64 => args[0] + .cast_to(&Int64, None)? + .cast_to(&Timestamp(Nanosecond, self.timezone.clone()), None), Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( &args, "to_timestamp_nanos", diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index dbb924ef7aa63..83b57bc6fabdd 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -4422,6 +4422,616 @@ FROM (VALUES 1970-01-01T00:00:00.000000005Z +########## +## to_timestamp functions with all numeric types +########## + +# Test to_timestamp with all integer types +# Int8 +query P +SELECT to_timestamp(arrow_cast(0, 'Int8')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(100, 'Int8')); +---- +1970-01-01T00:01:40 + +# Int16 +query P +SELECT to_timestamp(arrow_cast(0, 'Int16')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(1000, 'Int16')); +---- +1970-01-01T00:16:40 + +# Int32 +query P +SELECT to_timestamp(arrow_cast(0, 'Int32')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(86400, 'Int32')); +---- +1970-01-02T00:00:00 + +# Int64 +query P +SELECT to_timestamp(arrow_cast(0, 'Int64')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(86400, 'Int64')); +---- +1970-01-02T00:00:00 + +# UInt8 +query P +SELECT to_timestamp(arrow_cast(0, 'UInt8')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(100, 'UInt8')); +---- +1970-01-01T00:01:40 + +# UInt16 +query P +SELECT to_timestamp(arrow_cast(0, 'UInt16')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(1000, 'UInt16')); +---- +1970-01-01T00:16:40 + +# UInt32 +query P +SELECT to_timestamp(arrow_cast(0, 'UInt32')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(86400, 'UInt32')); +---- +1970-01-02T00:00:00 + +# UInt64 +query P +SELECT to_timestamp(arrow_cast(0, 'UInt64')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(86400, 'UInt64')); +---- +1970-01-02T00:00:00 + +# Float32 +query P +SELECT to_timestamp(arrow_cast(0.0, 'Float32')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(1.5, 'Float32')); +---- +1970-01-01T00:00:01.500 + +# Float64 +query P +SELECT to_timestamp(arrow_cast(0.0, 'Float64')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(1.5, 'Float64')); +---- +1970-01-01T00:00:01.500 + +# Test to_timestamp_seconds with all integer types +# Int8 +query P +SELECT to_timestamp_seconds(arrow_cast(0, 'Int8')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp_seconds(arrow_cast(100, 'Int8')); +---- +1970-01-01T00:01:40 + +# Int16 +query P +SELECT to_timestamp_seconds(arrow_cast(1000, 'Int16')); +---- +1970-01-01T00:16:40 + +# UInt8 +query P +SELECT to_timestamp_seconds(arrow_cast(100, 'UInt8')); +---- +1970-01-01T00:01:40 + +# UInt16 +query P +SELECT to_timestamp_seconds(arrow_cast(1000, 'UInt16')); +---- +1970-01-01T00:16:40 + +# UInt32 +query P +SELECT to_timestamp_seconds(arrow_cast(86400, 'UInt32')); +---- +1970-01-02T00:00:00 + +# UInt64 +query P +SELECT to_timestamp_seconds(arrow_cast(86400, 'UInt64')); +---- +1970-01-02T00:00:00 + +# Float32 +query P +SELECT to_timestamp_seconds(arrow_cast(1.9, 'Float32')); +---- +1970-01-01T00:00:01 + +# Float64 +query P +SELECT to_timestamp_seconds(arrow_cast(1.9, 'Float64')); +---- +1970-01-01T00:00:01 + +# Test to_timestamp_millis with all integer types +# Int8 +query P +SELECT to_timestamp_millis(arrow_cast(0, 'Int8')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp_millis(arrow_cast(100, 'Int8')); +---- +1970-01-01T00:00:00.100 + +# Int16 +query P +SELECT to_timestamp_millis(arrow_cast(1000, 'Int16')); +---- +1970-01-01T00:00:01 + +# UInt8 +query P +SELECT to_timestamp_millis(arrow_cast(100, 'UInt8')); +---- +1970-01-01T00:00:00.100 + +# UInt16 +query P +SELECT to_timestamp_millis(arrow_cast(1000, 'UInt16')); +---- +1970-01-01T00:00:01 + +# UInt32 +query P +SELECT to_timestamp_millis(arrow_cast(86400000, 'UInt32')); +---- +1970-01-02T00:00:00 + +# UInt64 +query P +SELECT to_timestamp_millis(arrow_cast(86400000, 'UInt64')); +---- +1970-01-02T00:00:00 + +# Float32 +query P +SELECT to_timestamp_millis(arrow_cast(1000.9, 'Float32')); +---- +1970-01-01T00:00:01 + +# Float64 +query P +SELECT to_timestamp_millis(arrow_cast(1000.9, 'Float64')); +---- +1970-01-01T00:00:01 + +# Test to_timestamp_micros with all integer types +# Int8 +query P +SELECT to_timestamp_micros(arrow_cast(0, 'Int8')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp_micros(arrow_cast(100, 'Int8')); +---- +1970-01-01T00:00:00.000100 + +# Int16 +query P +SELECT to_timestamp_micros(arrow_cast(1000, 'Int16')); +---- +1970-01-01T00:00:00.001 + +# UInt8 +query P +SELECT to_timestamp_micros(arrow_cast(100, 'UInt8')); +---- +1970-01-01T00:00:00.000100 + +# UInt16 +query P +SELECT to_timestamp_micros(arrow_cast(1000, 'UInt16')); +---- +1970-01-01T00:00:00.001 + +# UInt32 +query P +SELECT to_timestamp_micros(arrow_cast(1000000, 'UInt32')); +---- +1970-01-01T00:00:01 + +# UInt64 +query P +SELECT to_timestamp_micros(arrow_cast(1000000, 'UInt64')); +---- +1970-01-01T00:00:01 + +# Float32 +query P +SELECT to_timestamp_micros(arrow_cast(1000000.9, 'Float32')); +---- +1970-01-01T00:00:01 + +# Float64 +query P +SELECT to_timestamp_micros(arrow_cast(1000000.9, 'Float64')); +---- +1970-01-01T00:00:01 + +# Test to_timestamp_nanos with all integer types +# Int8 +query P +SELECT to_timestamp_nanos(arrow_cast(0, 'Int8')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp_nanos(arrow_cast(100, 'Int8')); +---- +1970-01-01T00:00:00.000000100 + +# Int16 +query P +SELECT to_timestamp_nanos(arrow_cast(1000, 'Int16')); +---- +1970-01-01T00:00:00.000001 + +# UInt8 +query P +SELECT to_timestamp_nanos(arrow_cast(100, 'UInt8')); +---- +1970-01-01T00:00:00.000000100 + +# UInt16 +query P +SELECT to_timestamp_nanos(arrow_cast(1000, 'UInt16')); +---- +1970-01-01T00:00:00.000001 + +# UInt32 +query P +SELECT to_timestamp_nanos(arrow_cast(1000000000, 'UInt32')); +---- +1970-01-01T00:00:01 + +# UInt64 +query P +SELECT to_timestamp_nanos(arrow_cast(1000000000, 'UInt64')); +---- +1970-01-01T00:00:01 + +# Float32 +query P +SELECT to_timestamp_nanos(arrow_cast(1000000000.9, 'Float32')); +---- +1970-01-01T00:00:01 + +# Float64 +query P +SELECT to_timestamp_nanos(arrow_cast(1000000000.9, 'Float64')); +---- +1970-01-01T00:00:01 + +# Verify arrow_typeof for all to_timestamp functions with various input types +query T +SELECT arrow_typeof(to_timestamp(arrow_cast(0, 'Int8'))); +---- +Timestamp(ns) + +query T +SELECT arrow_typeof(to_timestamp(arrow_cast(0, 'UInt64'))); +---- +Timestamp(ns) + +query T +SELECT arrow_typeof(to_timestamp(arrow_cast(0.0, 'Float32'))); +---- +Timestamp(ns) + +query T +SELECT arrow_typeof(to_timestamp_seconds(arrow_cast(0, 'Int8'))); +---- +Timestamp(s) + +query T +SELECT arrow_typeof(to_timestamp_seconds(arrow_cast(0, 'UInt64'))); +---- +Timestamp(s) + +query T +SELECT arrow_typeof(to_timestamp_seconds(arrow_cast(0.0, 'Float32'))); +---- +Timestamp(s) + +query T +SELECT arrow_typeof(to_timestamp_millis(arrow_cast(0, 'Int8'))); +---- +Timestamp(ms) + +query T +SELECT arrow_typeof(to_timestamp_millis(arrow_cast(0, 'UInt64'))); +---- +Timestamp(ms) + +query T +SELECT arrow_typeof(to_timestamp_millis(arrow_cast(0.0, 'Float32'))); +---- +Timestamp(ms) + +query T +SELECT arrow_typeof(to_timestamp_micros(arrow_cast(0, 'Int8'))); +---- +Timestamp(µs) + +query T +SELECT arrow_typeof(to_timestamp_micros(arrow_cast(0, 'UInt64'))); +---- +Timestamp(µs) + +query T +SELECT arrow_typeof(to_timestamp_micros(arrow_cast(0.0, 'Float32'))); +---- +Timestamp(µs) + +query T +SELECT arrow_typeof(to_timestamp_nanos(arrow_cast(0, 'Int8'))); +---- +Timestamp(ns) + +query T +SELECT arrow_typeof(to_timestamp_nanos(arrow_cast(0, 'UInt64'))); +---- +Timestamp(ns) + +query T +SELECT arrow_typeof(to_timestamp_nanos(arrow_cast(0.0, 'Float32'))); +---- +Timestamp(ns) + +# Test Decimal128 support for all to_timestamp functions +# to_timestamp with Decimal128 +query P +SELECT to_timestamp(arrow_cast(0.0, 'Decimal128(10,1)')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(1.5, 'Decimal128(10,1)')); +---- +1970-01-01T00:00:01.500 + +query P +SELECT to_timestamp(arrow_cast(86400.123, 'Decimal128(10,3)')); +---- +1970-01-02T00:00:00.123 + +# to_timestamp_seconds with Decimal128 +query P +SELECT to_timestamp_seconds(arrow_cast(0, 'Decimal128(10,0)')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp_seconds(arrow_cast(86400, 'Decimal128(10,0)')); +---- +1970-01-02T00:00:00 + +# to_timestamp_millis with Decimal128 +query P +SELECT to_timestamp_millis(arrow_cast(0, 'Decimal128(10,0)')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp_millis(arrow_cast(1000, 'Decimal128(10,0)')); +---- +1970-01-01T00:00:01 + +query P +SELECT to_timestamp_millis(arrow_cast(86400000, 'Decimal128(15,0)')); +---- +1970-01-02T00:00:00 + +# to_timestamp_micros with Decimal128 +query P +SELECT to_timestamp_micros(arrow_cast(0, 'Decimal128(10,0)')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp_micros(arrow_cast(1000000, 'Decimal128(10,0)')); +---- +1970-01-01T00:00:01 + +query P +SELECT to_timestamp_micros(arrow_cast(86400000000, 'Decimal128(15,0)')); +---- +1970-01-02T00:00:00 + +# to_timestamp_nanos with Decimal128 +query P +SELECT to_timestamp_nanos(arrow_cast(0, 'Decimal128(10,0)')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp_nanos(arrow_cast(1000000000, 'Decimal128(15,0)')); +---- +1970-01-01T00:00:01 + +query P +SELECT to_timestamp_nanos(arrow_cast(86400000000000, 'Decimal128(20,0)')); +---- +1970-01-02T00:00:00 + +# Verify arrow_typeof for Decimal128 inputs +query T +SELECT arrow_typeof(to_timestamp(arrow_cast(0, 'Decimal128(10,0)'))); +---- +Timestamp(ns) + +query T +SELECT arrow_typeof(to_timestamp_seconds(arrow_cast(0, 'Decimal128(10,0)'))); +---- +Timestamp(s) + +query T +SELECT arrow_typeof(to_timestamp_millis(arrow_cast(0, 'Decimal128(10,0)'))); +---- +Timestamp(ms) + +query T +SELECT arrow_typeof(to_timestamp_micros(arrow_cast(0, 'Decimal128(10,0)'))); +---- +Timestamp(µs) + +query T +SELECT arrow_typeof(to_timestamp_nanos(arrow_cast(0, 'Decimal128(10,0)'))); +---- +Timestamp(ns) + +# Test negative values +# to_timestamp with negative seconds +query P +SELECT to_timestamp(arrow_cast(-86400, 'Int32')); +---- +1969-12-31T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(-1, 'Int64')); +---- +1969-12-31T23:59:59 + +query P +SELECT to_timestamp(arrow_cast(-0.5, 'Float64')); +---- +1969-12-31T23:59:59.500 + +# to_timestamp_seconds with negative values +query P +SELECT to_timestamp_seconds(arrow_cast(-86400, 'Int32')); +---- +1969-12-31T00:00:00 + +query P +SELECT to_timestamp_seconds(arrow_cast(-1, 'Int64')); +---- +1969-12-31T23:59:59 + +# to_timestamp_millis with negative values +query P +SELECT to_timestamp_millis(arrow_cast(-1000, 'Int32')); +---- +1969-12-31T23:59:59 + +query P +SELECT to_timestamp_millis(arrow_cast(-1, 'Int64')); +---- +1969-12-31T23:59:59.999 + +# to_timestamp_micros with negative values +query P +SELECT to_timestamp_micros(arrow_cast(-1000000, 'Int32')); +---- +1969-12-31T23:59:59 + +query P +SELECT to_timestamp_micros(arrow_cast(-1, 'Int64')); +---- +1969-12-31T23:59:59.999999 + +# to_timestamp_nanos with negative values +query P +SELECT to_timestamp_nanos(arrow_cast(-1000000000, 'Int64')); +---- +1969-12-31T23:59:59 + +query P +SELECT to_timestamp_nanos(arrow_cast(-1, 'Int64')); +---- +1969-12-31T23:59:59.999999999 + +# Test large unsigned values +query P +SELECT to_timestamp_seconds(arrow_cast(4294967295, 'UInt64')); +---- +2106-02-07T06:28:15 + +# Large UInt64 value for milliseconds +query P +SELECT to_timestamp_millis(arrow_cast(4294967295000, 'UInt64')); +---- +2106-02-07T06:28:15 + +# Test boundary values for to_timestamp +query P +SELECT to_timestamp(arrow_cast(9223372036, 'Int64')); +---- +2262-04-11T23:47:16 + +# Minimum value for to_timestamp +query P +SELECT to_timestamp(arrow_cast(-9223372036, 'Int64')); +---- +1677-09-21T00:12:44 + +# Overflow error when value exceeds valid range +query error Arithmetic overflow +SELECT to_timestamp(arrow_cast(9223372037, 'Int64')); + +# Float truncation behavior +query P +SELECT to_timestamp_seconds(arrow_cast(-1.9, 'Float64')); +---- +1969-12-31T23:59:59 + +query P +SELECT to_timestamp_millis(arrow_cast(-1.9, 'Float64')); +---- +1969-12-31T23:59:59.999 + + ########## ## Common timestamp data ########## From 5fe2cf0d1a9a9287753cfc4acbe2db71589de0f9 Mon Sep 17 00:00:00 2001 From: gokselk Date: Wed, 7 Jan 2026 10:34:55 +0300 Subject: [PATCH 02/10] Add Float16 support to to_timestamp functions --- .../functions/src/datetime/to_timestamp.rs | 10 +++--- .../test_files/datetime/timestamps.slt | 35 +++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index 318d3b52886cb..f658a9fa867ea 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -384,7 +384,7 @@ impl ScalarUDFImpl for ToTimestampFunc { .cast_to(&Timestamp(Second, None), None)? .cast_to(&Timestamp(Nanosecond, tz), None), Null | Timestamp(_, _) => args[0].cast_to(&Timestamp(Nanosecond, tz), None), - Float32 | Float64 => { + Float16 | Float32 | Float64 => { let arg = args[0].cast_to(&Float64, None)?; let rescaled = arrow::compute::kernels::numeric::mul( &arg.to_array(1)?, @@ -485,7 +485,7 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc { | UInt64 | Timestamp(_, _) | Decimal128(_, _) => args[0].cast_to(&Timestamp(Second, tz), None), - Float32 | Float64 => args[0] + Float16 | Float32 | Float64 => args[0] .cast_to(&Int64, None)? .cast_to(&Timestamp(Second, tz), None), Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( @@ -558,7 +558,7 @@ impl ScalarUDFImpl for ToTimestampMillisFunc { | Decimal128(_, _) => { args[0].cast_to(&Timestamp(Millisecond, self.timezone.clone()), None) } - Float32 | Float64 => args[0] + Float16 | Float32 | Float64 => args[0] .cast_to(&Int64, None)? .cast_to(&Timestamp(Millisecond, self.timezone.clone()), None), Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( @@ -631,7 +631,7 @@ impl ScalarUDFImpl for ToTimestampMicrosFunc { | Decimal128(_, _) => { args[0].cast_to(&Timestamp(Microsecond, self.timezone.clone()), None) } - Float32 | Float64 => args[0] + Float16 | Float32 | Float64 => args[0] .cast_to(&Int64, None)? .cast_to(&Timestamp(Microsecond, self.timezone.clone()), None), Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( @@ -704,7 +704,7 @@ impl ScalarUDFImpl for ToTimestampNanosFunc { | Decimal128(_, _) => { args[0].cast_to(&Timestamp(Nanosecond, self.timezone.clone()), None) } - Float32 | Float64 => args[0] + Float16 | Float32 | Float64 => args[0] .cast_to(&Int64, None)? .cast_to(&Timestamp(Nanosecond, self.timezone.clone()), None), Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index 83b57bc6fabdd..87f267a422b0f 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -4515,6 +4515,17 @@ SELECT to_timestamp(arrow_cast(86400, 'UInt64')); ---- 1970-01-02T00:00:00 +# Float16 +query P +SELECT to_timestamp(arrow_cast(0.0, 'Float16')); +---- +1970-01-01T00:00:00 + +query P +SELECT to_timestamp(arrow_cast(1.5, 'Float16')); +---- +1970-01-01T00:00:01.500 + # Float32 query P SELECT to_timestamp(arrow_cast(0.0, 'Float32')); @@ -4579,6 +4590,12 @@ SELECT to_timestamp_seconds(arrow_cast(86400, 'UInt64')); ---- 1970-01-02T00:00:00 +# Float16 +query P +SELECT to_timestamp_seconds(arrow_cast(1.9, 'Float16')); +---- +1970-01-01T00:00:01 + # Float32 query P SELECT to_timestamp_seconds(arrow_cast(1.9, 'Float32')); @@ -4633,6 +4650,12 @@ SELECT to_timestamp_millis(arrow_cast(86400000, 'UInt64')); ---- 1970-01-02T00:00:00 +# Float16 +query P +SELECT to_timestamp_millis(arrow_cast(1000, 'Float16')); +---- +1970-01-01T00:00:01 + # Float32 query P SELECT to_timestamp_millis(arrow_cast(1000.9, 'Float32')); @@ -4687,6 +4710,12 @@ SELECT to_timestamp_micros(arrow_cast(1000000, 'UInt64')); ---- 1970-01-01T00:00:01 +# Float16 +query P +SELECT to_timestamp_micros(arrow_cast(1000, 'Float16')); +---- +1970-01-01T00:00:00.001 + # Float32 query P SELECT to_timestamp_micros(arrow_cast(1000000.9, 'Float32')); @@ -4741,6 +4770,12 @@ SELECT to_timestamp_nanos(arrow_cast(1000000000, 'UInt64')); ---- 1970-01-01T00:00:01 +# Float16 +query P +SELECT to_timestamp_nanos(arrow_cast(1000, 'Float16')); +---- +1970-01-01T00:00:00.000001 + # Float32 query P SELECT to_timestamp_nanos(arrow_cast(1000000000.9, 'Float32')); From d6cb17018ce74d6ba62398b4c66464e9a175316a Mon Sep 17 00:00:00 2001 From: gokselk Date: Wed, 7 Jan 2026 10:34:55 +0300 Subject: [PATCH 03/10] Expand decimal type support in to_timestamp functions --- .../functions/src/datetime/to_timestamp.rs | 120 ++++++++++++++---- .../test_files/datetime/timestamps.slt | 117 ++++++++++++----- 2 files changed, 181 insertions(+), 56 deletions(-) diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index f658a9fa867ea..0f5b71c0be8a3 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -19,8 +19,10 @@ use std::any::Any; use std::sync::Arc; use crate::datetime::common::*; -use arrow::array::Float64Array; use arrow::array::timezone::Tz; +use arrow::array::{ + Array, Decimal128Array, Decimal256Array, Float64Array, TimestampNanosecondArray, +}; use arrow::datatypes::DataType::*; use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second}; use arrow::datatypes::{ @@ -325,6 +327,74 @@ impl_to_timestamp_constructors!(ToTimestampMillisFunc); impl_to_timestamp_constructors!(ToTimestampMicrosFunc); impl_to_timestamp_constructors!(ToTimestampNanosFunc); +fn decimal_to_nanoseconds(value: i128, scale: i8) -> i64 { + let scale_factor = 10_i128.pow(scale as u32); + let seconds = value / scale_factor; + let fraction = value % scale_factor; + let nanos = (fraction * 1_000_000_000) / scale_factor; + let timestamp_nanos = seconds * 1_000_000_000 + nanos; + timestamp_nanos as i64 +} + +fn decimal128_to_timestamp_nanos( + arg: &ColumnarValue, + tz: Option>, +) -> Result { + match arg { + ColumnarValue::Scalar(ScalarValue::Decimal128(Some(value), _, scale)) => { + let timestamp_nanos = decimal_to_nanoseconds(*value, *scale); + Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( + Some(timestamp_nanos), + tz, + ))) + } + ColumnarValue::Scalar(ScalarValue::Decimal128(None, _, _)) => Ok( + ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, tz)), + ), + ColumnarValue::Array(arr) => { + let decimal_arr = downcast_arg!(arr, Decimal128Array); + let scale = decimal_arr.scale(); + let result: TimestampNanosecondArray = decimal_arr + .iter() + .map(|v| v.map(|val| decimal_to_nanoseconds(val, scale))) + .collect(); + let result = result.with_timezone_opt(tz); + Ok(ColumnarValue::Array(Arc::new(result))) + } + _ => exec_err!("Invalid Decimal128 value for to_timestamp"), + } +} + +fn decimal256_to_timestamp_nanos( + arg: &ColumnarValue, + tz: Option>, +) -> Result { + match arg { + ColumnarValue::Scalar(ScalarValue::Decimal256(Some(value), _, scale)) => { + let value_i128 = value.as_i128(); + let timestamp_nanos = decimal_to_nanoseconds(value_i128, *scale); + Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( + Some(timestamp_nanos), + tz, + ))) + } + ColumnarValue::Scalar(ScalarValue::Decimal256(None, _, _)) => Ok( + ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, tz)), + ), + ColumnarValue::Array(arr) => { + let decimal_arr = downcast_arg!(arr, Decimal256Array); + let scale = decimal_arr.scale(); + let result: TimestampNanosecondArray = decimal_arr + .iter() + .map(|v| v.map(|val| decimal_to_nanoseconds(val.as_i128(), scale))) + .collect(); + let result = result.with_timezone_opt(tz); + Ok(ColumnarValue::Array(Arc::new(result))) + } + _ => exec_err!("Invalid Decimal256 value for to_timestamp"), + } +} + /// to_timestamp SQL function /// /// Note: `to_timestamp` returns `Timestamp(Nanosecond)` though its arguments are interpreted as **seconds**. @@ -398,31 +468,15 @@ impl ScalarUDFImpl for ToTimestampFunc { &DEFAULT_CAST_OPTIONS, )?)) } + Decimal32(_, _) | Decimal64(_, _) => { + let arg = args[0].cast_to(&Decimal128(38, 9), None)?; + decimal128_to_timestamp_nanos(&arg, tz) + } + Decimal128(_, _) => decimal128_to_timestamp_nanos(&args[0], tz), + Decimal256(_, _) => decimal256_to_timestamp_nanos(&args[0], tz), Utf8View | LargeUtf8 | Utf8 => { to_timestamp_impl::(&args, "to_timestamp", &tz) } - Decimal128(_, _) => { - match &args[0] { - ColumnarValue::Scalar(ScalarValue::Decimal128( - Some(value), - _, - scale, - )) => { - // Convert decimal to seconds and nanoseconds - let scale_factor = 10_i128.pow(*scale as u32); - let seconds = value / scale_factor; - let fraction = value % scale_factor; - let nanos = (fraction * 1_000_000_000) / scale_factor; - let timestamp_nanos = seconds * 1_000_000_000 + nanos; - - Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(timestamp_nanos as i64), - tz, - ))) - } - _ => exec_err!("Invalid decimal value"), - } - } other => { exec_err!("Unsupported data type {other} for function to_timestamp") } @@ -484,7 +538,10 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc { | UInt32 | UInt64 | Timestamp(_, _) - | Decimal128(_, _) => args[0].cast_to(&Timestamp(Second, tz), None), + | Decimal32(_, _) + | Decimal64(_, _) + | Decimal128(_, _) + | Decimal256(_, _) => args[0].cast_to(&Timestamp(Second, tz), None), Float16 | Float32 | Float64 => args[0] .cast_to(&Int64, None)? .cast_to(&Timestamp(Second, tz), None), @@ -555,7 +612,10 @@ impl ScalarUDFImpl for ToTimestampMillisFunc { | UInt32 | UInt64 | Timestamp(_, _) - | Decimal128(_, _) => { + | Decimal32(_, _) + | Decimal64(_, _) + | Decimal128(_, _) + | Decimal256(_, _) => { args[0].cast_to(&Timestamp(Millisecond, self.timezone.clone()), None) } Float16 | Float32 | Float64 => args[0] @@ -628,7 +688,10 @@ impl ScalarUDFImpl for ToTimestampMicrosFunc { | UInt32 | UInt64 | Timestamp(_, _) - | Decimal128(_, _) => { + | Decimal32(_, _) + | Decimal64(_, _) + | Decimal128(_, _) + | Decimal256(_, _) => { args[0].cast_to(&Timestamp(Microsecond, self.timezone.clone()), None) } Float16 | Float32 | Float64 => args[0] @@ -701,7 +764,10 @@ impl ScalarUDFImpl for ToTimestampNanosFunc { | UInt32 | UInt64 | Timestamp(_, _) - | Decimal128(_, _) => { + | Decimal32(_, _) + | Decimal64(_, _) + | Decimal128(_, _) + | Decimal256(_, _) => { args[0].cast_to(&Timestamp(Nanosecond, self.timezone.clone()), None) } Float16 | Float32 | Float64 => args[0] diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index 87f267a422b0f..c23b5a2e02d34 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -4864,83 +4864,112 @@ SELECT arrow_typeof(to_timestamp_nanos(arrow_cast(0.0, 'Float32'))); ---- Timestamp(ns) -# Test Decimal128 support for all to_timestamp functions -# to_timestamp with Decimal128 +# Test decimal type support for all to_timestamp functions +# Decimal32 query P -SELECT to_timestamp(arrow_cast(0.0, 'Decimal128(10,1)')); +SELECT to_timestamp(arrow_cast(1.5, 'Decimal32(5,1)')); ---- -1970-01-01T00:00:00 +1970-01-01T00:00:01.500 query P -SELECT to_timestamp(arrow_cast(1.5, 'Decimal128(10,1)')); +SELECT to_timestamp_seconds(arrow_cast(86400, 'Decimal32(9,0)')); +---- +1970-01-02T00:00:00 + +query P +SELECT to_timestamp_millis(arrow_cast(1000, 'Decimal32(9,0)')); +---- +1970-01-01T00:00:01 + +query P +SELECT to_timestamp_micros(arrow_cast(1000000, 'Decimal32(9,0)')); +---- +1970-01-01T00:00:01 + +query P +SELECT to_timestamp_nanos(arrow_cast(1000000, 'Decimal32(9,0)')); +---- +1970-01-01T00:00:00.001 + +# Decimal64 +query P +SELECT to_timestamp(arrow_cast(1.5, 'Decimal64(10,1)')); ---- 1970-01-01T00:00:01.500 query P -SELECT to_timestamp(arrow_cast(86400.123, 'Decimal128(10,3)')); +SELECT to_timestamp_seconds(arrow_cast(86400, 'Decimal64(18,0)')); ---- -1970-01-02T00:00:00.123 +1970-01-02T00:00:00 -# to_timestamp_seconds with Decimal128 query P -SELECT to_timestamp_seconds(arrow_cast(0, 'Decimal128(10,0)')); +SELECT to_timestamp_millis(arrow_cast(86400000, 'Decimal64(18,0)')); ---- -1970-01-01T00:00:00 +1970-01-02T00:00:00 query P -SELECT to_timestamp_seconds(arrow_cast(86400, 'Decimal128(10,0)')); +SELECT to_timestamp_micros(arrow_cast(86400000000, 'Decimal64(18,0)')); ---- 1970-01-02T00:00:00 -# to_timestamp_millis with Decimal128 query P -SELECT to_timestamp_millis(arrow_cast(0, 'Decimal128(10,0)')); +SELECT to_timestamp_nanos(arrow_cast(86400000000000, 'Decimal64(18,0)')); ---- -1970-01-01T00:00:00 +1970-01-02T00:00:00 +# Decimal128 query P -SELECT to_timestamp_millis(arrow_cast(1000, 'Decimal128(10,0)')); +SELECT to_timestamp(arrow_cast(1.5, 'Decimal128(10,1)')); ---- -1970-01-01T00:00:01 +1970-01-01T00:00:01.500 + +query P +SELECT to_timestamp_seconds(arrow_cast(86400, 'Decimal128(10,0)')); +---- +1970-01-02T00:00:00 query P SELECT to_timestamp_millis(arrow_cast(86400000, 'Decimal128(15,0)')); ---- 1970-01-02T00:00:00 -# to_timestamp_micros with Decimal128 query P -SELECT to_timestamp_micros(arrow_cast(0, 'Decimal128(10,0)')); +SELECT to_timestamp_micros(arrow_cast(86400000000, 'Decimal128(15,0)')); ---- -1970-01-01T00:00:00 +1970-01-02T00:00:00 query P -SELECT to_timestamp_micros(arrow_cast(1000000, 'Decimal128(10,0)')); +SELECT to_timestamp_nanos(arrow_cast(86400000000000, 'Decimal128(20,0)')); ---- -1970-01-01T00:00:01 +1970-01-02T00:00:00 +# Decimal256 query P -SELECT to_timestamp_micros(arrow_cast(86400000000, 'Decimal128(15,0)')); +SELECT to_timestamp(arrow_cast(1.5, 'Decimal256(10,1)')); +---- +1970-01-01T00:00:01.500 + +query P +SELECT to_timestamp_seconds(arrow_cast(86400, 'Decimal256(38,0)')); ---- 1970-01-02T00:00:00 -# to_timestamp_nanos with Decimal128 query P -SELECT to_timestamp_nanos(arrow_cast(0, 'Decimal128(10,0)')); +SELECT to_timestamp_millis(arrow_cast(86400000, 'Decimal256(38,0)')); ---- -1970-01-01T00:00:00 +1970-01-02T00:00:00 query P -SELECT to_timestamp_nanos(arrow_cast(1000000000, 'Decimal128(15,0)')); +SELECT to_timestamp_micros(arrow_cast(86400000000, 'Decimal256(38,0)')); ---- -1970-01-01T00:00:01 +1970-01-02T00:00:00 query P -SELECT to_timestamp_nanos(arrow_cast(86400000000000, 'Decimal128(20,0)')); +SELECT to_timestamp_nanos(arrow_cast(86400000000000, 'Decimal256(38,0)')); ---- 1970-01-02T00:00:00 -# Verify arrow_typeof for Decimal128 inputs +# Verify arrow_typeof for decimal inputs query T SELECT arrow_typeof(to_timestamp(arrow_cast(0, 'Decimal128(10,0)'))); ---- @@ -4966,6 +4995,36 @@ SELECT arrow_typeof(to_timestamp_nanos(arrow_cast(0, 'Decimal128(10,0)'))); ---- Timestamp(ns) +# Test decimal array inputs for to_timestamp +statement ok +CREATE TABLE test_decimal_timestamps ( + d128 DECIMAL(20, 9), + d256 DECIMAL(40, 9) +) AS VALUES + (1.5, 1.5), + (86400.123456789, 86400.123456789), + (0.0, 0.0), + (NULL, NULL); + +query P +SELECT to_timestamp(d128) FROM test_decimal_timestamps ORDER BY d128 NULLS LAST; +---- +1970-01-01T00:00:00 +1970-01-01T00:00:01.500 +1970-01-02T00:00:00.123456789 +NULL + +query P +SELECT to_timestamp(d256) FROM test_decimal_timestamps ORDER BY d256 NULLS LAST; +---- +1970-01-01T00:00:00 +1970-01-01T00:00:01.500 +1970-01-02T00:00:00.123456789 +NULL + +statement ok +DROP TABLE test_decimal_timestamps; + # Test negative values # to_timestamp with negative seconds query P From ae457c32cc06bb4cc29af92073050500a6d77ee9 Mon Sep 17 00:00:00 2001 From: gokselk Date: Wed, 7 Jan 2026 10:34:55 +0300 Subject: [PATCH 04/10] Add test for UInt64 values exceeding i64::MAX --- datafusion/sqllogictest/test_files/datetime/timestamps.slt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index c23b5a2e02d34..aff3df7ed9d5f 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -5098,6 +5098,10 @@ SELECT to_timestamp_millis(arrow_cast(4294967295000, 'UInt64')); ---- 2106-02-07T06:28:15 +# Test UInt64 value larger than i64::MAX (9223372036854775808 = i64::MAX + 1) +query error Cast error: Can't cast value 9223372036854775808 to type Int64 +SELECT to_timestamp_nanos(arrow_cast(9223372036854775808, 'UInt64')); + # Test boundary values for to_timestamp query P SELECT to_timestamp(arrow_cast(9223372036, 'Int64')); From 1a2facb4613a9f28ecf53ffccc7eeac84d1a784b Mon Sep 17 00:00:00 2001 From: gokselk Date: Wed, 7 Jan 2026 12:00:07 +0300 Subject: [PATCH 05/10] Fix decimal_to_nanoseconds for negative scales --- .../functions/src/datetime/to_timestamp.rs | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index 0f5b71c0be8a3..ac5ac995abf82 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -328,11 +328,12 @@ impl_to_timestamp_constructors!(ToTimestampMicrosFunc); impl_to_timestamp_constructors!(ToTimestampNanosFunc); fn decimal_to_nanoseconds(value: i128, scale: i8) -> i64 { - let scale_factor = 10_i128.pow(scale as u32); - let seconds = value / scale_factor; - let fraction = value % scale_factor; - let nanos = (fraction * 1_000_000_000) / scale_factor; - let timestamp_nanos = seconds * 1_000_000_000 + nanos; + let nanos_exponent = 9_i16 - scale as i16; + let timestamp_nanos = if nanos_exponent >= 0 { + value * 10_i128.pow(nanos_exponent as u32) + } else { + value / 10_i128.pow(nanos_exponent.unsigned_abs() as u32) + }; timestamp_nanos as i64 } @@ -1852,4 +1853,23 @@ mod tests { assert_contains!(actual, expected); } } + + #[test] + fn test_decimal_to_nanoseconds_negative_scale() { + // scale -2: internal value 5 represents 5 * 10^2 = 500 seconds + let nanos = decimal_to_nanoseconds(5, -2); + assert_eq!(nanos, 500_000_000_000); // 500 seconds in nanoseconds + + // scale -1: internal value 10 represents 10 * 10^1 = 100 seconds + let nanos = decimal_to_nanoseconds(10, -1); + assert_eq!(nanos, 100_000_000_000); + + // scale 0: internal value 5 represents 5 seconds + let nanos = decimal_to_nanoseconds(5, 0); + assert_eq!(nanos, 5_000_000_000); + + // scale 3: internal value 1500 represents 1.5 seconds + let nanos = decimal_to_nanoseconds(1500, 3); + assert_eq!(nanos, 1_500_000_000); + } } From e8587cf49aa13a02402b6750391b744b96ffba4b Mon Sep 17 00:00:00 2001 From: gokselk Date: Thu, 8 Jan 2026 12:48:52 +0300 Subject: [PATCH 06/10] Add positive Int32/Int64 tests for to_timestamp_millis --- .../sqllogictest/test_files/datetime/timestamps.slt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index aff3df7ed9d5f..82d45cd53b8a5 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -4626,6 +4626,18 @@ SELECT to_timestamp_millis(arrow_cast(1000, 'Int16')); ---- 1970-01-01T00:00:01 +# Int32 +query P +SELECT to_timestamp_millis(arrow_cast(86400000, 'Int32')); +---- +1970-01-02T00:00:00 + +# Int64 +query P +SELECT to_timestamp_millis(arrow_cast(86400000, 'Int64')); +---- +1970-01-02T00:00:00 + # UInt8 query P SELECT to_timestamp_millis(arrow_cast(100, 'UInt8')); From b1949302d2d9866a9dcf02c9fb3298cef6f01ffc Mon Sep 17 00:00:00 2001 From: gokselk Date: Thu, 8 Jan 2026 12:48:52 +0300 Subject: [PATCH 07/10] Add missing positive Int32/Int64 tests for to_timestamp functions --- .../test_files/datetime/timestamps.slt | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index 82d45cd53b8a5..0ad3ef2ecd956 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -4566,6 +4566,18 @@ SELECT to_timestamp_seconds(arrow_cast(1000, 'Int16')); ---- 1970-01-01T00:16:40 +# Int32 +query P +SELECT to_timestamp_seconds(arrow_cast(86400, 'Int32')); +---- +1970-01-02T00:00:00 + +# Int64 +query P +SELECT to_timestamp_seconds(arrow_cast(86400, 'Int64')); +---- +1970-01-02T00:00:00 + # UInt8 query P SELECT to_timestamp_seconds(arrow_cast(100, 'UInt8')); @@ -4698,6 +4710,18 @@ SELECT to_timestamp_micros(arrow_cast(1000, 'Int16')); ---- 1970-01-01T00:00:00.001 +# Int32 +query P +SELECT to_timestamp_micros(arrow_cast(1000000, 'Int32')); +---- +1970-01-01T00:00:01 + +# Int64 +query P +SELECT to_timestamp_micros(arrow_cast(86400000000, 'Int64')); +---- +1970-01-02T00:00:00 + # UInt8 query P SELECT to_timestamp_micros(arrow_cast(100, 'UInt8')); @@ -4758,6 +4782,18 @@ SELECT to_timestamp_nanos(arrow_cast(1000, 'Int16')); ---- 1970-01-01T00:00:00.000001 +# Int32 +query P +SELECT to_timestamp_nanos(arrow_cast(1000000000, 'Int32')); +---- +1970-01-01T00:00:01 + +# Int64 +query P +SELECT to_timestamp_nanos(arrow_cast(86400000000000, 'Int64')); +---- +1970-01-02T00:00:00 + # UInt8 query P SELECT to_timestamp_nanos(arrow_cast(100, 'UInt8')); From 6fb4435acdc62e212bebb0699c7fa2d05c9caff2 Mon Sep 17 00:00:00 2001 From: gokselk Date: Thu, 8 Jan 2026 18:51:59 +0300 Subject: [PATCH 08/10] Add missing negative integer type tests for to_timestamp functions --- .../test_files/datetime/timestamps.slt | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index 0ad3ef2ecd956..700f894cebd83 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -5075,55 +5075,131 @@ DROP TABLE test_decimal_timestamps; # Test negative values # to_timestamp with negative seconds +# Int8 +query P +SELECT to_timestamp(arrow_cast(-1, 'Int8')); +---- +1969-12-31T23:59:59 + +# Int16 +query P +SELECT to_timestamp(arrow_cast(-1, 'Int16')); +---- +1969-12-31T23:59:59 + +# Int32 query P SELECT to_timestamp(arrow_cast(-86400, 'Int32')); ---- 1969-12-31T00:00:00 +# Int64 query P SELECT to_timestamp(arrow_cast(-1, 'Int64')); ---- 1969-12-31T23:59:59 +# Float64 query P SELECT to_timestamp(arrow_cast(-0.5, 'Float64')); ---- 1969-12-31T23:59:59.500 # to_timestamp_seconds with negative values +# Int8 +query P +SELECT to_timestamp_seconds(arrow_cast(-1, 'Int8')); +---- +1969-12-31T23:59:59 + +# Int16 +query P +SELECT to_timestamp_seconds(arrow_cast(-1, 'Int16')); +---- +1969-12-31T23:59:59 + +# Int32 query P SELECT to_timestamp_seconds(arrow_cast(-86400, 'Int32')); ---- 1969-12-31T00:00:00 +# Int64 query P SELECT to_timestamp_seconds(arrow_cast(-1, 'Int64')); ---- 1969-12-31T23:59:59 # to_timestamp_millis with negative values +# Int8 +query P +SELECT to_timestamp_millis(arrow_cast(-1, 'Int8')); +---- +1969-12-31T23:59:59.999 + +# Int16 +query P +SELECT to_timestamp_millis(arrow_cast(-1, 'Int16')); +---- +1969-12-31T23:59:59.999 + +# Int32 query P SELECT to_timestamp_millis(arrow_cast(-1000, 'Int32')); ---- 1969-12-31T23:59:59 +# Int64 query P SELECT to_timestamp_millis(arrow_cast(-1, 'Int64')); ---- 1969-12-31T23:59:59.999 # to_timestamp_micros with negative values +# Int8 +query P +SELECT to_timestamp_micros(arrow_cast(-1, 'Int8')); +---- +1969-12-31T23:59:59.999999 + +# Int16 +query P +SELECT to_timestamp_micros(arrow_cast(-1, 'Int16')); +---- +1969-12-31T23:59:59.999999 + +# Int32 query P SELECT to_timestamp_micros(arrow_cast(-1000000, 'Int32')); ---- 1969-12-31T23:59:59 +# Int64 query P SELECT to_timestamp_micros(arrow_cast(-1, 'Int64')); ---- 1969-12-31T23:59:59.999999 # to_timestamp_nanos with negative values +# Int8 +query P +SELECT to_timestamp_nanos(arrow_cast(-1, 'Int8')); +---- +1969-12-31T23:59:59.999999999 + +# Int16 +query P +SELECT to_timestamp_nanos(arrow_cast(-1, 'Int16')); +---- +1969-12-31T23:59:59.999999999 + +# Int32 +query P +SELECT to_timestamp_nanos(arrow_cast(-1000000000, 'Int32')); +---- +1969-12-31T23:59:59 + +# Int64 query P SELECT to_timestamp_nanos(arrow_cast(-1000000000, 'Int64')); ---- From 07258356a2d3199c8b311e444f4fcaddced6f8e1 Mon Sep 17 00:00:00 2001 From: gokselk Date: Fri, 9 Jan 2026 13:21:43 +0300 Subject: [PATCH 09/10] Handle float types natively in to_timestamp --- .../functions/src/datetime/to_timestamp.rs | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index ac5ac995abf82..2e8cc5c56d6a4 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -21,7 +21,8 @@ use std::sync::Arc; use crate::datetime::common::*; use arrow::array::timezone::Tz; use arrow::array::{ - Array, Decimal128Array, Decimal256Array, Float64Array, TimestampNanosecondArray, + Array, Decimal128Array, Decimal256Array, Float16Array, Float32Array, Float64Array, + TimestampNanosecondArray, }; use arrow::datatypes::DataType::*; use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second}; @@ -30,7 +31,6 @@ use arrow::datatypes::{ TimestampNanosecondType, TimestampSecondType, }; use datafusion_common::config::ConfigOptions; -use datafusion_common::format::DEFAULT_CAST_OPTIONS; use datafusion_common::{Result, ScalarType, ScalarValue, exec_err}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarUDF, ScalarUDFImpl, Signature, Volatility, @@ -455,19 +455,26 @@ impl ScalarUDFImpl for ToTimestampFunc { .cast_to(&Timestamp(Second, None), None)? .cast_to(&Timestamp(Nanosecond, tz), None), Null | Timestamp(_, _) => args[0].cast_to(&Timestamp(Nanosecond, tz), None), - Float16 | Float32 | Float64 => { - let arg = args[0].cast_to(&Float64, None)?; - let rescaled = arrow::compute::kernels::numeric::mul( - &arg.to_array(1)?, - &arrow::array::Scalar::new(Float64Array::from(vec![ - 1_000_000_000f64, - ])), - )?; - Ok(ColumnarValue::Array(arrow::compute::cast_with_options( - &rescaled, - &Timestamp(Nanosecond, tz), - &DEFAULT_CAST_OPTIONS, - )?)) + Float16 => { + let arr = args[0].to_array(1)?; + let f16_arr = downcast_arg!(&arr, Float16Array); + let result: TimestampNanosecondArray = + f16_arr.unary(|x| (x.to_f64() * 1_000_000_000.0) as i64); + Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz)))) + } + Float32 => { + let arr = args[0].to_array(1)?; + let f32_arr = downcast_arg!(&arr, Float32Array); + let result: TimestampNanosecondArray = + f32_arr.unary(|x| (x as f64 * 1_000_000_000.0) as i64); + Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz)))) + } + Float64 => { + let arr = args[0].to_array(1)?; + let f64_arr = downcast_arg!(&arr, Float64Array); + let result: TimestampNanosecondArray = + f64_arr.unary(|x| (x * 1_000_000_000.0) as i64); + Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz)))) } Decimal32(_, _) | Decimal64(_, _) => { let arg = args[0].cast_to(&Decimal128(38, 9), None)?; From 376bd9d3c8fc7bcbd3179cd983ce7180051a1010 Mon Sep 17 00:00:00 2001 From: gokselk Date: Fri, 9 Jan 2026 13:23:39 +0300 Subject: [PATCH 10/10] Cast Decimal256 to Decimal128 in to_timestamp --- .../functions/src/datetime/to_timestamp.rs | 35 ++----------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index 2e8cc5c56d6a4..1c5d3dbd88bcd 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use crate::datetime::common::*; use arrow::array::timezone::Tz; use arrow::array::{ - Array, Decimal128Array, Decimal256Array, Float16Array, Float32Array, Float64Array, + Array, Decimal128Array, Float16Array, Float32Array, Float64Array, TimestampNanosecondArray, }; use arrow::datatypes::DataType::*; @@ -366,36 +366,6 @@ fn decimal128_to_timestamp_nanos( } } -fn decimal256_to_timestamp_nanos( - arg: &ColumnarValue, - tz: Option>, -) -> Result { - match arg { - ColumnarValue::Scalar(ScalarValue::Decimal256(Some(value), _, scale)) => { - let value_i128 = value.as_i128(); - let timestamp_nanos = decimal_to_nanoseconds(value_i128, *scale); - Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(timestamp_nanos), - tz, - ))) - } - ColumnarValue::Scalar(ScalarValue::Decimal256(None, _, _)) => Ok( - ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, tz)), - ), - ColumnarValue::Array(arr) => { - let decimal_arr = downcast_arg!(arr, Decimal256Array); - let scale = decimal_arr.scale(); - let result: TimestampNanosecondArray = decimal_arr - .iter() - .map(|v| v.map(|val| decimal_to_nanoseconds(val.as_i128(), scale))) - .collect(); - let result = result.with_timezone_opt(tz); - Ok(ColumnarValue::Array(Arc::new(result))) - } - _ => exec_err!("Invalid Decimal256 value for to_timestamp"), - } -} - /// to_timestamp SQL function /// /// Note: `to_timestamp` returns `Timestamp(Nanosecond)` though its arguments are interpreted as **seconds**. @@ -476,12 +446,11 @@ impl ScalarUDFImpl for ToTimestampFunc { f64_arr.unary(|x| (x * 1_000_000_000.0) as i64); Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz)))) } - Decimal32(_, _) | Decimal64(_, _) => { + Decimal32(_, _) | Decimal64(_, _) | Decimal256(_, _) => { let arg = args[0].cast_to(&Decimal128(38, 9), None)?; decimal128_to_timestamp_nanos(&arg, tz) } Decimal128(_, _) => decimal128_to_timestamp_nanos(&args[0], tz), - Decimal256(_, _) => decimal256_to_timestamp_nanos(&args[0], tz), Utf8View | LargeUtf8 | Utf8 => { to_timestamp_impl::(&args, "to_timestamp", &tz) }