Fix binary date/time conversion; add Date32

CHANGELOG.md

@@ -25,6 +25,21 @@ All notable changes to this project will be documented in this file. It uses the
     docs](doc/pg_clickhouse.md), including the new `PREPARE`/`EXECUTE` support
     and `INSERT`, `SET`, `COPY`, as well as shared library preloading.
 
+### 🪲 Bug Fixes
+
+*   Fixed the http engine's parsing of UUID arrays selected from ClickHouse.
+*   Fixed the binary engine's conversion of Date values, which in arrays ended
+    up too large by several orders of magnitude (e.g., `2025-12-05` would be
+    converted to `10529827-09-17` 😱). Thanks to Tom Lane for the pointer to
+    the proper function to easily convert epoch seconds to a date.
+*   Fixed a binary engine bug where dates and timestamps for epoch 0
+    (`1970-01-01 00:00:00`) rendered as `NULL`.
+*   Added support for the `Date32` ClickHouse type.
+*   Fixed conversion of `array_agg()` to support ClickHouse versions prior to
+    23.8.
+*   Fixed the precision of fractional seconds in the binary engine's
+    conversion of ClickHouse DateTime64 values to Postgres TIMESTAMP (#114).
+
 ### 📔 Notes
 
 *   Removed unused code designed to support custom PostgreSQL extensions:

doc/pg_clickhouse.md

@@ -788,6 +788,7 @@ types:
 ------------+------------------+-------------------------------
  Bool       | boolean          | 
  Date       | date             | 
+ Date32     | date             | 
  DateTime   | timestamp        | 
  Decimal    | numeric          | 
  Float32    | real             | 
@@ -916,6 +917,8 @@ pushed down.
 
 These PostgreSQL aggregate functions pushdown to ClickHouse.
 
+*   [array_agg](https://clickhouse.com/docs/sql-reference/aggregate-functions/reference/grouparray)
+*   [avg](https://clickhouse.com/docs/sql-reference/aggregate-functions/reference/avg)
 *   [count](https://clickhouse.com/docs/sql-reference/aggregate-functions/reference/count)
 
 ### Custom Aggregates

src/binary.cpp

@@ -299,6 +299,7 @@ static Oid get_corr_postgres_type(const TypeRef & type)
 		case Type::Code::LowCardinality:
 			return get_corr_postgres_type(type->As<LowCardinalityType>()->GetNestedType());
 		case Type::Code::Date:
+		case Type::Code::Date32:
 			return DATEOID;
 		case Type::Code::DateTime:
 			return TIMESTAMPOID;
@@ -538,14 +539,17 @@ static void column_append(clickhouse::ColumnRef col, Datum val, Oid valtype, boo
 			break;
 		}
 		case DATEOID: {
-			Timestamp t = date2timestamp_no_overflow(DatumGetDateADT(val));
-			pg_time_t d = timestamptz_to_time_t(t);
+           Timestamp t = date2timestamp_no_overflow(DatumGetDateADT(val));
+           pg_time_t d = timestamptz_to_time_t(t);
 
 			switch (col->Type()->GetCode())
 			{
 				case Type::Code::Date:
 					col->As<ColumnDate>()->Append(d);
 					break;
+				case Type::Code::Date32:
+					col->As<ColumnDate32>()->Append(d);
+					break;
 				default:
 					THROW_UNEXPECTED_COLUMN("DATE", col);
 			}
@@ -864,44 +868,28 @@ static Datum make_datum(clickhouse::ColumnRef col, size_t row, Oid * valtype, bo
 		case Type::Code::Date: {
 			auto val = static_cast<pg_time_t>(col->As<ColumnDate>()->At(row));
 			*valtype = DATEOID;
-
-			if (val == 0)
-				/* clickhouse special case */
-				*is_null = true;
-			else
-			{
-				Timestamp t = (Timestamp)time_t_to_timestamptz(val);
-				ret = TimestampGetDatum(t);
-			}
+			ret = DirectFunctionCall1(timestamp_date, time_t_to_timestamptz(val));
+		}
+		break;
+		case Type::Code::Date32: {
+			auto val = static_cast<pg_time_t>(col->As<ColumnDate32>()->At(row));
+			*valtype = DATEOID;
+			ret = DirectFunctionCall1(timestamp_date, time_t_to_timestamptz(val));
 		}
 		break;
 		case Type::Code::DateTime: {
 			auto val = static_cast<pg_time_t>(col->As<ColumnDateTime>()->At(row));
 			*valtype = TIMESTAMPOID;
-
-			if (val == 0)
-				*is_null = true;
-			else
-			{
-				Timestamp t = (Timestamp)time_t_to_timestamptz(val);
-				ret = TimestampGetDatum(t);
-			}
+			ret = TimestampGetDatum(time_t_to_timestamptz(val));
 		}
 		break;
 		case Type::Code::DateTime64: {
 			auto dt_col = col->As<ColumnDateTime64>();
 			auto val = dt_col->At(row);
-
+			int64 power = pow(10, dt_col->GetPrecision());
 			*valtype = TIMESTAMPOID;
-
-			if (val == 0)
-				*is_null = true;
-			else
-			{
-				ret = ((1.0 * val) / pow(10, dt_col->GetPrecision())
-					   - (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)
-					* USECS_PER_SEC;
-			}
+			ret = TimestampGetDatum(time_t_to_timestamptz(val / power))
+				+ (val % power) * (USECS_PER_SEC / power);
 		}
 		break;
 		case Type::Code::UUID: {

src/convert.c

@@ -181,13 +181,6 @@ convert_bool_to_int16(ch_convert_output_state * state, Datum val)
 	return Int16GetDatum(DatumGetBool(val) ? 1 : 0);
 }
 
-static Datum
-convert_date(ch_convert_state * state, Datum val)
-{
-	val = DirectFunctionCall1(timestamp_date, val);
-	return convert_generic(state, val);
-}
-
 Datum
 ch_binary_convert_datum(void *state, Datum val)
 {
@@ -206,9 +199,7 @@ ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype)
 	state->typmod = -1;
 	state->ctype = COERCION_PATH_NONE;
 
-	if (intype == DATEOID)
-		state->func = convert_date;
-	else if (intype == ANYARRAYOID)
+	if (intype == ANYARRAYOID)
 	{
 		ch_binary_array_t *slot = (ch_binary_array_t *) DatumGetPointer(val);
 

src/custom_types.c

@@ -37,6 +37,7 @@
 #define F_DATE_PART_TEXT_DATE 1384
 #define F_PERCENTILE_CONT_FLOAT8_FLOAT8 3974
 #define F_PERCENTILE_CONT_FLOAT8_INTERVAL 3976
+#define F_ARRAY_AGG_ANYNONARRAY 2335
 
 /*
  * Prior to Postgres 14 EXTRACT mapped directly to DATE_PART.
@@ -195,6 +196,7 @@ chfdw_check_for_custom_function(Oid funcid)
 			case F_REGEXP_LIKE_TEXT_TEXT:
 			case F_PERCENTILE_CONT_FLOAT8_FLOAT8:
 			case F_PERCENTILE_CONT_FLOAT8_INTERVAL:
+			case F_ARRAY_AGG_ANYNONARRAY:
 				special_builtin = true;
 				break;
 			default:
@@ -269,6 +271,11 @@ chfdw_check_for_custom_function(Oid funcid)
 					strcpy(entry->custom_name, "quantile");
 					break;
 				}
+			case F_ARRAY_AGG_ANYNONARRAY:
+				{
+					strcpy(entry->custom_name, "groupArray");
+					break;
+				}
 		}
 
 		if (special_builtin)

src/fdw.c.in

@@ -905,6 +905,9 @@ fetch_tuple(ChFdwScanState * fsstate, TupleDesc tupdesc)
 						valstr[pos] = '{';
 					if (valstr[pos] == ']')
 						valstr[pos] = '}';
+					if (valstr[pos] == '\'' && pgtype == UUIDARRAYOID)
+						/* Remove ClickHouse's single quotes around UUIDs */
+						valstr[pos] = ' ';
 					pos++;
 				}
 			}

src/pglink.c

@@ -382,7 +382,7 @@ chfdw_datum_to_ch_literal(Datum value, Oid type)
 				text = OidOutputFunctionCall(typoutput, value);
 				len = strlen(text);
 				result = palloc(len * 2 + 1);
-				ch_escape_string(result, text, len+1);
+				ch_escape_string(result, text, len + 1);
 				return result;
 			}
 		case DATEOID:
@@ -785,6 +785,7 @@ binary_insert_tuple(void *istate, TupleTableSlot * slot)
 		('String',   'text',             ''),
 		('DateTime', 'timestamp',        ''),
 		('Date',     'date',             ''),
+		('Date32',   'date',             ''),
 		('UUID',     'uuid',             ''),
 		('IPv4',     'inet',             ''),
 		('IPv6',     'inet',             ''),
@@ -810,6 +811,7 @@ static char *str_types_map[][2] = {
 	{"String", "TEXT"},
 	{"DateTime", "TIMESTAMP"},
 	{"Date", "DATE"},			/* must come after other Date types */
+	{"Date32", "DATE"},
 	{"UUID", "UUID"},
 	{"IPv4", "inet"},
 	{"IPv6", "inet"},