diff --git a/CHANGELOG.md b/CHANGELOG.md index 6993b57..151ab6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # DuckDB Connector Changelog This changelog documents changes between release tags. +## [0.1.1] - 2025-01-08 +* Add table comments to descriptions +* Special case Timestamps and BigInt filtering + ## [0.1.0] - 2024-08-22 * Update Documentation for ndc-hub diff --git a/generate-config.ts b/generate-config.ts index bb75746..024904e 100644 --- a/generate-config.ts +++ b/generate-config.ts @@ -16,7 +16,7 @@ const con = db.connect(); const determineType = (t: string): string => { switch (t) { case "BIGINT": - return "Int"; + return "BigInt"; case "BIT": return "String"; case "BOOLEAN": @@ -28,7 +28,7 @@ const determineType = (t: string): string => { case "DOUBLE": return "Float"; case "HUGEINT": - return "String"; + return "HugeInt"; case "INTEGER": return "Int"; case "INTERVAL": @@ -42,13 +42,15 @@ const determineType = (t: string): string => { case "TIME": return "String"; case "TIMESTAMP": - return "String"; + return "Timestamp"; case "TIMESTAMP WITH TIME ZONE": - return "String"; + return "TimestampTz"; case "TINYINT": return "Int"; case "UBIGINT": - return "String"; + return "UBigInt"; + case "UHUGEINT": + return "UHugeInt" case "UINTEGER": return "Int"; case "USMALLINT": @@ -59,8 +61,6 @@ const determineType = (t: string): string => { return "String"; case "VARCHAR": return "String"; - case "HUGEINT": - return "String"; default: if (t.startsWith("DECIMAL")){ return "Float"; @@ -86,22 +86,55 @@ async function main() { const tableAliases: {[k: string]: string} = {}; const objectTypes: { [k: string]: ObjectType } = {}; const tables = await queryAll(con, "SHOW ALL TABLES"); + + // Get table comments + const tableComments = await queryAll(con, ` + SELECT table_name, comment + FROM duckdb_tables() + WHERE schema_name = 'main' + `); + + // Create a map of table comments for easier lookup + const tableCommentMap = new Map( + tableComments.map(row => [row.table_name, row.comment || "No description available"]) + ); + // Get all column comments upfront + const columnComments = await queryAll(con, ` + SELECT table_name, column_name, comment + FROM duckdb_columns() + WHERE schema_name = 'main' + `); + // Create a nested map for column comments: table_name -> column_name -> comment + const columnCommentMap = new Map(); + for (const row of columnComments) { + if (!columnCommentMap.has(row.table_name)) { + columnCommentMap.set(row.table_name, new Map()); + } + columnCommentMap.get(row.table_name).set(row.column_name, row.comment || "No description available"); + } + for (let table of tables){ - const tableName = `${table.database}_${table.schema}_${table.name}`; + const tableName = table.name; const aliasName = `${table.database}.${table.schema}.${table.name}`; tableNames.push(tableName); tableAliases[tableName] = aliasName; if (!objectTypes[tableName]){ objectTypes[tableName] = { - fields: {} + fields: {}, + description: tableCommentMap.get(tableName) || "No Description Available" }; } for (let i = 0; i < table.column_names.length; i++){ - objectTypes[tableName]['fields'][table.column_names[i]] = { + const columnName = table.column_names[i]; + objectTypes[tableName]["fields"][columnName] = { type: { - type: "named", - name: determineType(table.column_types[i]) - } + type: "nullable", + underlying_type: { + type: "named", + name: determineType(table.column_types[i]), + }, + }, + description: columnCommentMap.get(tableName)?.get(columnName) || "No description available" } } } diff --git a/package-lock.json b/package-lock.json index 4e85334..3f410e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "duckdb-sdk", - "version": "0.1.0", + "version": "0.1.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "duckdb-sdk", - "version": "0.1.0", + "version": "0.1.1", "dependencies": { "@hasura/ndc-sdk-typescript": "^6.0.0", "duckdb": "^1.0.0", diff --git a/package.json b/package.json index d6e0cf0..c1a9b96 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "duckdb-sdk", - "version": "0.1.0", + "version": "0.1.1", "description": "", "main": "index.js", "scripts": { diff --git a/src/constants.ts b/src/constants.ts index d846c01..20fb79b 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -16,7 +16,197 @@ export const CAPABILITIES_RESPONSE: Capabilities = { }; export const MAX_32_INT: number = 2147483647; export const SCALAR_TYPES: { [key: string]: ScalarType } = { + BigInt: { + representation: { + type: "biginteger" + }, + aggregate_functions: { + // sum: { + // result_type: { + // type: "named", + // name: "Int" + // } + // } + }, + comparison_operators: { + _eq: { + type: "equal", + }, + _gt: { + type: "custom", + argument_type: { + type: "named", + name: "BigInt", + }, + }, + _lt: { + type: "custom", + argument_type: { + type: "named", + name: "BigInt", + }, + }, + _gte: { + type: "custom", + argument_type: { + type: "named", + name: "BigInt", + }, + }, + _lte: { + type: "custom", + argument_type: { + type: "named", + name: "BigInt", + }, + }, + _neq: { + type: "custom", + argument_type: { + type: "named", + name: "BigInt", + }, + }, + }, + }, + UBigInt: { + representation: { + type: "biginteger" + }, + aggregate_functions: {}, + comparison_operators: { + _eq: { type: "equal" }, + _gt: { type: "custom", argument_type: { type: "named", name: "UBigInt" }}, + _lt: { type: "custom", argument_type: { type: "named", name: "UBigInt" }}, + _gte: { type: "custom", argument_type: { type: "named", name: "UBigInt" }}, + _lte: { type: "custom", argument_type: { type: "named", name: "UBigInt" }}, + _neq: { type: "custom", argument_type: { type: "named", name: "UBigInt" }}, + }, + }, + HugeInt: { + representation: { + type: "biginteger" + }, + aggregate_functions: {}, + comparison_operators: { + _eq: { type: "equal" }, + _gt: { type: "custom", argument_type: { type: "named", name: "HugeInt" }}, + _lt: { type: "custom", argument_type: { type: "named", name: "HugeInt" }}, + _gte: { type: "custom", argument_type: { type: "named", name: "HugeInt" }}, + _lte: { type: "custom", argument_type: { type: "named", name: "HugeInt" }}, + _neq: { type: "custom", argument_type: { type: "named", name: "HugeInt" }}, + }, + }, + UHugeInt: { + representation: { + type: "biginteger" + }, + aggregate_functions: {}, + comparison_operators: { + _eq: { type: "equal" }, + _gt: { type: "custom", argument_type: { type: "named", name: "UHugeInt" }}, + _lt: { type: "custom", argument_type: { type: "named", name: "UHugeInt" }}, + _gte: { type: "custom", argument_type: { type: "named", name: "UHugeInt" }}, + _lte: { type: "custom", argument_type: { type: "named", name: "UHugeInt" }}, + _neq: { type: "custom", argument_type: { type: "named", name: "UHugeInt" }}, + }, + }, + Timestamp: { + representation: { + type: "timestamp" + }, + aggregate_functions: {}, + comparison_operators: { + _eq: { + type: "equal", + }, + _gt: { + type: "custom", + argument_type: { + type: "named", + name: "Timestamp", + }, + }, + _lt: { + type: "custom", + argument_type: { + type: "named", + name: "Timestamp", + }, + }, + _gte: { + type: "custom", + argument_type: { + type: "named", + name: "Timestamp", + }, + }, + _lte: { + type: "custom", + argument_type: { + type: "named", + name: "Timestamp", + }, + }, + _neq: { + type: "custom", + argument_type: { + type: "named", + name: "Timestamp", + }, + }, + }, + }, + TimestampTz: { + representation: { + type: "timestamptz" + }, + aggregate_functions: {}, + comparison_operators: { + _eq: { + type: "equal", + }, + _gt: { + type: "custom", + argument_type: { + type: "named", + name: "TimestampTz", + }, + }, + _lt: { + type: "custom", + argument_type: { + type: "named", + name: "TimestampTz", + }, + }, + _gte: { + type: "custom", + argument_type: { + type: "named", + name: "TimestampTz", + }, + }, + _lte: { + type: "custom", + argument_type: { + type: "named", + name: "TimestampTz", + }, + }, + _neq: { + type: "custom", + argument_type: { + type: "named", + name: "TimestampTz", + }, + }, + }, + }, Int: { + representation: { + type: "int64" + }, aggregate_functions: { // sum: { // result_type: { @@ -67,6 +257,9 @@ export const SCALAR_TYPES: { [key: string]: ScalarType } = { } }, Float: { + representation: { + type: "float64" + }, aggregate_functions: { // sum: { // result_type: { @@ -117,6 +310,9 @@ export const SCALAR_TYPES: { [key: string]: ScalarType } = { } }, String: { + representation: { + type: "string" + }, aggregate_functions: {}, comparison_operators: { _eq: { @@ -174,6 +370,9 @@ export const SCALAR_TYPES: { [key: string]: ScalarType } = { } }, Boolean: { + representation: { + type: "boolean" + }, aggregate_functions: {}, comparison_operators: { _eq: { diff --git a/src/handlers/query.ts b/src/handlers/query.ts index e0f400f..d812404 100644 --- a/src/handlers/query.ts +++ b/src/handlers/query.ts @@ -1,3 +1,5 @@ +//TODO: Aggregates https://github.com/hasura/ndc-duckduckapi/commit/29cacaddb811cbf0610a98a61d05a0d29da27554 + import { QueryRequest, QueryResponse, @@ -15,6 +17,77 @@ import { MAX_32_INT } from "../constants"; const escape_single = (s: any) => SqlString.escape(s); const escape_double = (s: any) => `"${SqlString.escape(s).slice(1, -1)}"`; + +function getColumnExpression(field_def: any, collection_alias: string, column: string): string { + // Helper function to handle the actual type + function handleNamedType(type: any): string { + if (type.name === "BigInt") { + return `CAST(${escape_double(collection_alias)}.${escape_double(column)} AS TEXT)`; + } + return `${escape_double(collection_alias)}.${escape_double(column)}`; + } + // Helper function to traverse the type structure + function processType(type: any): string { + if (type.type === "nullable") { + if (type.underlying_type.type === "named") { + return handleNamedType(type.underlying_type); + } else if (type.underlying_type.type === "array") { + // Handle array type + return processType(type.underlying_type); + } else { + return processType(type.underlying_type); + } + } else if (type.type === "array") { + // Handle array type + return processType(type.element_type); + } else if (type.type === "named") { + return handleNamedType(type); + } + // Default case + return `${escape_double(collection_alias)}.${escape_double(column)}`; + } + return processType(field_def.type); +} + +function isTimestampType(field_def: any): boolean { + if (!field_def) return false; + + function checkType(type: any): boolean { + if (type.type === "nullable") { + return checkType(type.underlying_type); + } + return type.type === "named" && type.name === "Timestamp"; + } + + return checkType(field_def.type); +} +function getIntegerType(field_def: any): string | null { + if (!field_def) return null; + + function checkType(type: any): string | null { + if (type.type === "nullable") { + return checkType(type.underlying_type); + } + if (type.type === "named") { + switch (type.name) { + case "BigInt": return "BIGINT"; + case "UBigInt": return "UBIGINT"; + case "HugeInt": return "HUGEINT"; + case "UHugeInt": return "UHUGEINT"; + default: return null; + } + } + return null; + } + + return checkType(field_def.type); +} + +function getRhsExpression(type: string | null): string { + if (!type) return "?"; + return `CAST(? AS ${type})`; +} + type QueryVariables = { [key: string]: any; }; @@ -96,7 +169,9 @@ function build_where( args: any[], variables: QueryVariables, prefix: string, - collection_aliases: {[k: string]: string} + collection_aliases: { [k: string]: string }, + config: Configuration, + query_request: QueryRequest ): string { let sql = ""; switch (expression.type) { @@ -112,6 +187,13 @@ function build_where( } break; case "binary_comparison_operator": + const object_type = config.config?.object_types[query_request.collection]; + const field_def = object_type?.fields[expression.column.name]; + const isTimestamp = isTimestampType(field_def); + const integerType = getIntegerType(field_def); + const type = isTimestamp ? "TIMESTAMP" : integerType; + const lhs = escape_double(expression.column.name); + const rhs = getRhsExpression(type); switch (expression.value.type) { case "scalar": args.push(expression.value.value); @@ -128,33 +210,33 @@ function build_where( } switch (expression.operator) { case "_eq": - sql = `${expression.column.name} = ?`; + sql = `${lhs} = ${rhs}`; break; case "_like": args[args.length - 1] = `%${args[args.length - 1]}%`; - sql = `${expression.column.name} LIKE ?`; + sql = `${lhs} LIKE ?`; break; case "_glob": - sql = `${expression.column.name} GLOB ?`; + sql = `${lhs} GLOB ?`; break; case "_neq": - sql = `${expression.column.name} != ?`; + sql = `${lhs} != ${rhs}`; break; case "_gt": - sql = `${expression.column.name} > ?`; + sql = `${lhs} > ${rhs}`; break; case "_lt": - sql = `${expression.column.name} < ?`; + sql = `${lhs} < ${rhs}`; break; case "_gte": - sql = `${expression.column.name} >= ?`; + sql = `${lhs} >= ${rhs}`; break; case "_lte": - sql = `${expression.column.name} <= ?`; + sql = `${lhs} <= ${rhs}`; break; default: throw new Forbidden( - "Binary Comparison Custom Operator not implemented", + `Binary Comparison Custom Operator ${expression.operator} not implemented`, {} ); } @@ -165,7 +247,7 @@ function build_where( } else { const clauses = []; for (const expr of expression.expressions) { - const res = build_where(expr, collection_relationships, args, variables, prefix, collection_aliases); + const res = build_where(expr, collection_relationships, args, variables, prefix, collection_aliases, config, query_request); clauses.push(res); } sql = `(${clauses.join(` AND `)})`; @@ -177,14 +259,14 @@ function build_where( } else { const clauses = []; for (const expr of expression.expressions) { - const res = build_where(expr, collection_relationships, args, variables, prefix, collection_aliases); + const res = build_where(expr, collection_relationships, args, variables, prefix, collection_aliases, config, query_request); clauses.push(res); } sql = `(${clauses.join(` OR `)})`; } break; case "not": - const not_result = build_where(expression.expression, collection_relationships, args, variables, prefix, collection_aliases); + const not_result = build_where(expression.expression, collection_relationships, args, variables, prefix, collection_aliases, config, query_request); sql = `NOT (${not_result})`; break; case "exists": @@ -198,7 +280,7 @@ function build_where( subquery_sql = ` SELECT 1 FROM ${from_collection_alias} AS ${escape_double(subquery_alias)} - WHERE ${predicate ? build_where(predicate, collection_relationships, args, variables, prefix, collection_aliases) : '1 = 1'} + WHERE ${predicate ? build_where(predicate, collection_relationships, args, variables, prefix, collection_aliases, config, query_request) : '1 = 1'} AND ${Object.entries(relationship.column_mapping).map(([from, to]) => { return `${escape_double(prefix)}.${escape_double(from)} = ${escape_double(subquery_alias)}.${escape_double(to)}`; }).join(" AND ")} @@ -256,7 +338,9 @@ function build_query( collect_rows.push(escape_single(field_name)); switch (field_value.type) { case "column": - collect_rows.push(`${escape_double(collection_alias)}.${escape_double(field_value.column)}`); + const object_type = config.config?.object_types[query_request.collection]; + const field_def = object_type.fields[field_name]; + collect_rows.push(getColumnExpression(field_def, collection_alias, field_value.column)); break; case "relationship": let relationship_collection = query_request.collection_relationships[field_value.relationship].target_collection; @@ -303,7 +387,7 @@ function build_query( const filter_joins: string[] = []; if (query.predicate) { - where_conditions.push(`(${build_where(query.predicate, query_request.collection_relationships, args, variables, collection_alias, config.config.collection_aliases)})`); + where_conditions.push(`(${build_where(query.predicate, query_request.collection_relationships, args, variables, collection_alias, config.config.collection_aliases, config, query_request)})`); } if (query.order_by && config.config) { diff --git a/src/handlers/schema.ts b/src/handlers/schema.ts index 8877082..b33c399 100644 --- a/src/handlers/schema.ts +++ b/src/handlers/schema.ts @@ -16,7 +16,8 @@ export function do_get_schema(configuration: Configuration): SchemaResponse { arguments: {}, type: cn, uniqueness_constraints: {}, - foreign_keys: {} + foreign_keys: {}, + description: object_types[cn].description }) } });