From a5fb595e043064208862b15cde4bd701dbfe985f Mon Sep 17 00:00:00 2001 From: Greg Richardson Date: Tue, 7 May 2024 20:19:59 -0600 Subject: [PATCH] feat(sql-to-rest): basic aggregate poc --- packages/sql-to-rest/src/processor.ts | 212 +++++++++++++----- .../sql-to-rest/src/renderers/http.test.ts | 120 +++++++++- .../src/renderers/supabase-js.test.ts | 4 +- packages/sql-to-rest/src/renderers/util.ts | 19 ++ packages/sql-to-rest/src/types/libpg-query.ts | 17 +- 5 files changed, 314 insertions(+), 58 deletions(-) diff --git a/packages/sql-to-rest/src/processor.ts b/packages/sql-to-rest/src/processor.ts index 188b6b4eca0..dba3e8c6335 100644 --- a/packages/sql-to-rest/src/processor.ts +++ b/packages/sql-to-rest/src/processor.ts @@ -6,12 +6,14 @@ import { ColumnRef, Field, FromExpression, + FuncCall, ParsedQuery, PgString, SelectResTarget, SelectStmt, SortBy, Stmt, + TypeCast, WhereExpression, } from './types/libpg-query' @@ -137,7 +139,19 @@ export type EmbeddedTarget = { flatten?: boolean } -export type Target = ColumnTarget | EmbeddedTarget +/** + * Represents a aggregate column target in the select. + */ +export type AggregateTarget = { + type: 'aggregate-target' + column: string + functionName: string + alias?: string + inputCast?: string + outputCast?: string +} + +export type Target = ColumnTarget | AggregateTarget | EmbeddedTarget export type Sort = { column: string @@ -209,7 +223,7 @@ function processSelectStmt(stmt: SelectStmt): Select { } if (stmt.SelectStmt.withClause) { - throw new UnsupportedError('WITH clauses are not supported') + throw new UnsupportedError('CTEs are not supported') } const [fromClause] = stmt.SelectStmt.fromClause @@ -405,51 +419,14 @@ function processTargetList( from: string, embeddedTargets: EmbeddedTarget[] ): Target[] { - const flattenedColumnTargets: ColumnTarget[] = targetList.map((resTarget) => { - let columnRef: ColumnRef - let cast: string | undefined + const flattenedColumnTargets: (ColumnTarget | AggregateTarget)[] = targetList.map((resTarget) => { + const target = processQueryTarget(resTarget.ResTarget.val) + target.alias = resTarget.ResTarget.name - if ('TypeCast' in resTarget.ResTarget.val) { - cast = resTarget.ResTarget.val.TypeCast.typeName.names - .map((name) => name.String.sval) - .join('.') - - columnRef = resTarget.ResTarget.val.TypeCast.arg - } else if ('ColumnRef' in resTarget.ResTarget.val) { - columnRef = resTarget.ResTarget.val - } else if ('A_Expr' in resTarget.ResTarget.val) { - try { - const column = renderJsonExpression(resTarget.ResTarget.val) - const alias = resTarget.ResTarget.name - - return { - type: 'column-target', - column, - alias, - cast, - } - } catch (err) { - throw new UnsupportedError(`Expressions not supported in select target`) - } - } else { - throw new UnsupportedError('Only columns allowed in select targets') - } - - const { fields } = columnRef.ColumnRef - - const column = renderFields(fields) - - const alias = resTarget.ResTarget.name - - return { - type: 'column-target', - column, - alias, - cast, - } + return target }) - // Transfer resource embedding columns and to `embeddedTargets` + // Transfer resource embedding columns to `embeddedTargets` const columnTargets = flattenedColumnTargets.filter((target) => { const qualifiedName = target.column.split('.') @@ -513,6 +490,91 @@ function processTargetList( return [...columnTargets, ...nestedEmbeddedTargets] } +function processQueryTarget( + queryTarget: TypeCast | ColumnRef | FuncCall | A_Expr +): ColumnTarget | AggregateTarget { + if ('TypeCast' in queryTarget) { + const cast = renderDataType(queryTarget.TypeCast.typeName.names) + + if ('A_Const' in queryTarget.TypeCast.arg) { + throw new UnsupportedError( + 'Only columns, JSON fields, and aggregates are supported as query targets' + ) + } + + const nestedTarget = processQueryTarget(queryTarget.TypeCast.arg) + + const { type } = nestedTarget + + if (type === 'aggregate-target') { + return { + ...nestedTarget, + outputCast: cast, + } + } else if (type === 'column-target') { + return { + ...nestedTarget, + cast, + } + } else { + throw new UnsupportedError(`Cannot process target with type '${type}'`) + } + } else if ('ColumnRef' in queryTarget) { + return { + type: 'column-target', + column: renderFields(queryTarget.ColumnRef.fields), + } + } else if ('A_Expr' in queryTarget) { + try { + return processJsonTarget(queryTarget) + } catch (err) { + const maybeJsonHint = + err instanceof Error && err.message === 'Invalid JSON path' + ? ' Did you forget to quote a JSON path?' + : '' + throw new UnsupportedError(`Expressions not supported as targets.${maybeJsonHint}`) + } + } else if ('FuncCall' in queryTarget) { + const functionName = renderFields(queryTarget.FuncCall.funcname) + const supportedAggregateFunctions = ['avg', 'count', 'max', 'min', 'sum'] + + if (!supportedAggregateFunctions.includes(functionName)) { + throw new UnsupportedError( + `Only the following aggregate functions are supported: ${JSON.stringify(supportedAggregateFunctions)}` + ) + } + + if (functionName !== 'count' && queryTarget.FuncCall.args.length === 0) { + throw new UnsupportedError(`Aggregate function '${functionName}' requires a column argument`) + } + + if (queryTarget.FuncCall.args.length > 1) { + throw new UnsupportedError(`Aggregate functions only accept one argument`) + } + + const [arg] = queryTarget.FuncCall.args + + const nestedTarget = processQueryTarget(arg) + + if (nestedTarget.type === 'aggregate-target') { + throw new UnsupportedError(`Aggregate functions cannot contain another function`) + } + + const { cast, ...columnTarget } = nestedTarget + + return { + ...columnTarget, + type: 'aggregate-target', + functionName, + inputCast: cast, + } + } else { + throw new UnsupportedError( + 'Only columns, JSON fields, and aggregates are supported as query targets' + ) + } +} + function processWhereClause(expression: WhereExpression): Filter { if ('A_Expr' in expression) { if ('TypeCast' in expression.A_Expr.lexpr) { @@ -523,7 +585,8 @@ function processWhereClause(expression: WhereExpression): Filter { if ('A_Expr' in expression.A_Expr.lexpr) { try { - column = renderJsonExpression(expression.A_Expr.lexpr) + const target = processJsonTarget(expression.A_Expr.lexpr) + column = target.column } catch (err) { throw new UnsupportedError(`Non column expressions not supported in WHERE clause`) } @@ -774,14 +837,45 @@ function renderFields(fields: Field[]) { } else if ('A_Star' in field) { return '*' } else { - const [fieldType] = Object.keys(field) - throw new UnsupportedError(`Unsupported ColumnRef field type '${fieldType}'`) + const [internalType] = Object.keys(field) + throw new UnsupportedError( + `Unsupported internal type '${internalType}' for data type names` + ) } }) .join('.') } -function renderJsonExpression(expression: A_Expr): string { +function renderDataType(names: PgString[]) { + const [first, ...rest] = names + + if (first.String.sval === 'pg_catalog' && rest.length === 1) { + const [name] = rest + + // The PG parser converts some data types, eg. int -> pg_catalog.int4 + // so we'll map those back + switch (name.String.sval) { + case 'int2': + return 'smallint' + case 'int4': + return 'int' + case 'int8': + return 'bigint' + case 'float8': + return 'float' + default: + return name.String.sval + } + } else if (rest.length > 0) { + throw new UnsupportedError( + `Casts can only reference data types by their unqualified name (not schema-qualified)` + ) + } else { + return first.String.sval + } +} + +function processJsonTarget(expression: A_Expr): ColumnTarget { if (expression.A_Expr.name.length > 1) { throw new UnsupportedError('Only one operator name supported per expression') } @@ -793,6 +887,7 @@ function renderJsonExpression(expression: A_Expr): string { throw new UnsupportedError(`Invalid JSON operator`) } + let cast: string | undefined = undefined let left: string let right: string @@ -803,7 +898,8 @@ function renderJsonExpression(expression: A_Expr): string { throw new UnsupportedError('Invalid JSON path') } } else if ('A_Expr' in expression.A_Expr.lexpr) { - left = renderJsonExpression(expression.A_Expr.lexpr) + const { column } = processJsonTarget(expression.A_Expr.lexpr) + left = column } else if ('ColumnRef' in expression.A_Expr.lexpr) { left = renderFields(expression.A_Expr.lexpr.ColumnRef.fields) } else { @@ -816,9 +912,25 @@ function renderJsonExpression(expression: A_Expr): string { } else { throw new UnsupportedError('Invalid JSON path') } + } else if ('TypeCast' in expression.A_Expr.rexpr) { + cast = renderDataType(expression.A_Expr.rexpr.TypeCast.typeName.names) + + if ('A_Const' in expression.A_Expr.rexpr.TypeCast.arg) { + if ('sval' in expression.A_Expr.rexpr.TypeCast.arg.A_Const) { + right = expression.A_Expr.rexpr.TypeCast.arg.A_Const.sval.sval + } else { + throw new UnsupportedError('Invalid JSON path') + } + } else { + throw new UnsupportedError('Invalid JSON path') + } } else { throw new UnsupportedError('Invalid JSON path') } - return `${left}${operator}${right}` + return { + type: 'column-target', + column: `${left}${operator}${right}`, + cast, + } } diff --git a/packages/sql-to-rest/src/renderers/http.test.ts b/packages/sql-to-rest/src/renderers/http.test.ts index d2e7199ef12..41c194fb3df 100644 --- a/packages/sql-to-rest/src/renderers/http.test.ts +++ b/packages/sql-to-rest/src/renderers/http.test.ts @@ -596,7 +596,7 @@ describe('select', () => { const { method, fullPath } = await renderHttp(statement) expect(method).toBe('GET') - expect(fullPath).toBe('/books?select=pages::pg_catalog.float8') + expect(fullPath).toBe('/books?select=pages::float') }) test('cast with alias', async () => { @@ -611,7 +611,7 @@ describe('select', () => { const { method, fullPath } = await renderHttp(statement) expect(method).toBe('GET') - expect(fullPath).toBe('/books?select=partialPages:pages::pg_catalog.float8') + expect(fullPath).toBe('/books?select=partialPages:pages::pg_catalog.float') }) test('cast in where clause fails', async () => { @@ -1017,6 +1017,21 @@ describe('select', () => { expect(fullPath).toBe('/books?select=address->city->>name') }) + test('select json column with cast', async () => { + const sql = stripIndents` + select + order_details->'tax_amount'::numeric + from + orders + ` + + const statement = await processSql(sql) + const { method, fullPath } = await renderHttp(statement) + + expect(method).toBe('GET') + expect(fullPath).toBe('/orders?select=order_details->tax_amount::numeric') + }) + test('filter by json column', async () => { const sql = stripIndents` select @@ -1033,4 +1048,105 @@ describe('select', () => { expect(method).toBe('GET') expect(fullPath).toBe('/books?select=address->city->>name&address->city->>code=eq.SFO') }) + + test('aggregate', async () => { + const sql = stripIndents` + select + sum(amount) + from + orders + ` + + const statement = await processSql(sql) + const { method, fullPath } = await renderHttp(statement) + + expect(method).toBe('GET') + expect(fullPath).toBe('/orders?select=amount.sum()') + }) + + test('aggregate with alias', async () => { + const sql = stripIndents` + select + sum(amount) as total + from + orders + ` + + const statement = await processSql(sql) + const { method, fullPath } = await renderHttp(statement) + + expect(method).toBe('GET') + expect(fullPath).toBe('/orders?select=total:amount.sum()') + }) + + test('aggregate with input cast', async () => { + const sql = stripIndents` + select + sum(amount::float) + from + orders + ` + + const statement = await processSql(sql) + const { method, fullPath } = await renderHttp(statement) + + expect(method).toBe('GET') + expect(fullPath).toBe('/orders?select=amount::float.sum()') + }) + + test('aggregate with output cast', async () => { + const sql = stripIndents` + select + sum(amount)::float + from + orders + ` + + const statement = await processSql(sql) + const { method, fullPath } = await renderHttp(statement) + + expect(method).toBe('GET') + expect(fullPath).toBe('/orders?select=amount.sum()::float') + }) + + test('aggregate with input and output cast', async () => { + const sql = stripIndents` + select + sum(amount::int)::float + from + orders + ` + + const statement = await processSql(sql) + const { method, fullPath } = await renderHttp(statement) + + expect(method).toBe('GET') + expect(fullPath).toBe('/orders?select=amount::int.sum()::float') + }) + + test('unsupported aggregate fails', async () => { + const sql = stripIndents` + select + custom_sum(amount::int)::float + from + orders + ` + + await expect(processSql(sql)).rejects.toThrowError() + }) + + test('aggregate on a json target', async () => { + const sql = stripIndents` + select + sum(order_details->'tax_amount'::numeric) + from + orders + ` + + const statement = await processSql(sql) + const { method, fullPath } = await renderHttp(statement) + + expect(method).toBe('GET') + expect(fullPath).toBe('/orders?select=order_details->tax_amount::numeric.sum()') + }) }) diff --git a/packages/sql-to-rest/src/renderers/supabase-js.test.ts b/packages/sql-to-rest/src/renderers/supabase-js.test.ts index 97463fbc419..f07740a50d4 100644 --- a/packages/sql-to-rest/src/renderers/supabase-js.test.ts +++ b/packages/sql-to-rest/src/renderers/supabase-js.test.ts @@ -706,7 +706,7 @@ describe('select', () => { expect(code).toBe(stripIndent` const { data, error } = await supabase .from('books') - .select('pages::pg_catalog.float8') + .select('pages::float') `) }) @@ -725,7 +725,7 @@ describe('select', () => { const { data, error } = await supabase .from('books') .select( - 'partialPages:pages::pg_catalog.float8', + 'partialPages:pages::float', ) `) }) diff --git a/packages/sql-to-rest/src/renderers/util.ts b/packages/sql-to-rest/src/renderers/util.ts index 3603e3f2c75..2ee0c88384e 100644 --- a/packages/sql-to-rest/src/renderers/util.ts +++ b/packages/sql-to-rest/src/renderers/util.ts @@ -24,6 +24,25 @@ export function renderTargets( value = `${indentation}${value}` + return value + } else if (target.type === 'aggregate-target') { + const { column, alias, functionName, inputCast, outputCast } = target + let value = column + + if (alias && alias !== column) { + value = `${alias}:${value}` + } + + if (inputCast) { + value = `${value}::${inputCast}` + } + + value = `${indentation}${value}.${functionName}()` + + if (outputCast) { + value = `${value}::${outputCast}` + } + return value } else if (target.type === 'embedded-target') { const { relation, alias, joinType, targets, flatten } = target diff --git a/packages/sql-to-rest/src/types/libpg-query.ts b/packages/sql-to-rest/src/types/libpg-query.ts index 1f4c3bd9838..922a455ed19 100644 --- a/packages/sql-to-rest/src/types/libpg-query.ts +++ b/packages/sql-to-rest/src/types/libpg-query.ts @@ -57,8 +57,8 @@ export type A_Expr = { | 'AEXPR_BETWEEN_SYM' | 'AEXPR_NOT_BETWEEN_SYM' name: PgString[] - lexpr: A_Const | A_Expr | ColumnRef - rexpr: A_Const | A_Expr | ColumnRef + lexpr: A_Const | A_Expr | ColumnRef | TypeCast + rexpr: A_Const | A_Expr | ColumnRef | TypeCast location: number } } @@ -95,7 +95,7 @@ export type ColumnRef = { export type TypeCast = { TypeCast: { - arg: ColumnRef + arg: ColumnRef | A_Const typeName: { names: PgString[] typemod: number @@ -105,6 +105,15 @@ export type TypeCast = { } } +export type FuncCall = { + FuncCall: { + funcname: PgString[] + args: (TypeCast | ColumnRef | FuncCall | A_Expr)[] + funcformat: string + location: number + } +} + /** * Result target, used in target list of pre-transformed parse trees (select, insert, update). * @@ -112,7 +121,7 @@ export type TypeCast = { */ export type SelectResTarget = { ResTarget: { - val: ColumnRef | TypeCast | A_Expr + val: ColumnRef | TypeCast | A_Expr | FuncCall name?: string location: number }