mirror of
https://github.com/supabase/supabase.git
synced 2026-10-10 20:05:06 +03:00
## Motivation When an Assistant tool needs a higher opt-in level than the org has, the user only sees the model say it has no access with (inconsistent) written instructions how to fix, instead of Assistant proactively facilitating the fix. <img width="400" alt="CleanShot 2026-10-07 at 3 44 29 PM@2x" src="https://github.com/user-attachments/assets/ba95568a-5db4-4d8c-b4ff-64855f2b87ec" /> The majority of recent labelled [Assistant Issues](https://www.braintrust.dev/app/supabase.io/p/Assistant/topics) in Braintrust are tools or query results blocked by the org opt-in level. We've considered making Assistant disabled entirely when data opt-in is off to eliminate the most common footgun (AI-405), but that's an intrusive change which could break legitimate use cases. ## Changes https://github.com/user-attachments/assets/206a2438-630e-4368-9cec-2b04b66c9cd4 A new `update_opt_in_level` tool renders an inline card which prompts admins to review the opt-in level and highlights the proposed change (non-admins see a message telling them to contact their admin to change the setting). Saving approves the call and the turn resumes at the new level. Results appear in a Braintrust tool span based on https://github.com/supabase/supabase/pull/45654. I added 5 evals that simulate the user answering the opt-in card to verify that the Assistant asks when blocked, continues after the user accepts, and doesn't invent data after a skip. Tool Usage and Correctness are at 100% in the [sample run](https://www.braintrust.dev/app/supabase.io/p/Dev%20(mattrossman%2FAssistant)/experiments/mattrossman%2Fai-158-prompt-users-to-update-settings-instead-of-showing-no-tool-1791483991). <details> <summary>📸 Screenshots</summary> | Admin, pending | Modal, Current and Proposed | | -- | -- | | <img width="100%" alt="CleanShot 2026-10-07 at 3 23 37 PM@2x" src="https://github.com/user-attachments/assets/2fc8ba11-0530-4028-a056-40ca02afd3ef" /> | <img width="1184" height="1754" alt="CleanShot 2026-10-07 at 3 32 37 PM@2x" src="https://github.com/user-attachments/assets/73639633-89ac-4d4c-b869-d9301735d5b9" /> | | **After saving, turn continues** | **Non-admin** | | <img width="100%" alt="CleanShot 2026-10-07 at 3 27 14 PM@2x" src="https://github.com/user-attachments/assets/a91c6760-6dfb-4f51-9ef8-4bdbe1b22495" /> | <img width="100%" alt="CleanShot 2026-10-07 at 3 15 33 PM@2x" src="https://github.com/user-attachments/assets/ecb9aa87-26dc-4b69-a086-cb77a28bb860" /> | </details> ## Verification To test in staging, set your org's Assistant opt-in level to Disabled, ask "What tables do I have?", and review the card. Try selecting the proposed (or different) opt-in level and saving to continue. [This trace](https://www.braintrust.dev/app/supabase.io/p/Dev%20(mattrossman%2FAssistant)/experiments/mattrossman%2Fai-158-prompt-users-to-update-settings-instead-of-showing-no-tool-1791483991?r=90066cd9-4da3-4817-9650-c369d2b42cea) is a sample from the accept eval case after saving Schema Only, note the `update_opt_in_level` span. ## Safety considerations I added a banner to the modal to make it more obvious that this setting impacts the whole org, not just the current chat or project: <img width="592" height="78" alt="CleanShot 2026-10-08 at 2 04 44 PM@2x" src="https://github.com/user-attachments/assets/6b140ed6-65fb-4bc0-b5cc-9c82fe2a67aa" /> I leave the current opt-in value selected by default in the form so the user has to consciously select the proposed value instead of mindlessly clicking save without understanding implications. `execute_sql` and `run_notebook` outputs are now stamped with the level they ran under, and history is sanitized by the least permissive of the stamped vs current opt-in levels. That way raising the level mid-chat doesn't unexpectedly expose earlier query rows to the model. Closes AI-158
188 lines
5.8 KiB
TypeScript
188 lines
5.8 KiB
TypeScript
import { untrustedSql } from '@supabase/pg-meta'
|
|
import dayjs from 'dayjs'
|
|
import isEqual from 'lodash/isEqual'
|
|
|
|
import { DEFAULT_CELL_ROW_LIMIT } from '@/components/interfaces/Explorer/QueryCell/QueryCell.utils'
|
|
import { type ExplorerQueryModel } from '@/components/interfaces/Explorer/QueryEditor'
|
|
import { type QueryDisplay, type QueryResult } from '@/components/interfaces/Explorer/types'
|
|
import { untrustedLogSql } from '@/data/logs/safe-analytics-sql'
|
|
import {
|
|
toQuerySourceBinding,
|
|
type QuerySourceBinding,
|
|
} from '@/data/query-sources/query-source-registry'
|
|
import { executeSqlOutputSchema } from '@/lib/ai/tools/tool-sanitizer'
|
|
|
|
export const DEFAULT_ASSISTANT_QUERY_TITLE = 'SQL query'
|
|
export const DEFAULT_ASSISTANT_LOGS_QUERY_TITLE = 'Logs query'
|
|
|
|
const TIME_COLUMN_RE =
|
|
/^(timestamp|time|date|hour|minute|day|week|month|year|ts|datetime|bucket|interval|period)$/i
|
|
const PREFERRED_Y_COLUMN_RE = /^(count|cnt|n|total|sum|avg|average|value|errors?|requests?)$/i
|
|
const SKIP_AS_DIMENSION_RE = /(message|sql|query|error|stack|body|payload|detail|hint)/i
|
|
const AGGREGATE_SQL_RE = /\b(group\s+by|(?:count|sum|avg|max|min)\s*\()/i
|
|
|
|
const EMPTY_CHART = {
|
|
type: 'bar' as const,
|
|
x_column: '',
|
|
y_series: [] as string[],
|
|
cumulative: false,
|
|
scale: 'linear' as const,
|
|
show_labels: false,
|
|
}
|
|
|
|
export function isChartableAssistantSql(sql: string): boolean {
|
|
const withoutComments = sql.replace(/--.*$/gm, ' ').replace(/\/\*[\s\S]*?\*\//g, ' ')
|
|
return AGGREGATE_SQL_RE.test(withoutComments)
|
|
}
|
|
|
|
export function getAssistantQueryDisplay({
|
|
view,
|
|
xAxis,
|
|
yAxis,
|
|
sql,
|
|
rows,
|
|
}: {
|
|
view?: 'table' | 'chart'
|
|
xAxis?: string
|
|
yAxis?: string
|
|
sql?: string
|
|
rows?: readonly Record<string, unknown>[]
|
|
}): QueryDisplay {
|
|
const hasChartAxes = Boolean(xAxis || yAxis)
|
|
|
|
if (hasChartAxes) {
|
|
return {
|
|
view: view ?? 'table',
|
|
chart: {
|
|
...EMPTY_CHART,
|
|
x_column: xAxis ?? '',
|
|
y_series: yAxis ? [yAxis] : [],
|
|
},
|
|
}
|
|
}
|
|
|
|
if (rows && rows.length > 0) {
|
|
const inferred = inferAssistantChartDisplay(rows)
|
|
return { ...inferred, view: view ?? inferred.view }
|
|
}
|
|
|
|
if (view) return { view, chart: undefined }
|
|
|
|
if (sql && isChartableAssistantSql(sql)) {
|
|
return { view: 'chart', chart: undefined }
|
|
}
|
|
|
|
return { view: 'table', chart: undefined }
|
|
}
|
|
|
|
export function inferAssistantChartDisplay(rows: readonly Record<string, unknown>[]): QueryDisplay {
|
|
if (rows.length === 0) return { view: 'table', chart: undefined }
|
|
|
|
const columns = Object.keys(rows[0] ?? {})
|
|
if (columns.length < 2) return { view: 'table', chart: undefined }
|
|
|
|
const sample = rows.slice(0, 20)
|
|
const numericColumns = columns.filter((column) => isNumericColumn(sample, column))
|
|
const timeColumn = columns.find((column) => isTimeLikeColumn(column, sample))
|
|
const xColumn =
|
|
timeColumn ??
|
|
columns.find(
|
|
(column) => !numericColumns.includes(column) && !SKIP_AS_DIMENSION_RE.test(column)
|
|
) ??
|
|
columns[0]
|
|
const yCandidates = numericColumns.filter((column) => column !== xColumn)
|
|
const yColumn = yCandidates.find((column) => PREFERRED_Y_COLUMN_RE.test(column)) ?? yCandidates[0]
|
|
|
|
if (!xColumn || !yColumn || SKIP_AS_DIMENSION_RE.test(xColumn)) {
|
|
return { view: 'table', chart: undefined }
|
|
}
|
|
|
|
return {
|
|
view: 'chart',
|
|
chart: {
|
|
...EMPTY_CHART,
|
|
type: timeColumn ? 'line' : 'bar',
|
|
x_column: xColumn,
|
|
y_series: [yColumn],
|
|
},
|
|
}
|
|
}
|
|
|
|
export function toAssistantQueryResult(output: unknown): QueryResult | undefined {
|
|
const rows = Array.isArray(output) ? output : executeSqlOutputSchema.safeParse(output).data?.rows
|
|
return rows ? { rows: rows.filter(isPlainRow) } : undefined
|
|
}
|
|
|
|
export function createAssistantQueryModel(
|
|
sql: string,
|
|
source: QuerySourceBinding = { _tag: 'database' }
|
|
): ExplorerQueryModel {
|
|
if (source._tag === 'logs') {
|
|
return { ...source, uncheckedSql: untrustedLogSql(sql) }
|
|
}
|
|
|
|
return {
|
|
...source,
|
|
uncheckedSql: untrustedSql(sql),
|
|
rowLimit: DEFAULT_CELL_ROW_LIMIT,
|
|
}
|
|
}
|
|
|
|
export function setAssistantQuerySql(query: ExplorerQueryModel, sql: string): ExplorerQueryModel {
|
|
if (query._tag === 'logs') {
|
|
return { ...query, uncheckedSql: untrustedLogSql(sql) }
|
|
}
|
|
|
|
return { ...query, uncheckedSql: untrustedSql(sql) }
|
|
}
|
|
|
|
export function changeAssistantQuerySource(
|
|
query: ExplorerQueryModel,
|
|
source: QuerySourceBinding
|
|
): ExplorerQueryModel {
|
|
if (source._tag === 'logs') {
|
|
return { ...source, uncheckedSql: untrustedLogSql(query.uncheckedSql) }
|
|
}
|
|
|
|
return {
|
|
...source,
|
|
uncheckedSql: untrustedSql(query.uncheckedSql),
|
|
rowLimit: query._tag === 'database' ? query.rowLimit : DEFAULT_CELL_ROW_LIMIT,
|
|
}
|
|
}
|
|
|
|
export function shouldClearAssistantQueryResult(
|
|
query: ExplorerQueryModel,
|
|
nextSource: QuerySourceBinding
|
|
): boolean {
|
|
return !isEqual(toQuerySourceBinding(query), toQuerySourceBinding(nextSource))
|
|
}
|
|
|
|
function isNumericValue(value: unknown): boolean {
|
|
if (typeof value === 'number') return Number.isFinite(value)
|
|
if (typeof value === 'bigint') return true
|
|
if (typeof value !== 'string' || value.trim().length === 0) return false
|
|
return Number.isFinite(Number(value))
|
|
}
|
|
|
|
function isNumericColumn(rows: readonly Record<string, unknown>[], column: string): boolean {
|
|
const values = rows.map((row) => row[column]).filter((value) => value != null)
|
|
return values.length > 0 && values.every(isNumericValue)
|
|
}
|
|
|
|
function isTimeLikeColumn(column: string, rows: readonly Record<string, unknown>[]): boolean {
|
|
if (TIME_COLUMN_RE.test(column)) return true
|
|
|
|
const values = rows.map((row) => row[column]).filter((value) => value != null)
|
|
if (values.length === 0) return false
|
|
|
|
return values.every((value) => {
|
|
if (typeof value !== 'string' || !/[-T:]/.test(value)) return false
|
|
return dayjs(value).isValid()
|
|
})
|
|
}
|
|
|
|
function isPlainRow(row: unknown): row is Record<string, unknown> {
|
|
return row !== null && typeof row === 'object' && !Array.isArray(row)
|
|
}
|