diff --git a/apps/studio/components/grid/SupabaseGrid.utils.ts b/apps/studio/components/grid/SupabaseGrid.utils.ts index b8997e3504e..b6401bbbb8b 100644 --- a/apps/studio/components/grid/SupabaseGrid.utils.ts +++ b/apps/studio/components/grid/SupabaseGrid.utils.ts @@ -5,13 +5,13 @@ import { CalculatedColumn, CellKeyboardEvent } from 'react-data-grid' import type { Filter, SavedState } from 'components/grid/types' import { Entity, isTableLike } from 'data/table-editor/table-editor-types' +import { useSearchParams } from 'next/navigation' +import { parseAsBoolean, parseAsNativeArrayOf, parseAsString, useQueryStates } from 'nuqs' import { copyToClipboard } from 'ui' import { FilterOperatorOptions } from './components/header/filter/Filter.constants' import { STORAGE_KEY_PREFIX } from './constants' import type { Sort, SupaColumn, SupaTable } from './types' import { formatClipboardValue } from './utils/common' -import { parseAsNativeArrayOf, parseAsBoolean, parseAsString, useQueryStates } from 'nuqs' -import { useSearchParams } from 'next/navigation' export const LOAD_TAB_FROM_CACHE_PARAM = 'loadFromCache' @@ -67,6 +67,7 @@ export function filtersToUrlParams(filters: Filter[]) { export function parseSupaTable(table: Entity): SupaTable { const columns = table.columns const primaryKeys = isTableLike(table) ? table.primary_keys : [] + const uniqueIndexes = isTableLike(table) ? table.unique_indexes : [] const relationships = isTableLike(table) ? table.relationships : [] const supaColumns: SupaColumn[] = columns.map((column) => { @@ -116,8 +117,14 @@ export function parseSupaTable(table: Entity): SupaTable { name: table.name, comment: table.comment, schema: table.schema, + type: table.entity_type, columns: supaColumns, estimateRowCount: isTableLike(table) ? table.live_rows_estimate : 0, + primaryKey: primaryKeys?.length > 0 ? primaryKeys.map((col) => col.name) : undefined, + uniqueIndexes: + !!uniqueIndexes && uniqueIndexes.length > 0 + ? uniqueIndexes.map(({ columns }) => columns) + : undefined, } } diff --git a/apps/studio/components/grid/components/header/ExportDialog.tsx b/apps/studio/components/grid/components/header/ExportDialog.tsx index 93aa9ef8ecc..2b8b1e40757 100644 --- a/apps/studio/components/grid/components/header/ExportDialog.tsx +++ b/apps/studio/components/grid/components/header/ExportDialog.tsx @@ -68,9 +68,9 @@ export const ExportDialog = ({ const queryChains = !table ? undefined : getAllTableRowsSql({ table, sorts, filters }) const query = !!queryChains ? ignoreRoleImpersonation - ? queryChains.toSql() + ? queryChains.sql.toSql() : wrapWithRoleImpersonation( - queryChains.toSql(), + queryChains.sql.toSql(), roleImpersonationState as RoleImpersonationState ) : '' diff --git a/apps/studio/components/grid/components/header/Header.tsx b/apps/studio/components/grid/components/header/Header.tsx index 95982e4d22f..e3e2178ea59 100644 --- a/apps/studio/components/grid/components/header/Header.tsx +++ b/apps/studio/components/grid/components/header/Header.tsx @@ -1,5 +1,4 @@ import { PermissionAction } from '@supabase/shared-types/out/constants' -import saveAs from 'file-saver' import { ArrowUp, ChevronDown, FileText, Trash } from 'lucide-react' import Link from 'next/link' import { ReactNode, useState } from 'react' @@ -10,9 +9,14 @@ import { useTableFilter } from 'components/grid/hooks/useTableFilter' import { useTableSort } from 'components/grid/hooks/useTableSort' import { GridHeaderActions } from 'components/interfaces/TableGridEditor/GridHeaderActions' import { formatTableRowsToSQL } from 'components/interfaces/TableGridEditor/TableEntity.utils' +import { + useExportAllRowsAsCsv, + useExportAllRowsAsJson, + useExportAllRowsAsSql, +} from 'components/layouts/TableEditorLayout/ExportAllRows' import { ButtonTooltip } from 'components/ui/ButtonTooltip' import { useTableRowsCountQuery } from 'data/table-rows/table-rows-count-query' -import { fetchAllTableRows, useTableRowsQuery } from 'data/table-rows/table-rows-query' +import { useTableRowsQuery } from 'data/table-rows/table-rows-query' import { useSendEventMutation } from 'data/telemetry/send-event-mutation' import { useAsyncCheckPermissions } from 'hooks/misc/useCheckPermissions' import { useSelectedOrganizationQuery } from 'hooks/misc/useSelectedOrganization' @@ -34,12 +38,12 @@ import { DropdownMenuItem, DropdownMenuTrigger, Separator, - SonnerProgress, } from 'ui' import { ExportDialog } from './ExportDialog' import { FilterPopover } from './filter/FilterPopover' import { formatRowsForCSV } from './Header.utils' import { SortPopover } from './sort/SortPopover' + // [Joshen] CSV exports require this guard as a fail-safe if the table is // just too large for a browser to keep all the rows in memory before // exporting. Either that or export as multiple CSV sheets with max n rows each @@ -312,170 +316,82 @@ const RowHeader = ({ tableQueriesEnabled = true }: RowHeaderProps) => { toast.success('Copied rows to clipboard') } + const exportParams = snap.allRowsSelected + ? ({ type: 'fetch_all', filters, sorts } as const) + : ({ + type: 'provided_rows', + table: snap.table, + rows: allRows.filter((x) => snap.selectedRows.has(x.idx)), + } as const) + + const { exportCsv, confirmationModal: exportCsvConfirmationModal } = useExportAllRowsAsCsv( + project + ? { + enabled: true, + projectRef: project.ref, + connectionString: project?.connectionString ?? null, + entity: snap.table, + totalRows, + ...exportParams, + } + : { enabled: false } + ) const onRowsExportCSV = async () => { setIsExporting(true) - if (snap.allRowsSelected && totalRows > MAX_EXPORT_ROW_COUNT) { - toast.error( -
{MAX_EXPORT_ROW_COUNT_MESSAGE}
- ) - return setIsExporting(false) - } - if (!project) { toast.error('Project is required') return setIsExporting(false) } - const toastId = snap.allRowsSelected - ? toast( - , - { - closeButton: false, - duration: Infinity, - } - ) - : toast.loading( - `Exporting ${snap.selectedRows.size} row${snap.selectedRows.size > 1 ? 's' : ''} from ${snap.table.name}` - ) + exportCsv() - const rows = snap.allRowsSelected - ? await fetchAllTableRows({ - projectRef: project.ref, - connectionString: project.connectionString, - table: snap.table, - filters, - sorts, - roleImpersonationState: roleImpersonationState as RoleImpersonationState, - progressCallback: (value: number) => { - const progress = Math.min((value / totalRows) * 100, 100) - toast( - , - { - id: toastId, - closeButton: false, - duration: Infinity, - } - ) - }, - }) - : allRows.filter((x) => snap.selectedRows.has(x.idx)) - - if (rows.length === 0) { - toast.dismiss(toastId) - toast.error('Export failed, please try exporting again') - setIsExporting(false) - return - } - - const csv = formatRowsForCSV({ - rows, - columns: snap.table!.columns.map((column) => column.name), - }) - const csvData = new Blob([csv], { type: 'text/csv;charset=utf-8;' }) - toast.success(`Downloaded ${rows.length} rows as CSV`, { - id: toastId, - closeButton: true, - duration: 4000, - }) - saveAs(csvData, `${snap.table!.name}_rows.csv`) setIsExporting(false) } + const { exportSql, confirmationModal: exportSqlConfirmationModal } = useExportAllRowsAsSql( + project + ? { + enabled: true, + projectRef: project.ref, + connectionString: project?.connectionString ?? null, + entity: snap.table, + ...exportParams, + } + : { enabled: false } + ) const onRowsExportSQL = async () => { setIsExporting(true) - if (snap.allRowsSelected && totalRows > MAX_EXPORT_ROW_COUNT) { - toast.error( -
{MAX_EXPORT_ROW_COUNT_MESSAGE}
- ) - return setIsExporting(false) - } - if (!project) { toast.error('Project is required') return setIsExporting(false) } - if (snap.allRowsSelected && totalRows === 0) { - toast.error('Export failed, please try exporting again') - return setIsExporting(false) - } + exportSql() - const toastId = snap.allRowsSelected - ? toast( - , - { - closeButton: false, - duration: Infinity, - } - ) - : toast.loading( - `Exporting ${snap.selectedRows.size} row${snap.selectedRows.size > 1 ? 's' : ''} from ${snap.table.name}` - ) - - const rows = snap.allRowsSelected - ? await fetchAllTableRows({ - projectRef: project.ref, - connectionString: project.connectionString, - table: snap.table, - filters, - sorts, - roleImpersonationState: roleImpersonationState as RoleImpersonationState, - progressCallback: (value: number) => { - const progress = Math.min((value / totalRows) * 100, 100) - toast( - , - { - id: toastId, - closeButton: false, - duration: Infinity, - } - ) - }, - }) - : allRows.filter((x) => snap.selectedRows.has(x.idx)) - - if (rows.length === 0) { - toast.error('Export failed, please exporting try again') - setIsExporting(false) - return - } - - const sqlStatements = formatTableRowsToSQL(snap.table, rows) - const sqlData = new Blob([sqlStatements], { type: 'text/sql;charset=utf-8;' }) - toast.success(`Downloading ${rows.length} rows as SQL`, { - id: toastId, - closeButton: true, - duration: 4000, - }) - saveAs(sqlData, `${snap.table!.name}_rows.sql`) setIsExporting(false) } + const { exportJson, confirmationModal: exportJsonConfirmationModal } = useExportAllRowsAsJson( + project + ? { + enabled: true, + projectRef: project.ref, + connectionString: project?.connectionString ?? null, + entity: snap.table, + ...exportParams, + } + : { enabled: false } + ) const onRowsExportJSON = async () => { if (!project) { return toast.error('Project is required') } setIsExporting(true) - const toastId = toast.loading( - `Exporting ${snap.selectedRows.size} row${snap.selectedRows.size > 1 ? 's' : ''} from ${snap.table.name}` - ) - const rows = allRows.filter((x) => snap.selectedRows.has(x.idx)) - const sqlData = new Blob([JSON.stringify(rows)], { type: 'text/sql;charset=utf-8;' }) - toast.success(`Downloading ${rows.length} rows as JSON`, { - id: toastId, - closeButton: true, - duration: 4000, - }) - saveAs(sqlData, `${snap.table!.name}_rows.json`) + + exportJson() setIsExporting(false) } @@ -600,6 +516,10 @@ const RowHeader = ({ tableQueriesEnabled = true }: RowHeaderProps) => { open={showExportModal} onOpenChange={() => setShowExportModal(false)} /> + + {exportCsvConfirmationModal} + {exportSqlConfirmationModal} + {exportJsonConfirmationModal} ) } diff --git a/apps/studio/components/grid/types/table.ts b/apps/studio/components/grid/types/table.ts index 8af19c6530b..b9179c00257 100644 --- a/apps/studio/components/grid/types/table.ts +++ b/apps/studio/components/grid/types/table.ts @@ -1,5 +1,6 @@ -import { GridForeignKey } from './base' +import type { ENTITY_TYPE } from 'data/entity-types/entity-type-constants' import type { Dictionary } from 'types' +import { GridForeignKey } from './base' export interface SupaColumn { readonly dataType: string @@ -20,11 +21,14 @@ export interface SupaColumn { export interface SupaTable { readonly id: number + readonly type: ENTITY_TYPE readonly columns: SupaColumn[] readonly name: string readonly schema?: string | null readonly comment?: string | null readonly estimateRowCount: number + readonly primaryKey?: string[] + readonly uniqueIndexes?: string[][] } export interface SupaRow extends Dictionary { diff --git a/apps/studio/components/interfaces/TableGridEditor/TableEntity.utils.test.ts b/apps/studio/components/interfaces/TableGridEditor/TableEntity.utils.test.ts index 9a52aa4795a..24b0133b77b 100644 --- a/apps/studio/components/interfaces/TableGridEditor/TableEntity.utils.test.ts +++ b/apps/studio/components/interfaces/TableGridEditor/TableEntity.utils.test.ts @@ -1,10 +1,13 @@ -import { describe, it, expect } from 'vitest' +import type { SupaTable } from 'components/grid/types' +import { ENTITY_TYPE } from 'data/entity-types/entity-type-constants' +import { describe, expect, it } from 'vitest' import { formatTableRowsToSQL } from './TableEntity.utils' describe('TableEntity.utils: formatTableRowsToSQL', () => { it('should format rows into a single SQL INSERT statement', () => { - const table = { + const table: SupaTable = { id: 1, + type: ENTITY_TYPE.TABLE, columns: [ { name: 'id', dataType: 'bigint', format: 'int8', position: 0 }, { name: 'name', dataType: 'text', format: 'text', position: 1 }, @@ -26,8 +29,9 @@ describe('TableEntity.utils: formatTableRowsToSQL', () => { }) it('should not stringify null values', () => { - const table = { + const table: SupaTable = { id: 1, + type: ENTITY_TYPE.TABLE, columns: [ { name: 'id', dataType: 'bigint', format: 'int8', position: 0 }, { name: 'name', dataType: 'text', format: 'text', position: 1 }, @@ -49,8 +53,9 @@ describe('TableEntity.utils: formatTableRowsToSQL', () => { }) it('should handle PG JSON and array columns', () => { - const table = { + const table: SupaTable = { id: 1, + type: ENTITY_TYPE.TABLE, columns: [ { name: 'id', dataType: 'bigint', format: 'int8', position: 0 }, { name: 'name', dataType: 'text', format: 'text', position: 1 }, @@ -79,13 +84,14 @@ describe('TableEntity.utils: formatTableRowsToSQL', () => { }, ] const result = formatTableRowsToSQL(table, rows) - const expected = `INSERT INTO "public"."demo" ("id", "name", "tags", "metadata") VALUES ('2', 'Person 1', '{"tag-a","tag-c"}', '{"version": 1}'), ('3', 'ONeil', '{"tag-a"}', '{"version": 1, "name": "O''Neil"}');` + const expected = `INSERT INTO "public"."demo" ("id", "name", "tags", "metadata") VALUES ('2', 'Person 1', ARRAY["tag-a","tag-c"], '{"version": 1}'), ('3', 'ONeil', ARRAY["tag-a"], '{"version": 1, "name": "O''Neil"}');` expect(result).toBe(expected) }) it('should return an empty string for empty rows', () => { - const table = { + const table: SupaTable = { id: 1, + type: ENTITY_TYPE.TABLE, columns: [ { name: 'id', dataType: 'bigint', format: 'int8', position: 0 }, { name: 'name', dataType: 'text', format: 'text', position: 1 }, @@ -100,8 +106,9 @@ describe('TableEntity.utils: formatTableRowsToSQL', () => { }) it('should remove the idx property', () => { - const table = { + const table: SupaTable = { id: 1, + type: ENTITY_TYPE.TABLE, columns: [ { name: 'id', dataType: 'bigint', format: 'int8', position: 0 }, { name: 'name', dataType: 'text', format: 'text', position: 1 }, diff --git a/apps/studio/components/interfaces/TableGridEditor/TableEntity.utils.ts b/apps/studio/components/interfaces/TableGridEditor/TableEntity.utils.ts index 7d208f3b351..c3fa6d403ca 100644 --- a/apps/studio/components/interfaces/TableGridEditor/TableEntity.utils.ts +++ b/apps/studio/components/interfaces/TableGridEditor/TableEntity.utils.ts @@ -45,7 +45,8 @@ export const formatTableRowsToSQL = (table: SupaTable, rows: any[]) => { if (val === null) { return 'null' } else if (dataType === 'ARRAY') { - return `'${JSON.stringify(val).replace('[', '{').replace(/.$/, '}')}'` + const array = Array.isArray(val) ? val : JSON.parse(val as string) + return `${formatArrayForSql(array as unknown[])}` } else if (format?.includes('json')) { return `${JSON.stringify(val).replace(/\\"/g, '"').replace(/'/g, "''").replace('"', "'").replace(/.$/, "'")}` } else if ( @@ -65,3 +66,60 @@ export const formatTableRowsToSQL = (table: SupaTable, rows: any[]) => { return `INSERT INTO "${table.schema}"."${table.name}" (${columns}) VALUES ${valuesSets};` } + +/** + * Generate a random tag for dollar-quoting of SQL strings + * + * @return A random tag in the format `$tag$` + */ +const generateRandomTag = (): `$${string}$` => { + const inner = Math.random().toString(36).substring(2, 15) + // Ensure the tag starts with a character not a digit to avoid conflicts with + // Postgres parameter syntax + return `$x${inner}$` +} + +/** + * Wrap a string in dollar-quote tags, ensuring the tag does not appear in the string + * + * @throws Error if unable to generate a unique tag after multiple attempts + */ +const safeDollarQuote = (str: string): string => { + let tag = generateRandomTag() + + let attempts = 0 + const maxAttempts = 100 + while (str.includes(tag)) { + if (attempts >= maxAttempts) { + throw new Error('Unable to generate a unique dollar-quote tag after multiple attempts.') + } + + attempts++ + tag = generateRandomTag() + } + return `${tag}${str}${tag}` +} + +const formatArrayForSql = (arr: unknown[]): string => { + let result = 'ARRAY[' + + arr.forEach((item, index) => { + if (Array.isArray(item)) { + result += formatArrayForSql(item) + } else if (typeof item === 'string') { + result += `"${item.replace(/"/g, '""')}"` + } else if (!!item && typeof item === 'object') { + result += `${safeDollarQuote(JSON.stringify(item))}::json` + } else { + result += `${item}` + } + + if (index < arr.length - 1) { + result += ',' + } + }) + + result += ']' + + return result +} diff --git a/apps/studio/components/layouts/TableEditorLayout/EntityListItem.tsx b/apps/studio/components/layouts/TableEditorLayout/EntityListItem.tsx index 9d34cf2652b..90676a4c35d 100644 --- a/apps/studio/components/layouts/TableEditorLayout/EntityListItem.tsx +++ b/apps/studio/components/layouts/TableEditorLayout/EntityListItem.tsx @@ -1,19 +1,12 @@ -import saveAs from 'file-saver' import { Copy, Download, Edit, Lock, MoreVertical, Trash } from 'lucide-react' import Link from 'next/link' -import Papa from 'papaparse' +import { type CSSProperties } from 'react' import { toast } from 'sonner' -import { IS_PLATFORM, useParams } from 'common' -import { - MAX_EXPORT_ROW_COUNT, - MAX_EXPORT_ROW_COUNT_MESSAGE, -} from 'components/grid/components/header/Header' -import { LOAD_TAB_FROM_CACHE_PARAM, parseSupaTable } from 'components/grid/SupabaseGrid.utils' -import { - formatTableRowsToSQL, - getEntityLintDetails, -} from 'components/interfaces/TableGridEditor/TableEntity.utils' +import { useParams } from 'common' +import { useTableFilter } from 'components/grid/hooks/useTableFilter' +import { LOAD_TAB_FROM_CACHE_PARAM } from 'components/grid/SupabaseGrid.utils' +import { getEntityLintDetails } from 'components/interfaces/TableGridEditor/TableEntity.utils' import { EntityTypeIcon } from 'components/ui/EntityTypeIcon' import { InlineLink } from 'components/ui/InlineLink' import { getTableDefinition } from 'data/database/table-definition-query' @@ -21,13 +14,14 @@ import { ENTITY_TYPE } from 'data/entity-types/entity-type-constants' import { Entity } from 'data/entity-types/entity-types-infinite-query' import { useProjectLintsQuery } from 'data/lint/lint-query' import { EditorTablePageLink } from 'data/prefetchers/project.$ref.editor.$id' -import { getTableEditor } from 'data/table-editor/table-editor-query' -import { isTableLike } from 'data/table-editor/table-editor-types' -import { fetchAllTableRows } from 'data/table-rows/table-rows-query' +import { useTableRowsCountQuery } from 'data/table-rows/table-rows-count-query' import { useQuerySchemaState } from 'hooks/misc/useSchemaQueryState' import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject' import { formatSql } from 'lib/formatSql' -import type { CSSProperties } from 'react' +import { + useRoleImpersonationStateSnapshot, + type RoleImpersonationState, +} from 'state/role-impersonation-state' import { useTableEditorStateSnapshot } from 'state/table-editor' import { createTabId, useTabsStateSnapshot } from 'state/tabs' import { @@ -48,6 +42,7 @@ import { TooltipTrigger, TreeViewItemVariant, } from 'ui' +import { useExportAllRowsAsCsv, useExportAllRowsAsSql } from './ExportAllRows' export interface EntityListItemProps { id: number | string @@ -85,6 +80,23 @@ const EntityListItem = ({ const isActive = Number(id) === entity.id const canEdit = isActive && !isLocked + const { filters } = useTableFilter() + const roleImpersonationState = useRoleImpersonationStateSnapshot() + const { data: countData } = useTableRowsCountQuery( + { + projectRef, + connectionString: project?.connectionString, + tableId: entity.id, + filters, + enforceExactCount: false, + roleImpersonationState: roleImpersonationState as RoleImpersonationState, + }, + { + enabled: isTableLikeEntityListItem(entity) && isActive, + } + ) + const rowCount = countData?.count + const { data: lints = [] } = useProjectLintsQuery({ projectRef: project?.ref, }) @@ -132,112 +144,23 @@ const EntityListItem = ({ return text.charAt(0).toUpperCase() + text.slice(1) } - const exportTableAsCSV = async () => { - if (IS_PLATFORM && !project?.connectionString) { - return console.error('Connection string is required') - } - const toastId = toast.loading(`Exporting ${entity.name} as CSV...`) + const { exportCsv, confirmationModal: exportCsvConfirmationModal } = useExportAllRowsAsCsv({ + enabled: true, + projectRef, + connectionString: project?.connectionString ?? null, + entity, + type: 'fetch_all', + totalRows: rowCount, + }) - try { - const table = await getTableEditor({ - id: entity.id, - projectRef, - connectionString: project?.connectionString, - }) - if (isTableLike(table) && table.live_rows_estimate > MAX_EXPORT_ROW_COUNT) { - return toast.error( -
{MAX_EXPORT_ROW_COUNT_MESSAGE}
, - { id: toastId } - ) - } - - const supaTable = table && parseSupaTable(table) - - if (!supaTable) { - return toast.error(`Failed to export table: ${entity.name}`, { id: toastId }) - } - - const rows = await fetchAllTableRows({ - projectRef, - connectionString: project?.connectionString, - table: supaTable, - }) - const formattedRows = rows.map((row) => { - const formattedRow = row - Object.keys(row).map((column) => { - if (typeof row[column] === 'object' && row[column] !== null) - formattedRow[column] = JSON.stringify(formattedRow[column]) - }) - return formattedRow - }) - - if (formattedRows.length > 0) { - const csv = Papa.unparse(formattedRows, { - columns: supaTable.columns.map((column) => column.name), - }) - const csvData = new Blob([csv], { type: 'text/csv;charset=utf-8;' }) - saveAs(csvData, `${entity!.name}_rows.csv`) - } - - toast.success(`Successfully exported ${entity.name} as CSV`, { id: toastId }) - } catch (error: any) { - toast.error(`Failed to export table: ${error.message}`, { id: toastId }) - } - } - - const exportTableAsSQL = async () => { - if (IS_PLATFORM && !project?.connectionString) { - return console.error('Connection string is required') - } - const toastId = toast.loading(`Exporting ${entity.name} as SQL...`) - - try { - const table = await getTableEditor({ - id: entity.id, - projectRef, - connectionString: project?.connectionString, - }) - - if (isTableLike(table) && table.live_rows_estimate > MAX_EXPORT_ROW_COUNT) { - return toast.error( -
{MAX_EXPORT_ROW_COUNT_MESSAGE}
, - { id: toastId } - ) - } - - const supaTable = table && parseSupaTable(table) - - if (!supaTable) { - return toast.error(`Failed to export table: ${entity.name}`, { id: toastId }) - } - - const rows = await fetchAllTableRows({ - projectRef, - connectionString: project?.connectionString, - table: supaTable, - }) - - const formattedRows = rows.map((row) => { - const formattedRow = { ...row } - Object.keys(row).forEach((column) => { - if (typeof row[column] === 'object' && row[column] !== null) { - formattedRow[column] = JSON.stringify(row[column]) - } - }) - return formattedRow - }) - - if (formattedRows.length > 0) { - const sqlStatements = formatTableRowsToSQL(supaTable, formattedRows) - const sqlData = new Blob([sqlStatements], { type: 'text/sql;charset=utf-8;' }) - saveAs(sqlData, `${entity!.name}_rows.sql`) - } - - toast.success(`Successfully exported ${entity.name} as SQL`, { id: toastId }) - } catch (error: any) { - toast.error(`Failed to export table: ${error.message}`, { id: toastId }) - } - } + const { exportSql, confirmationModal: exportSqlConfirmationModal } = useExportAllRowsAsSql({ + enabled: true, + projectRef, + connectionString: project?.connectionString ?? null, + entity, + type: 'fetch_all', + totalRows: rowCount, + }) return ( { e.stopPropagation() - exportTableAsCSV() + exportCsv() }} > Export table as CSV @@ -410,7 +333,7 @@ const EntityListItem = ({ className="gap-x-2" onClick={(e) => { e.stopPropagation() - exportTableAsSQL() + exportSql() }} > Export table as SQL @@ -446,6 +369,8 @@ const EntityListItem = ({ )} + {exportCsvConfirmationModal} + {exportSqlConfirmationModal} ) } diff --git a/apps/studio/components/layouts/TableEditorLayout/ExportAllRows.errors.ts b/apps/studio/components/layouts/TableEditorLayout/ExportAllRows.errors.ts new file mode 100644 index 00000000000..73d0e0065a9 --- /dev/null +++ b/apps/studio/components/layouts/TableEditorLayout/ExportAllRows.errors.ts @@ -0,0 +1,93 @@ +const normalizeCauseAsError = (cause: unknown): Error | undefined => { + if (!cause) return undefined + + if (cause instanceof Error) { + return cause + } + + if (typeof cause === 'object' && 'message' in cause) { + return new Error(String(cause.message)) + } + + return new Error(String(cause)) +} + +export class ExportAllRowsErrorFamily extends Error { + cause?: Error + + constructor(message: string, options: { cause?: unknown } = {}) { + super(message, options) + } +} + +export class NoConnectionStringError extends ExportAllRowsErrorFamily { + constructor() { + super('No connection string provided for database connection.') + this.name = 'NoConnectionStringError' + } +} + +export class TableDetailsFetchError extends ExportAllRowsErrorFamily { + constructor(tableName: string, _cause?: unknown) { + const cause = normalizeCauseAsError(_cause) + super(`Failed to fetch table details from the database for table ${tableName}.`, { cause }) + this.name = 'TableDetailsFetchError' + } +} + +export class NoTableError extends ExportAllRowsErrorFamily { + constructor(tableName: string) { + super(`The specified table "${tableName}" does not exist in the database.`) + this.name = 'NoTableError' + } +} + +export class NoRowsToExportError extends ExportAllRowsErrorFamily { + constructor(tableName: string) { + super(`There are no rows to export from the table "${tableName}".`) + this.name = 'NoRowsToExportError' + } +} + +export class TableTooLargeError extends ExportAllRowsErrorFamily { + constructor(tableName: string, rowCount: number, maxAllowed: number) { + super( + `The table "${tableName}" has ${rowCount} rows, which exceeds the maximum allowed limit of ${maxAllowed} rows for export.` + ) + this.name = 'TableTooLargeError' + } +} + +export class FetchRowsError extends ExportAllRowsErrorFamily { + constructor(tableName: string, _cause?: unknown) { + const cause = normalizeCauseAsError(_cause) + super(`An error occurred while fetching rows from the table "${tableName}".`, { cause }) + this.name = 'FetchRowsError' + } +} + +export class OutputConversionError extends ExportAllRowsErrorFamily { + constructor(_cause?: unknown) { + const cause = normalizeCauseAsError(_cause) + super('Failed to convert the fetched rows into the desired output format.', { + cause, + }) + this.name = 'OutputConversionError' + } +} + +export class BlobCreationError extends ExportAllRowsErrorFamily { + constructor(_cause?: unknown) { + const cause = normalizeCauseAsError(_cause) + super('An error occurred while creating a Blob for the exported data.', { cause }) + this.name = 'BlobCreationError' + } +} + +export class DownloadSaveError extends ExportAllRowsErrorFamily { + constructor(_cause?: unknown) { + const cause = normalizeCauseAsError(_cause) + super('An error occurred while saving the exported data to a file.', { cause }) + this.name = 'DownloadSaveError' + } +} diff --git a/apps/studio/components/layouts/TableEditorLayout/ExportAllRows.progress.tsx b/apps/studio/components/layouts/TableEditorLayout/ExportAllRows.progress.tsx new file mode 100644 index 00000000000..b83ded966dd --- /dev/null +++ b/apps/studio/components/layouts/TableEditorLayout/ExportAllRows.progress.tsx @@ -0,0 +1,104 @@ +import { useCallback, useRef, type ReactNode } from 'react' +import { toast } from 'sonner' +import { SonnerProgress } from 'ui' + +export const useProgressToasts = () => { + const toastIdsRef = useRef(new Map()) + + const startProgressTracker = useCallback( + ({ + id, + name, + trackPercentage = false, + }: { + id: number + name: string + trackPercentage?: boolean + }) => { + if (toastIdsRef.current.has(id)) return + + if (trackPercentage) { + toastIdsRef.current.set( + id, + toast(, { + closeButton: false, + duration: Infinity, + }) + ) + } else { + toastIdsRef.current.set(id, toast.loading(`Exporting ${name}...`)) + } + }, + [] + ) + + const trackPercentageProgress = useCallback( + ({ + id, + name, + value, + totalRows, + }: { + id: number + name: string + value: number + totalRows: number + }) => { + const savedToastId = toastIdsRef.current.get(id) + + const progress = Math.min((value / totalRows) * 100, 100) + const newToastId = toast( + , + { + id: savedToastId, + closeButton: false, + duration: Infinity, + } + ) + + if (!savedToastId) toastIdsRef.current.set(id, newToastId) + }, + [] + ) + + const stopTrackerWithError = useCallback( + (id: number, name: string, customMessage?: ReactNode) => { + const savedToastId = toastIdsRef.current.get(id) + if (savedToastId) { + toast.dismiss(savedToastId) + toastIdsRef.current.delete(id) + } + + toast.error(customMessage ?? `There was an error exporting ${name}`) + }, + [] + ) + + const dismissTrackerSilently = useCallback((id: number) => { + const savedToastId = toastIdsRef.current.get(id) + if (savedToastId) { + toast.dismiss(savedToastId) + toastIdsRef.current.delete(id) + } + }, []) + + const markTrackerComplete = useCallback((id: number, totalRows: number) => { + const savedToastId = toastIdsRef.current.get(id) + const deleteSavedToastId = () => toastIdsRef.current.delete(id) + + toast.success(`Successfully exported ${totalRows} rows`, { + id: savedToastId, + duration: 4000, + onAutoClose: deleteSavedToastId, + onDismiss: deleteSavedToastId, + }) + }, []) + + return { + startProgressTracker, + trackPercentageProgress, + stopTrackerWithError, + dismissTrackerSilently, + markTrackerComplete, + } +} diff --git a/apps/studio/components/layouts/TableEditorLayout/ExportAllRows.tsx b/apps/studio/components/layouts/TableEditorLayout/ExportAllRows.tsx new file mode 100644 index 00000000000..57a2e13578c --- /dev/null +++ b/apps/studio/components/layouts/TableEditorLayout/ExportAllRows.tsx @@ -0,0 +1,436 @@ +import saveAs from 'file-saver' +import Papa from 'papaparse' +import { useCallback, useState, type ReactNode } from 'react' + +import { useQueryClient, type QueryClient } from '@tanstack/react-query' +import { IS_PLATFORM } from 'common' +import { + MAX_EXPORT_ROW_COUNT, + MAX_EXPORT_ROW_COUNT_MESSAGE, +} from 'components/grid/components/header/Header' +import { parseSupaTable } from 'components/grid/SupabaseGrid.utils' +import type { Filter, Sort, SupaTable } from 'components/grid/types' +import { formatTableRowsToSQL } from 'components/interfaces/TableGridEditor/TableEntity.utils' +import { ENTITY_TYPE } from 'data/entity-types/entity-type-constants' +import type { Entity } from 'data/entity-types/entity-types-infinite-query' +import { tableEditorKeys } from 'data/table-editor/keys' +import { getTableEditor, type TableEditorData } from 'data/table-editor/table-editor-query' +import { isTableLike } from 'data/table-editor/table-editor-types' +import { fetchAllTableRows } from 'data/table-rows/table-rows-query' +import { useStaticEffectEvent } from 'hooks/useStaticEffectEvent' +import type { RoleImpersonationState } from 'lib/role-impersonation' +import { ConfirmationModal } from 'ui-patterns/Dialogs/ConfirmationModal' +import { + BlobCreationError, + DownloadSaveError, + FetchRowsError, + NoConnectionStringError, + NoRowsToExportError, + NoTableError, + OutputConversionError, + TableDetailsFetchError, + TableTooLargeError, + type ExportAllRowsErrorFamily, +} from './ExportAllRows.errors' +import { useProgressToasts } from './ExportAllRows.progress' + +type OutputCallbacks = { + convertToOutputFormat: (formattedRows: Record[], table: SupaTable) => string + convertToBlob: (str: string) => Blob + save: (blob: Blob, table: SupaTable) => void +} + +type FetchAllRowsParams = { + queryClient: QueryClient + projectRef: string + connectionString: string | null + entity: Pick + bypassConfirmation: boolean + filters?: Filter[] + sorts?: Sort[] + roleImpersonationState?: RoleImpersonationState + startCallback?: () => void + progressCallback?: (progress: number) => void +} & OutputCallbacks + +type FetchAllRowsReturn = + | { status: 'require_confirmation'; reason: string } + | { status: 'error'; error: ExportAllRowsErrorFamily } + | { status: 'success'; rowsExported: number } + +const fetchAllRows = async ({ + queryClient, + projectRef, + connectionString, + entity, + bypassConfirmation, + filters, + sorts, + roleImpersonationState, + startCallback, + progressCallback, + convertToOutputFormat, + convertToBlob, + save, +}: FetchAllRowsParams): Promise => { + if (IS_PLATFORM && !connectionString) { + return { status: 'error', error: new NoConnectionStringError() } + } + + let table: TableEditorData | undefined + try { + table = await queryClient.ensureQueryData({ + // Query is the same even if connectionString changes + // eslint-disable-next-line @tanstack/query/exhaustive-deps + queryKey: tableEditorKeys.tableEditor(projectRef, entity.id), + queryFn: ({ signal }) => + getTableEditor({ projectRef, connectionString, id: entity.id }, signal), + }) + } catch (error: unknown) { + return { status: 'error', error: new TableDetailsFetchError(entity.name, error) } + } + + if (!table) { + return { status: 'error', error: new NoTableError(entity.name) } + } + + const type = table.entity_type + if (type === ENTITY_TYPE.VIEW && !bypassConfirmation) { + return { + status: 'require_confirmation', + reason: `Exporting a view may cause consistency issues or performance issues on very large views. If possible, we recommend exporting the underlying table instead.`, + } + } else if (type === ENTITY_TYPE.MATERIALIZED_VIEW && !bypassConfirmation) { + return { + status: 'require_confirmation', + reason: `Exporting a materialized view may cause performance issues on very large views. If possible, we recommend exporting the underlying table instead.`, + } + } else if (type === ENTITY_TYPE.FOREIGN_TABLE && !bypassConfirmation) { + return { + status: 'require_confirmation', + reason: `Exporting a foreign table may cause consistency issues or performance issues on very large tables.`, + } + } + + if (isTableLike(table) && table.live_rows_estimate > MAX_EXPORT_ROW_COUNT) { + return { + status: 'error', + error: new TableTooLargeError(table.name, table.live_rows_estimate, MAX_EXPORT_ROW_COUNT), + } + } + + const supaTable = parseSupaTable(table) + + const primaryKey = supaTable.primaryKey + if (!primaryKey && !bypassConfirmation) { + return { + status: 'require_confirmation', + reason: `This table does not have a primary key defined, which may cause performance issues when exporting very large tables.`, + } + } + + startCallback?.() + + let rows: Record[] + try { + rows = await fetchAllTableRows({ + projectRef, + connectionString, + table: supaTable, + filters, + sorts, + roleImpersonationState, + progressCallback, + }) + } catch (error: unknown) { + return { status: 'error', error: new FetchRowsError(supaTable.name, error) } + } + + if (rows.length === 0) { + return { status: 'error', error: new NoRowsToExportError(entity.name) } + } + const formattedRows = formatRowsForExport(rows, supaTable) + + return convertAndDownload(formattedRows, supaTable, { + convertToOutputFormat, + convertToBlob, + save, + }) +} + +const formatRowsForExport = (rows: Record[], table: SupaTable) => { + return rows.map((row) => { + const formattedRow = { ...row } + Object.keys(row).map((column) => { + if (column === 'idx' && !table.columns.some((col) => col.name === 'idx')) { + // When we fetch this data from the database, we automatically add an + // 'idx' column if none exists. We shouldn't export this column since + // it's not actually part of the user's table. + delete formattedRow[column] + return + } + + if (typeof row[column] === 'object' && row[column] !== null) + formattedRow[column] = JSON.stringify(formattedRow[column]) + }) + return formattedRow + }) +} + +const convertAndDownload = ( + formattedRows: Record[], + table: SupaTable, + callbacks: OutputCallbacks +): + | { status: 'error'; error: ExportAllRowsErrorFamily } + | { status: 'success'; rowsExported: number } => { + let output: string + try { + output = callbacks.convertToOutputFormat(formattedRows, table) + } catch (error: unknown) { + return { status: 'error', error: new OutputConversionError(error) } + } + let data: Blob + try { + data = callbacks.convertToBlob(output) + } catch (error: unknown) { + return { status: 'error', error: new BlobCreationError(error) } + } + try { + callbacks.save(data, table) + } catch (error: unknown) { + return { status: 'error', error: new DownloadSaveError(error) } + } + + return { + status: 'success', + rowsExported: formattedRows.length, + } +} + +type UseExportAllRowsParams = + | { enabled: false } + | ({ + enabled: true + projectRef: string + connectionString: string | null + entity: Pick + /** + * If known, the total number of rows that will be exported. + * This is used to show progress percentage during export. + */ + totalRows?: number + } & ( + | { + /** + * Rows need to be fetched from the database. + */ + type: 'fetch_all' + filters?: Filter[] + sorts?: Sort[] + roleImpersonationState?: RoleImpersonationState + } + | { + /** + * Rows are already available and provided directly. + */ + type: 'provided_rows' + table: SupaTable + rows: Record[] + } + )) + +type UseExportAllRowsReturn = { + exportInDesiredFormat: () => Promise + confirmationModal: ReactNode | null +} + +export const useExportAllRowsGeneric = ( + params: UseExportAllRowsParams & OutputCallbacks +): UseExportAllRowsReturn => { + const queryClient = useQueryClient() + const { + startProgressTracker, + trackPercentageProgress, + stopTrackerWithError, + dismissTrackerSilently, + markTrackerComplete, + } = useProgressToasts() + + const { convertToOutputFormat, convertToBlob, save } = params + + const [confirmationMessage, setConfirmationMessage] = useState(null) + + const exportInternal = useStaticEffectEvent( + async ({ bypassConfirmation }: { bypassConfirmation: boolean }): Promise => { + if (!params.enabled) return + + const { projectRef, connectionString, entity, totalRows } = params + + const exportResult = + params.type === 'provided_rows' + ? convertAndDownload(formatRowsForExport(params.rows, params.table), params.table, { + convertToOutputFormat, + convertToBlob, + save, + }) + : await fetchAllRows({ + queryClient, + projectRef: projectRef, + connectionString: connectionString, + entity: entity, + bypassConfirmation, + filters: params.filters, + sorts: params.sorts, + roleImpersonationState: params.roleImpersonationState, + startCallback: () => { + startProgressTracker({ + id: entity.id, + name: entity.name, + trackPercentage: totalRows !== undefined, + }) + }, + progressCallback: totalRows + ? (value: number) => + trackPercentageProgress({ + id: entity.id, + name: entity.name, + totalRows: totalRows, + value, + }) + : undefined, + convertToOutputFormat, + convertToBlob, + save, + }) + + if (exportResult.status === 'error') { + const error = exportResult.error + if (error instanceof NoRowsToExportError) { + return stopTrackerWithError( + entity.id, + entity.name, + `The table ${entity.name} has no rows to export.` + ) + } + if (error instanceof TableTooLargeError) { + return stopTrackerWithError(entity.id, entity.name, MAX_EXPORT_ROW_COUNT_MESSAGE) + } + console.error( + `Export All Rows > Error: %s%s%s`, + error.message, + error.cause?.message ? `\n${error.cause.message}` : '', + error.cause?.stack ? `:\n${error.cause.stack}` : '' + ) + return stopTrackerWithError(entity.id, entity.name) + } + + if (exportResult.status === 'require_confirmation') { + return setConfirmationMessage(exportResult.reason) + } + + markTrackerComplete(entity.id, exportResult.rowsExported) + } + ) + + const exportInDesiredFormat = useCallback( + () => exportInternal({ bypassConfirmation: false }), + [exportInternal] + ) + + const onConfirmExport = () => { + exportInternal({ + bypassConfirmation: true, + }) + setConfirmationMessage(null) + } + const onCancelExport = () => { + if (!params.enabled) return + + dismissTrackerSilently(params.entity.id) + setConfirmationMessage(null) + } + + return { + exportInDesiredFormat, + confirmationModal: confirmationMessage ? ( + div>div>h5]:font-normal border-x-0 border-t-0 rounded-none mb-0' }, + title: confirmationMessage, + }} + /> + ) : null, + } +} + +type UseExportAllRowsAsCsvReturn = { + exportCsv: () => Promise + confirmationModal: ReactNode | null +} + +export const useExportAllRowsAsCsv = ( + params: UseExportAllRowsParams +): UseExportAllRowsAsCsvReturn => { + const { exportInDesiredFormat: exportCsv, confirmationModal } = useExportAllRowsGeneric({ + ...params, + convertToOutputFormat: (formattedRows, table) => + Papa.unparse(formattedRows, { + columns: table.columns.map((col) => col.name), + }), + convertToBlob: (csv) => new Blob([csv], { type: 'text/csv;charset=utf-8;' }), + save: (csvData, table) => saveAs(csvData, `${table.name}_rows.csv`), + }) + + return { + exportCsv, + confirmationModal, + } +} + +type UseExportAllRowsAsSqlReturn = { + exportSql: () => Promise + confirmationModal: ReactNode | null +} + +export const useExportAllRowsAsSql = ( + params: UseExportAllRowsParams +): UseExportAllRowsAsSqlReturn => { + const { exportInDesiredFormat: exportSql, confirmationModal } = useExportAllRowsGeneric({ + ...params, + convertToOutputFormat: (formattedRows, table) => formatTableRowsToSQL(table, formattedRows), + convertToBlob: (sqlStatements) => + new Blob([sqlStatements], { type: 'text/sql;charset=utf-8;' }), + save: (sqlData, table) => saveAs(sqlData, `${table.name}_rows.sql`), + }) + + return { + exportSql, + confirmationModal, + } +} + +type UseExportAllRowsAsJsonReturn = { + exportJson: () => Promise + confirmationModal: ReactNode | null +} + +export const useExportAllRowsAsJson = ( + params: UseExportAllRowsParams +): UseExportAllRowsAsJsonReturn => { + const { exportInDesiredFormat: exportJson, confirmationModal } = useExportAllRowsGeneric({ + ...params, + convertToOutputFormat: (formattedRows) => JSON.stringify(formattedRows), + convertToBlob: (jsonStr) => new Blob([jsonStr], { type: 'application/json;charset=utf-8;' }), + save: (jsonData, table) => saveAs(jsonData, `${table.name}_rows.json`), + }) + + return { + exportJson, + confirmationModal, + } +} diff --git a/apps/studio/data/table-editor/table-editor-query-sql.ts b/apps/studio/data/table-editor/table-editor-query-sql.ts index b525c9ea9a3..26fdacba943 100644 --- a/apps/studio/data/table-editor/table-editor-query-sql.ts +++ b/apps/studio/data/table-editor/table-editor-query-sql.ts @@ -5,7 +5,7 @@ export function getTableEditorSql(id?: number) { return minify(/* SQL */ ` with base_table_info as ( - select + select c.oid::int8 as id, nc.nspname as schema, c.relname as name, @@ -36,7 +36,7 @@ export function getTableEditorSql(id?: number) { ) ), table_stats as ( - select + select b.id, case when b.relreplident = 'd' then 'DEFAULT' @@ -52,23 +52,58 @@ export function getTableEditorSql(id?: number) { where b.relkind in ('r', 'p') ), primary_keys as ( - select + select i.indrelid as table_id, - jsonb_agg(jsonb_build_object( - 'schema', n.nspname, - 'table_name', c.relname, - 'table_id', i.indrelid::int8, - 'name', a.attname - )) as primary_keys + jsonb_agg( + jsonb_build_object( + 'schema', n.nspname, + 'table_name', c.relname, + 'table_id', i.indrelid::int8, + 'name', a.attname + ) + order by array_position(i.indkey, a.attnum) + ) as primary_keys from pg_index i join pg_class c on i.indrelid = c.oid - join pg_attribute a on (a.attrelid = c.oid and a.attnum = any(i.indkey)) join pg_namespace n on c.relnamespace = n.oid + join pg_attribute a on a.attrelid = c.oid and a.attnum = any(i.indkey) where i.indisprimary group by i.indrelid ), + index_cols as ( + select + i.indrelid as table_id, + i.indkey, + array_agg( + a.attname + order by array_position(i.indkey, a.attnum) + ) as columns + from pg_index i + join pg_class c on i.indrelid = c.oid + join pg_attribute a on a.attrelid = c.oid + and a.attnum = any(i.indkey) + where i.indisunique + and i.indisprimary = false + group by i.indrelid, i.indkey + ), + unique_indexes as ( + select + ic.table_id, + jsonb_agg( + jsonb_build_object( + 'schema', n.nspname, + 'table_name', c.relname, + 'table_id', ic.table_id::int8, + 'columns', ic.columns + ) + ) as unique_indexes + from index_cols ic + join pg_class c on c.oid = ic.table_id + join pg_namespace n on n.oid = c.relnamespace + group by ic.table_id + ), relationships as ( - select + select c.conrelid as source_id, c.confrelid as target_id, jsonb_build_object( @@ -93,7 +128,7 @@ export function getTableEditorSql(id?: number) { where c.contype = 'f' ), columns as ( - select + select a.attrelid as table_id, jsonb_agg(jsonb_build_object( 'id', (a.attrelid || '.' || a.attnum), @@ -102,19 +137,19 @@ export function getTableEditorSql(id?: number) { 'table', c.relname, 'ordinal_position', a.attnum, 'name', a.attname, - 'default_value', case + 'default_value', case when a.atthasdef then pg_get_expr(ad.adbin, ad.adrelid) else null end, - 'data_type', case - when t.typtype = 'd' then - case + 'data_type', case + when t.typtype = 'd' then + case when bt.typelem <> 0::oid and bt.typlen = -1 then 'ARRAY' when nbt.nspname = 'pg_catalog' then format_type(t.typbasetype, null) else 'USER-DEFINED' end - else - case + else + case when t.typelem <> 0::oid and t.typlen = -1 then 'ARRAY' when nt.nspname = 'pg_catalog' then format_type(a.atttypid, null) else 'USER-DEFINED' @@ -138,7 +173,7 @@ export function getTableEditorSql(id?: number) { 'is_generated', a.attgenerated in ('s'), 'is_nullable', not (a.attnotnull or t.typtype = 'd' and t.typnotnull), 'is_updatable', ( - b.relkind in ('r', 'p') or + b.relkind in ('r', 'p') or (b.relkind in ('v', 'f') and pg_column_is_updatable(b.id, a.attnum, false)) ), 'is_unique', uniques.table_id is not null, @@ -164,7 +199,7 @@ export function getTableEditorSql(id?: number) { left join pg_type bt on (t.typtype = 'd' and t.typbasetype = bt.oid) left join pg_namespace nbt on bt.typnamespace = nbt.oid left join ( - select + select conrelid as table_id, conkey[1] as ordinal_position from pg_catalog.pg_constraint @@ -183,13 +218,13 @@ export function getTableEditorSql(id?: number) { from pg_constraint where contype = 'c' and cardinality(conkey) = 1 order by conrelid, conkey[1], oid asc - ) as check_constraints on check_constraints.table_id = a.attrelid + ) as check_constraints on check_constraints.table_id = a.attrelid and check_constraints.ordinal_position = a.attnum - where a.attnum > 0 + where a.attnum > 0 and not a.attisdropped group by a.attrelid ) - select + select case b.relkind when 'r' then jsonb_build_object( 'entity_type', b.relkind, @@ -205,10 +240,11 @@ export function getTableEditorSql(id?: number) { 'dead_rows_estimate', ts.dead_rows_estimate, 'comment', b.comment, 'primary_keys', coalesce(pk.primary_keys, '[]'::jsonb), + 'unique_indexes', coalesce(ui.unique_indexes, '[]'::jsonb), 'relationships', coalesce( (select jsonb_agg(r.rel_info) from relationships r - where r.source_id = b.id or r.target_id = b.id), + where r.source_id = b.id or r.target_id = b.id), '[]'::jsonb ), 'columns', coalesce(c.columns, '[]'::jsonb) @@ -227,10 +263,11 @@ export function getTableEditorSql(id?: number) { 'dead_rows_estimate', ts.dead_rows_estimate, 'comment', b.comment, 'primary_keys', coalesce(pk.primary_keys, '[]'::jsonb), + 'unique_indexes', coalesce(ui.unique_indexes, '[]'::jsonb), 'relationships', coalesce( (select jsonb_agg(r.rel_info) from relationships r - where r.source_id = b.id or r.target_id = b.id), + where r.source_id = b.id or r.target_id = b.id), '[]'::jsonb ), 'columns', coalesce(c.columns, '[]'::jsonb) @@ -268,6 +305,7 @@ export function getTableEditorSql(id?: number) { from base_table_info b left join table_stats ts on b.id = ts.id left join primary_keys pk on b.id = pk.table_id + left join unique_indexes ui on b.id = ui.table_id left join columns c on b.id = c.table_id; `) } diff --git a/apps/studio/data/table-editor/table-editor-types.ts b/apps/studio/data/table-editor/table-editor-types.ts index 055791c77e4..df65503bd78 100644 --- a/apps/studio/data/table-editor/table-editor-types.ts +++ b/apps/studio/data/table-editor/table-editor-types.ts @@ -13,16 +13,25 @@ interface TableRelationship extends PostgresRelationship { update_action: 'a' | 'r' | 'c' | 'n' | 'd' } +interface TableUniqueIndex { + schema: string + table_name: string + table_id: number + columns: string[] +} + export interface Table extends PostgresTable { entity_type: ENTITY_TYPE.TABLE columns: PostgresColumn[] relationships: TableRelationship[] + unique_indexes?: TableUniqueIndex[] } export interface PartitionedTable extends PostgresTable { entity_type: ENTITY_TYPE.PARTITIONED_TABLE columns: PostgresColumn[] relationships: TableRelationship[] + unique_indexes?: TableUniqueIndex[] } export interface View extends PostgresView { diff --git a/apps/studio/data/table-rows/table-rows-query.ts b/apps/studio/data/table-rows/table-rows-query.ts index 61bea3a5d07..9ddbc15fb1f 100644 --- a/apps/studio/data/table-rows/table-rows-query.ts +++ b/apps/studio/data/table-rows/table-rows-query.ts @@ -1,10 +1,11 @@ -import { Query } from '@supabase/pg-meta/src/query' +import { Query, type QueryFilter } from '@supabase/pg-meta/src/query' import { getTableRowsSql } from '@supabase/pg-meta/src/query/table-row-query' import { useQuery, useQueryClient, type QueryClient } from '@tanstack/react-query' import { IS_PLATFORM } from 'common' import { parseSupaTable } from 'components/grid/SupabaseGrid.utils' import { Filter, Sort, SupaRow, SupaTable } from 'components/grid/types' +import { ENTITY_TYPE } from 'data/entity-types/entity-type-constants' import { prefetchTableEditor } from 'data/table-editor/table-editor-query' import { isMsSqlForeignTable } from 'data/table-editor/table-editor-types' import { @@ -27,15 +28,40 @@ export interface GetTableRowsArgs { roleImpersonationState?: RoleImpersonationState } -// return the primary key columns if exists, otherwise return the first column to use as a default sort -const getDefaultOrderByColumns = (table: SupaTable) => { - const primaryKeyColumns = table.columns.filter((col) => col?.isPrimaryKey).map((col) => col.name) - if (primaryKeyColumns.length === 0) { - const eligibleColumnsForSorting = table.columns.filter((x) => !x.dataType.includes('json')) - if (eligibleColumnsForSorting.length > 0) return [eligibleColumnsForSorting[0]?.name] - else return [] - } else { - return primaryKeyColumns +/** + * Get the preferred columns for sorting of a table. + * + * Use the primary key if it exists, otherwise use a unique index with + * non-nullable columns. If all else fails, fall back to any sortable column. + */ +const getPreferredOrderByColumns = ( + table: SupaTable +): { cursorPaginationEligible: string[][]; cursorPaginationNonEligible: string[] } => { + const cursorPaginationEligible: string[][] = [] + const cursorPaginationNonEligible: string[] = [] + + const primaryKeyColumns = table.primaryKey + if (primaryKeyColumns) { + cursorPaginationEligible.push(primaryKeyColumns) + } + + const uniqueIndexes = table.uniqueIndexes + const cursorFriendlyUniqueIndexes = uniqueIndexes?.filter((index) => { + return index.every((columnName) => { + const column = table.columns.find((column) => column.name === columnName) + return !!column && !column.isNullable + }) + }) + if (cursorFriendlyUniqueIndexes) { + cursorPaginationEligible.push(...cursorFriendlyUniqueIndexes) + } + + const eligibleColumnsForSorting = table.columns.filter((x) => !x.dataType.includes('json')) + cursorPaginationNonEligible.push(...eligibleColumnsForSorting.map((col) => col.name)) + + return { + cursorPaginationEligible, + cursorPaginationNonEligible, } } @@ -88,6 +114,11 @@ export async function executeWithRetry( throw new Error('Max retries reached without success') } +const checkIfCtidAvailable = (table: SupaTable): boolean => + table.type === ENTITY_TYPE.TABLE || + table.type === ENTITY_TYPE.PARTITIONED_TABLE || + table.type === ENTITY_TYPE.MATERIALIZED_VIEW + export const getAllTableRowsSql = ({ table, filters = [], @@ -96,7 +127,7 @@ export const getAllTableRowsSql = ({ table: SupaTable filters?: Filter[] sorts?: Sort[] -}) => { +}): { sql: QueryFilter; cursorColumns: string[] | false } => { const query = new Query() const arrayBasedColumns = table.columns @@ -116,36 +147,54 @@ export const getAllTableRowsSql = ({ queryChains = queryChains.filter(filter.column, filter.operator, value) }) - // Always enforce deterministic ordering for pagination/export - const primaryKeys = getDefaultOrderByColumns(table) + let cursorColumns: string[] | false = false + const { cursorPaginationEligible, cursorPaginationNonEligible } = + getPreferredOrderByColumns(table) + + const hasCtid = checkIfCtidAvailable(table) + if (sorts.length === 0) { - if (primaryKeys.length > 0) { - primaryKeys.forEach((col) => { + if (cursorPaginationEligible.length > 0) { + cursorColumns = cursorPaginationEligible[0] + cursorPaginationEligible[0].forEach((col) => { queryChains = queryChains.order(table.name, col) }) + // Cursor paginated columns do not require ctid fallback as they + // guarantee uniqueness + } else if (cursorPaginationNonEligible.length > 0) { + queryChains = queryChains.order(table.name, cursorPaginationNonEligible[0]) + if (hasCtid) { + queryChains = queryChains.order(table.name, 'ctid') + } + } else { + if (hasCtid) { + queryChains = queryChains.order(table.name, 'ctid') + } } } else { sorts.forEach((sort) => { queryChains = queryChains.order(sort.table, sort.column, sort.ascending, sort.nullsFirst) }) - // Add primary keys as tie-breakers so page order doesn't shuffle - if (primaryKeys.length > 0) { + // Add tie-breakers so page order doesn't shuffle + const tieBreaker = cursorPaginationEligible[0] + if (tieBreaker) { const sortedColumns = new Set( sorts.filter((s) => s.table === table.name).map((s) => s.column) ) - primaryKeys - .filter((pk) => !sortedColumns.has(pk)) - .forEach((pk) => { - queryChains = queryChains.order(table.name, pk) + tieBreaker + .filter((col) => !sortedColumns.has(col)) + .forEach((col) => { + queryChains = queryChains.order(table.name, col) }) + } else { + if (hasCtid) { + queryChains = queryChains.order(table.name, 'ctid') + } } } - // Final tie-breaker: use system column ctid to guarantee a stable, unique order - queryChains = queryChains.order(table.name, 'ctid') - - return queryChains + return { sql: queryChains, cursorColumns } } // TODO: fetchAllTableRows is used for CSV export, but since it doesn't actually truncate anything, (compare to getTableRows) @@ -177,35 +226,79 @@ export const fetchAllTableRows = async ({ } const rows: any[] = [] - const queryChains = getAllTableRowsSql({ table, sorts, filters }) + const { sql: queryChains, cursorColumns } = getAllTableRowsSql({ + table, + sorts, + filters, + }) const rowsPerPage = 500 const THROTTLE_DELAY = 500 - let page = -1 - while (true) { - page += 1 - const from = page * rowsPerPage - const to = (page + 1) * rowsPerPage - 1 - const query = wrapWithRoleImpersonation( - queryChains.range(from, to).toSql(), - roleImpersonationState - ) + if (cursorColumns) { + let cursor: Record | null = null + while (true) { + let queryChainsWithCursor = queryChains.clone() - try { - const { result } = await executeWithRetry(async () => - executeSql({ projectRef, connectionString, sql: query }) + if (cursor) { + queryChainsWithCursor = queryChainsWithCursor.filter( + cursorColumns, + '>', + cursorColumns.map((col) => cursor![col]) + ) + } + const query = wrapWithRoleImpersonation( + queryChainsWithCursor.range(0, rowsPerPage - 1).toSql(), + roleImpersonationState ) - rows.push(...result) - progressCallback?.(rows.length) - if (result.length < rowsPerPage) break + try { + const { result } = await executeWithRetry(async () => + executeSql({ projectRef, connectionString, sql: query }) + ) + rows.push(...result) + progressCallback?.(rows.length) - await sleep(THROTTLE_DELAY) - } catch (error) { - throw new Error( - `Error fetching all table rows: ${error instanceof Error ? error.message : 'Unknown error'}` + cursor = {} + for (const col of cursorColumns) { + cursor[col] = result[result.length - 1]?.[col] + } + + if (result.length < rowsPerPage) break + + await sleep(THROTTLE_DELAY) + } catch (error) { + throw new Error( + `Error fetching all table rows: ${error instanceof Error ? error.message : 'Unknown error'}` + ) + } + } + } else { + let page = -1 + while (true) { + page += 1 + const from = page * rowsPerPage + const to = (page + 1) * rowsPerPage - 1 + const query = wrapWithRoleImpersonation( + queryChains.range(from, to).toSql(), + roleImpersonationState ) + + try { + const { result } = await executeWithRetry(async () => + executeSql({ projectRef, connectionString, sql: query }) + ) + rows.push(...result) + progressCallback?.(rows.length) + + if (result.length < rowsPerPage) break + + await sleep(THROTTLE_DELAY) + } catch (error) { + throw new Error( + `Error fetching all table rows: ${error instanceof Error ? error.message : 'Unknown error'}` + ) + } } } diff --git a/apps/studio/hooks/useStaticEffectEvent.ts b/apps/studio/hooks/useStaticEffectEvent.ts new file mode 100644 index 00000000000..5fe642be75c --- /dev/null +++ b/apps/studio/hooks/useStaticEffectEvent.ts @@ -0,0 +1,15 @@ +import { useCallback, useLayoutEffect, useRef } from 'react' + +export const useStaticEffectEvent = (callback: Callback) => { + const callbackRef = useRef(callback) + + useLayoutEffect(() => { + callbackRef.current = callback + }) + + const eventFn = useCallback((...args: any) => { + return callbackRef.current(...args) + }, []) + + return eventFn as unknown as Callback +} diff --git a/e2e/studio/features/table-editor.spec.ts b/e2e/studio/features/table-editor.spec.ts index 37e5898613c..582e91000b0 100644 --- a/e2e/studio/features/table-editor.spec.ts +++ b/e2e/studio/features/table-editor.spec.ts @@ -662,7 +662,7 @@ testRunner('table editor', () => { const downloadJsonPath = await downloadJson.path() const jsonContent = fs.readFileSync(downloadJsonPath, 'utf-8') expect(jsonContent).toBe( - `[{"idx":0,"id":4,"created_at":"2025-01-01 12:00:00+00","pw_column":"value 4 to export"},{"idx":1,"id":5,"created_at":"2025-01-01 12:00:00+00","pw_column":"value 5 to export"},{"idx":2,"id":6,"created_at":"2025-01-01 12:00:00+00","pw_column":"value 6 to export"}]` + `[{"id":4,"created_at":"2025-01-01 12:00:00+00","pw_column":"value 4 to export"},{"id":5,"created_at":"2025-01-01 12:00:00+00","pw_column":"value 5 to export"},{"id":6,"created_at":"2025-01-01 12:00:00+00","pw_column":"value 6 to export"}]` ) await page.waitForTimeout(1000) // wait for event processing to complete fs.unlinkSync(downloadJsonPath) diff --git a/packages/pg-meta/src/query/QueryFilter.ts b/packages/pg-meta/src/query/QueryFilter.ts index 86e05a71e7b..1bd5e2a7682 100644 --- a/packages/pg-meta/src/query/QueryFilter.ts +++ b/packages/pg-meta/src/query/QueryFilter.ts @@ -44,6 +44,29 @@ export class QueryFilter implements IQueryFilter, IQueryModifier { return this._getQueryModifier().range(from, to) } + clone(): QueryFilter { + const clonedData = structuredClone({ + table: this.table, + action: this.action, + actionValue: this.actionValue, + actionOptions: this.actionOptions, + filters: this.filters, + sorts: this.sorts, + }) + + const cloned = new QueryFilter( + clonedData.table, + clonedData.action, + clonedData.actionValue, + clonedData.actionOptions + ) + + cloned.filters = clonedData.filters + cloned.sorts = clonedData.sorts + + return cloned + } + toSql(options?: { isCTE: boolean; isFinal: boolean }) { return this._getQueryModifier().toSql(options) } diff --git a/packages/ui-patterns/src/Dialogs/ConfirmationModal.tsx b/packages/ui-patterns/src/Dialogs/ConfirmationModal.tsx index bea5d1633a5..df87a26d441 100644 --- a/packages/ui-patterns/src/Dialogs/ConfirmationModal.tsx +++ b/packages/ui-patterns/src/Dialogs/ConfirmationModal.tsx @@ -91,7 +91,12 @@ export const ConfirmationModal = forwardRef< } }} > - + {title} {description && {description}}