fix(table-editor): handle multi-dimentional array truncation and preview (#36108)

* fix(table-editor): handle multi-dimentional array truncation and preview

* fix(SidePanelEditor): fully load truncated input fields

* Small tweaks

---------

Co-authored-by: Joshen Lim <joshenlimek@gmail.com>
This commit is contained in:
Andrew ValleteauandJoshen Lim authored and GitHub committed 2025-06-05 10:52:47 +08:00
1 parent cc650756ce
commit 2d0b321ec6
11 files changed
+306 -71

No files matched your search

@@ -7,7 +7,6 @@ import { useParams } from 'common'
import { useTableEditorQuery } from 'data/table-editor/table-editor-query'
import { isTableLike } from 'data/table-editor/table-editor-types'
import { useGetCellValueMutation } from 'data/table-rows/get-cell-value-mutation'
import { MAX_ARRAY_SIZE, MAX_CHARACTERS } from '@supabase/pg-meta/src/query/table-row-query'
import { useSelectedProject } from 'hooks/misc/useSelectedProject'
import { prettifyJSON, removeJSONTrailingComma, tryParseJson } from 'lib/helpers'
import { useTableEditorTableStateSnapshot } from 'state/table-editor-table'
@@ -16,6 +15,7 @@ import { BlockKeys } from '../common/BlockKeys'
import { MonacoEditor } from '../common/MonacoEditor'
import { NullValue } from '../common/NullValue'
import { TruncatedWarningOverlay } from './TruncatedWarningOverlay'
import { isValueTruncated } from 'components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils'
const verifyJSON = (value: string) => {
try {
@@ -73,19 +73,7 @@ export const JsonEditor = <TRow, TSummaryRow = unknown>({
const jsonString = prettifyJSON(initialValue ? tryFormatInitialValue(initialValue) : '')
const isTruncated =
(typeof initialValue === 'string' &&
initialValue.endsWith('...') &&
initialValue.length > MAX_CHARACTERS) ||
// if the value is an array which total representation is > MAX_CHARACTERS
// we'll select the first MAX_ARRAY_SIZE elements and add a "..." last element at the end of it
(typeof initialValue === 'string' &&
// If the string represent an array finishing with "..." element
initialValue.startsWith('["') &&
initialValue.endsWith(',"..."]') &&
// If the array have MAX_ARRAY_SIZE elements in it
// its a large truncated array
(initialValue.match(/","/g) || []).length === MAX_ARRAY_SIZE)
const isTruncated = isValueTruncated(initialValue)
const [isPopoverOpen, setIsPopoverOpen] = useState(true)
const [value, setValue] = useState<string | null>(jsonString)
@@ -7,7 +7,6 @@ import { useParams } from 'common'
import { useTableEditorQuery } from 'data/table-editor/table-editor-query'
import { isTableLike } from 'data/table-editor/table-editor-types'
import { useGetCellValueMutation } from 'data/table-rows/get-cell-value-mutation'
import { MAX_CHARACTERS } from '@supabase/pg-meta/src/query/table-row-query'
import { useSelectedProject } from 'hooks/misc/useSelectedProject'
import { useTableEditorTableStateSnapshot } from 'state/table-editor-table'
import { Button, Popover, Tooltip, TooltipContent, TooltipTrigger, cn } from 'ui'
@@ -17,6 +16,7 @@ import { EmptyValue } from '../common/EmptyValue'
import { MonacoEditor } from '../common/MonacoEditor'
import { NullValue } from '../common/NullValue'
import { TruncatedWarningOverlay } from './TruncatedWarningOverlay'
import { isValueTruncated } from 'components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils'
export const TextEditor = <TRow, TSummaryRow = unknown>({
row,
@@ -50,10 +50,7 @@ export const TextEditor = <TRow, TSummaryRow = unknown>({
const { mutate: getCellValue, isLoading, isSuccess } = useGetCellValueMutation()
const isTruncated =
typeof initialValue === 'string' &&
initialValue.endsWith('...') &&
initialValue.length > MAX_CHARACTERS
const isTruncated = isValueTruncated(initialValue)
const loadFullValue = () => {
if (selectedTable === undefined || project === undefined || !isTableLike(selectedTable)) return
@@ -12,6 +12,7 @@ interface ActionBarProps {
closePanel: () => void
formId?: string
}
const ActionBar = ({
loading = false,
disableApply = false,
@@ -1,5 +1,5 @@
import { includes, noop } from 'lodash'
import { Edit, Edit2, Eye } from 'lucide-react'
import { Edit, Eye } from 'lucide-react'
import {
Button,
@@ -22,6 +22,9 @@ import { DateTimeInput } from './DateTimeInput'
import type { EditValue, RowField } from './RowEditor.types'
import { isValueTruncated } from './RowEditor.utils'
const TRUNCATE_DESCRIPTION =
'Note: Value is too large to be rendered in the dashboard. Please expand the editor to edit the value'
export interface InputFieldProps {
field: RowField
errors: any
@@ -149,16 +152,11 @@ const InputField = ({
data-testid={`${field.name}-input`}
layout="horizontal"
label={field.name}
className="text-sm"
className="input-sm"
descriptionText={
<>
{field.comment && <p>{field.comment}</p>}
{isTruncated && (
<p>
Note: Value is too large to be rendered in the dashboard. Please expand the editor
to edit the value
</p>
)}
{isTruncated && <p>{TRUNCATE_DESCRIPTION}</p>}
</>
}
textAreaClassName="pr-8"
@@ -213,12 +211,7 @@ const InputField = ({
descriptionText={
<>
{field.comment && <p>{field.comment}</p>}
{isTruncated && (
<p>
Note: Value is too large to be rendered in the dashboard. Please expand the editor
to edit the value
</p>
)}
{isTruncated && <p>{TRUNCATE_DESCRIPTION}</p>}
</>
}
labelOptional={field.format}
@@ -231,9 +224,9 @@ const InputField = ({
type="default"
htmlType="button"
onClick={() => onEditJson({ column: field.name, value: field.value })}
icon={isEditable ? <Edit2 /> : <Eye />}
icon={isEditable ? <Edit /> : <Eye />}
>
{isEditable ? 'Edit JSON' : 'View JSON'}
{isEditable ? 'Edit' : 'View JSON'}
</Button>
}
/>
@@ -321,12 +314,19 @@ const InputField = ({
)
}
const isTruncated = isValueTruncated(field.value)
return (
<Input
data-testid={`${field.name}-input`}
layout="horizontal"
label={field.name}
descriptionText={field.comment}
descriptionText={
<>
{field.comment && <p>{field.comment}</p>}
{isTruncated && <p>{TRUNCATE_DESCRIPTION}</p>}
</>
}
labelOptional={field.format}
error={errors[field.name]}
value={field.value ?? ''}
@@ -337,8 +337,20 @@ const InputField = ({
? `Default: ${field.defaultValue}`
: 'NULL'
}
disabled={!isEditable}
disabled={!isEditable || isTruncated}
onChange={(event: any) => onUpdateField({ [field.name]: event.target.value })}
actions={
isTruncated ? (
<Button
type="default"
htmlType="button"
onClick={() => onEditJson({ column: field.name, value: field.value })}
icon={isEditable ? <Edit /> : <Eye />}
>
{isEditable ? 'Edit' : 'View'}
</Button>
) : undefined
}
/>
)
}
@@ -1,3 +1,4 @@
import { MAX_CHARACTERS } from '@supabase/pg-meta/src/query/table-row-query'
import { AlignLeft } from 'lucide-react'
import { useCallback, useEffect, useState } from 'react'
import { toast } from 'sonner'
@@ -8,7 +9,6 @@ import TwoOptionToggle from 'components/ui/TwoOptionToggle'
import { useTableEditorQuery } from 'data/table-editor/table-editor-query'
import { isTableLike } from 'data/table-editor/table-editor-types'
import { useGetCellValueMutation } from 'data/table-rows/get-cell-value-mutation'
import { MAX_CHARACTERS } from '@supabase/pg-meta/src/query/table-row-query'
import { useSelectedProject } from 'hooks/misc/useSelectedProject'
import { minifyJSON, prettifyJSON, removeJSONTrailingComma, tryParseJson } from 'lib/helpers'
import { Button, SidePanel, cn } from 'ui'
@@ -126,11 +126,11 @@ const JsonEdit = ({
<div className="flex items-center justify-between">
{view === 'edit' ? (
<p>
{readOnly ? 'Viewing' : 'Editing'} JSON Field: <code>{column}</code>
{readOnly ? 'Viewing' : 'Editing'} value of: <code>{column}</code>
</p>
) : (
<p>
Viewing JSON Field: <code>{column}</code>
Viewing value of: <code>{column}</code>
</p>
)}
{(!isTruncated || (isTruncated && isSuccess)) && (
@@ -170,27 +170,29 @@ const RowEditor = ({
<form onSubmit={(e) => onSaveChanges(e)} className="h-full">
<div className="flex h-full flex-col">
<div className="flex flex-grow flex-col">
<SidePanel.Content>
<div className="space-y-10 py-6">
{requiredFields.map((field: RowField) => {
return (
<InputField
key={field.id}
field={field}
errors={errors}
onUpdateField={onUpdateField}
onEditJson={setSelectedValueForJsonEdit}
onEditText={setSelectedValueForTextEdit}
onSelectForeignKey={() => onOpenForeignRowSelector(field)}
isEditable={editable}
/>
)
})}
</div>
</SidePanel.Content>
{requiredFields.length > 0 && (
<SidePanel.Content>
<div className="space-y-10 py-6">
{requiredFields.map((field: RowField) => {
return (
<InputField
key={field.id}
field={field}
errors={errors}
onUpdateField={onUpdateField}
onEditJson={setSelectedValueForJsonEdit}
onEditText={setSelectedValueForTextEdit}
onSelectForeignKey={() => onOpenForeignRowSelector(field)}
isEditable={editable}
/>
)
})}
</div>
</SidePanel.Content>
)}
{optionalFields.length > 0 && (
<>
<SidePanel.Separator />
{requiredFields.length > 0 && <SidePanel.Separator />}
<SidePanel.Content>
<div className="space-y-10 py-6">
<div>
@@ -3,7 +3,7 @@ import dayjs from 'dayjs'
import { compact, isEqual, isNull, isString, omitBy } from 'lodash'
import type { Dictionary } from 'types'
import { MAX_CHARACTERS } from '@supabase/pg-meta/src/query/table-row-query'
import { MAX_ARRAY_SIZE, MAX_CHARACTERS } from '@supabase/pg-meta/src/query/table-row-query'
import { minifyJSON, tryParseJson } from 'lib/helpers'
import { ForeignKey } from '../ForeignKeySelector/ForeignKeySelector.types'
import {
@@ -262,7 +262,21 @@ export const generateUpdateRowPayload = (originalRow: any, fields: RowField[]) =
* Checks if the value is truncated. The JSON types are usually truncated if they're too big to show in the editor.
*/
export const isValueTruncated = (value: string | null | undefined) => {
return value?.endsWith('...') && (value ?? '').length > MAX_CHARACTERS
return (
(typeof value === 'string' && value.endsWith('...') && value.length > MAX_CHARACTERS) ||
// if the value is an array which total representation is > MAX_CHARACTERS
// we'll select the first MAX_ARRAY_SIZE elements and add a "..." last element at the end of it
(typeof value === 'string' &&
// If the string represent an array finishing with "..." element
value.startsWith('["') &&
value.endsWith(',"..."]') &&
// If the array have MAX_ARRAY_SIZE elements in it
// its a large truncated array
(value.match(/","/g) || []).length === MAX_ARRAY_SIZE) ||
// if the string represent a multi-dimentional array we always consider it as possibly truncated
// so user load the whole value before edition
(typeof value === 'string' && value.startsWith('[["'))
)
}
export const convertByteaToHex = (value: { type: 'Buffer'; data: number[] }) => {
@@ -1,4 +1,5 @@
import { Editor } from '@monaco-editor/react'
import { MAX_CHARACTERS } from '@supabase/pg-meta/src/query/table-row-query'
import { Loader } from 'lucide-react'
import { useCallback, useEffect, useState } from 'react'
import remarkGfm from 'remark-gfm'
@@ -10,7 +11,6 @@ import TwoOptionToggle from 'components/ui/TwoOptionToggle'
import { useTableEditorQuery } from 'data/table-editor/table-editor-query'
import { isTableLike } from 'data/table-editor/table-editor-types'
import { useGetCellValueMutation } from 'data/table-rows/get-cell-value-mutation'
import { MAX_CHARACTERS } from '@supabase/pg-meta/src/query/table-row-query'
import { useSelectedProject } from 'hooks/misc/useSelectedProject'
import { Button, SidePanel, cn } from 'ui'
import ActionBar from '../ActionBar'
+7
View File
@@ -193,6 +193,13 @@ a {
}
}
.input-sm {
input,
textarea {
@apply text-sm;
}
}
.input-xs {
input,
textarea {
+11 -1
View File
@@ -194,10 +194,20 @@ export const getTableRowsSql = ({
// This returns the first MAX_ARRAY_SIZE elements of the array (adjustable) and adds '...' if truncated
// NOTE: this is not optimal, as the first element in the array could still be very large (more than 10Kb) and in such case
// the trimming might fail.
// Also handle multi-dimentionals array truncation, but won't happen the extra `...` element to it as we can't determine what's
// the right number of items to generate within the array. Studio side, we'll consider any multi-dimentional array as possibly
// truncated.
selectExpressions[index] = `
case
when octet_length(${ident(columnName)}::text) > ${maxCharacters}
then (select array_cat(${ident(columnName)}[1:${maxArraySize}]::${typeCast}, ${lastElement}::${typeCast}))::${typeCast}
then
case
when array_ndims(${ident(columnName)}) = 1
then
(select array_cat(${ident(columnName)}[1:${maxArraySize}]::${typeCast}, ${lastElement}::${typeCast}))::${typeCast}
else
${ident(columnName)}[1:${maxArraySize}]::${typeCast}
end
else ${ident(columnName)}::${typeCast}
end
`
@@ -83,7 +83,14 @@ describe('Table Row Query', () => {
end as name,status,
case
when octet_length(history::text) > 10240
then (select array_cat(history[1:50]::text[], array['...']::text[]))::text[]
then
case
when array_ndims(history) = 1
then
(select array_cat(history[1:50]::text[], array['...']::text[]))::text[]
else
history[1:50]::text[]
end
else history::text[]
end
from _base_query;"
@@ -149,7 +156,14 @@ describe('Table Row Query', () => {
end as name,status,
case
when octet_length(history::text) > 10240
then (select array_cat(history[1:50]::text[], array['...']::text[]))::text[]
then
case
when array_ndims(history) = 1
then
(select array_cat(history[1:50]::text[], array['...']::text[]))::text[]
else
history[1:50]::text[]
end
else history::text[]
end
from _base_query;"
@@ -188,7 +202,14 @@ describe('Table Row Query', () => {
end as name,status,
case
when octet_length(history::text) > 10240
then (select array_cat(history[1:50]::text[], array['...']::text[]))::text[]
then
case
when array_ndims(history) = 1
then
(select array_cat(history[1:50]::text[], array['...']::text[]))::text[]
else
history[1:50]::text[]
end
else history::text[]
end
from _base_query;"
@@ -234,7 +255,14 @@ describe('Table Row Query', () => {
end as name,
case
when octet_length(tags::text) > 10240
then (select array_cat(tags[1:50]::text[], array['...']::text[]))::text[]
then
case
when array_ndims(tags) = 1
then
(select array_cat(tags[1:50]::text[], array['...']::text[]))::text[]
else
tags[1:50]::text[]
end
else tags::text[]
end
from _base_query;"
@@ -374,7 +402,14 @@ describe('Table Row Query', () => {
end as name,
case
when octet_length(large_array::text) > 2048
then (select array_cat(large_array[1:10]::text[], array['...']::text[]))::text[]
then
case
when array_ndims(large_array) = 1
then
(select array_cat(large_array[1:10]::text[], array['...']::text[]))::text[]
else
large_array[1:10]::text[]
end
else large_array::text[]
end
from _base_query;"
@@ -601,13 +636,27 @@ describe('Table Row Query', () => {
end as name,
case
when octet_length(large_array_jsonb::text) > 2048
then (select array_cat(large_array_jsonb[1:10]::jsonb[], array['{"truncated": true}'::json]::jsonb[]))::jsonb[]
then
case
when array_ndims(large_array_jsonb) = 1
then
(select array_cat(large_array_jsonb[1:10]::jsonb[], array['{"truncated": true}'::json]::jsonb[]))::jsonb[]
else
large_array_jsonb[1:10]::jsonb[]
end
else large_array_jsonb::jsonb[]
end
,
case
when octet_length(large_array_json::text) > 2048
then (select array_cat(large_array_json[1:10]::json[], array['{"truncated": true}'::json]::json[]))::json[]
then
case
when array_ndims(large_array_json) = 1
then
(select array_cat(large_array_json[1:10]::json[], array['{"truncated": true}'::json]::json[]))::json[]
else
large_array_json[1:10]::json[]
end
else large_array_json::json[]
end
from _base_query;"
@@ -964,7 +1013,14 @@ describe('Table Row Query', () => {
end as name,
case
when octet_length(large_array::text) > 256
then (select array_cat(large_array[1:50]::text[], array['...']::text[]))::text[]
then
case
when array_ndims(large_array) = 1
then
(select array_cat(large_array[1:50]::text[], array['...']::text[]))::text[]
else
large_array[1:50]::text[]
end
else large_array::text[]
end
from _base_query;"
@@ -1525,4 +1581,152 @@ describe('Table Row Query', () => {
`)
})
})
withTestDatabase('should handle large multi-dimensional arrays correctly', async (db) => {
// Create test table with multi-dimensional arrays
await db.executeQuery(`
CREATE TABLE public.monitor_data (
subject_id TEXT,
"timestamp" TIMESTAMP[],
"PPG" FLOAT8[][],
"ACC" FLOAT8[][]
);
INSERT INTO public.monitor_data (subject_id, "timestamp", "PPG", "ACC")
VALUES (
'subject-1',
ARRAY['2024-01-01 00:00:00'::timestamp, '2024-01-02 00:00:00'::timestamp],
ARRAY[
[1.1, 1.2, 1.3, 1.4, 1.5, 1.6],
[2.1, 2.2, 2.3, 2.4, 2.5, 2.6],
[3.1, 3.2, 3.3, 3.4, 3.5, 3.6]
]::FLOAT8[][],
ARRAY[
[4.1, 4.2, 4.3, 4.4, 4.5, 4.6],
[5.1, 5.2, 5.3, 5.4, 5.5, 5.6],
[6.1, 6.2, 6.3, 6.4, 6.5, 6.6]
]::FLOAT8[][]
);
INSERT INTO public.monitor_data (subject_id, "timestamp", "PPG", "ACC")
VALUES (
'subject-large',
-- large 1D timestamp array (e.g., 1000 timestamps)
ARRAY(
SELECT generate_series('2024-01-01'::timestamp, '2024-01-01'::timestamp + interval '999 minutes', '1 minute')
),
-- large 2D float8 arrays (e.g., 1000 x 6)
ARRAY(
SELECT ARRAY[
random(), random(), random(), random(), random(), random()
]::float8[]
FROM generate_series(1, 1000)
)::float8[][],
ARRAY(
SELECT ARRAY[
random(), random(), random(), random(), random(), random()
]::float8[]
FROM generate_series(1, 1000)
)::float8[][]
);
`)
// Get table metadata
const { sql: tablesSql, zod: tablesZod } = pgMeta.tables.list()
const tables = tablesZod.parse(await db.executeQuery(tablesSql))
const testTable = tables.find((table) => table.name === 'monitor_data')
expect(testTable).toBeDefined()
// Generate SQL with default settings
const sql = getTableRowsSql({
table: testTable!,
page: 1,
limit: 10,
})
// Verify SQL generation with snapshot
expect(sql).toMatchInlineSnapshot(`
"with _base_query as (select * from public.monitor_data order by monitor_data.subject_id asc nulls last limit 10 offset 0)
select case
when octet_length(subject_id::text) > 10240
then left(subject_id::text, 10240) || '...'
else subject_id::text
end as subject_id,
case
when octet_length(timestamp::text) > 10240
then
case
when array_ndims(timestamp) = 1
then
(select array_cat(timestamp[1:50]::text[], array['...']::text[]))::text[]
else
timestamp[1:50]::text[]
end
else timestamp::text[]
end
,
case
when octet_length("PPG"::text) > 10240
then
case
when array_ndims("PPG") = 1
then
(select array_cat("PPG"[1:50]::text[], array['...']::text[]))::text[]
else
"PPG"[1:50]::text[]
end
else "PPG"::text[]
end
,
case
when octet_length("ACC"::text) > 10240
then
case
when array_ndims("ACC") = 1
then
(select array_cat("ACC"[1:50]::text[], array['...']::text[]))::text[]
else
"ACC"[1:50]::text[]
end
else "ACC"::text[]
end
from _base_query;"
`)
// Execute the SQL and verify results
const queryResult = await db.executeQuery(sql)
expect(queryResult.length).toBe(2)
// Verify the first row (small arrays)
const smallRow = queryResult.find((row: any) => row.subject_id === 'subject-1')
expect(smallRow).toBeDefined()
expect(smallRow.timestamp).toHaveLength(2)
expect(smallRow.PPG).toHaveLength(3)
expect(smallRow.PPG[0]).toHaveLength(6)
expect(smallRow.ACC).toHaveLength(3)
expect(smallRow.ACC[0]).toHaveLength(6)
// Verify the second row (large arrays)
const largeRow = queryResult.find((row: any) => row.subject_id === 'subject-large')
expect(largeRow).toBeDefined()
expect(largeRow.timestamp).toHaveLength(51) // Has the extra '...' element
expect(largeRow.PPG).toHaveLength(50)
expect(largeRow.PPG[0]).toHaveLength(6)
expect(largeRow.ACC).toHaveLength(50)
expect(largeRow.ACC[0]).toHaveLength(6)
// Test with custom maxArraySize
const sqlWithCustomSize = getTableRowsSql({
table: testTable!,
page: 1,
limit: 10,
maxArraySize: 10,
})
const customSizeResult = await db.executeQuery(sqlWithCustomSize)
const largeRowCustom = customSizeResult.find((row: any) => row.subject_id === 'subject-large')
expect(largeRowCustom.timestamp).toHaveLength(11) // Has the extra '...' element
expect(largeRowCustom.PPG).toHaveLength(10) // multi-dimentional array are truncated
expect(largeRowCustom.ACC).toHaveLength(10)
})
})