From 8456108fb16e7a8e8d337befdbba6da2871f7553 Mon Sep 17 00:00:00 2001 From: Ryan Martin <51780559+rmrt1n@users.noreply.github.com> Date: Thu, 8 Aug 2024 23:19:32 +0700 Subject: [PATCH] feat(studio): check column types in imported spreadsheet (#17072) * feat(studio): check column types in imported spreadsheet * refactor: more checks on check inferred column type * fix: spreadsheet import type check not working because selectedtable is undefined * rebuild --------- Co-authored-by: Ryan Martin Co-authored-by: Terry Sutton --- .../SpreadsheetImport/SpreadsheetImport.tsx | 36 +++++++++++++++++-- .../SpreadsheetImportPreview.tsx | 34 ++++++++++++++++-- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.tsx index c587a914a41..b51ae3d0179 100644 --- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.tsx +++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.tsx @@ -60,13 +60,39 @@ const SpreadsheetImport = ({ }) const [errors, setErrors] = useState([]) const [selectedHeaders, setSelectedHeaders] = useState([]) + const [selectedColumnTypeMap, setSelectedColumnTypeMap] = useState([]) - const selectedTableColumns = (selectedTable?.columns ?? []).map((column) => column.name) + const selectedTableColumns = (selectedTable?.columns ?? []).map((column) => ({ + name: column.name, + format: column.format, + })) const incompatibleHeaders = selectedHeaders.filter( - (header) => !selectedTableColumns.includes(header) + (header) => !selectedTableColumns.some((column) => column.name === header) ) const isCompatible = selectedTable !== undefined ? incompatibleHeaders.length === 0 : true + const checkInferredColumnTypes = (columnTypeMap: any) => { + // all other types can in theory be casted into a string + // e.g. a column with type of text can still contain the value "1", but inferColumnType() will + // still infer it as an int8 + const stringTypes = ['text', 'varchar'] + const lookup: { [key: string]: string[] } = { + int8: ['int2', 'int4', 'int8', 'float4', 'float8', 'numeric', ...stringTypes], + float8: ['float4', 'float8', 'numeric', ...stringTypes], + jsonb: ['json', 'jsonb', ...stringTypes], + text: ['uuid', ...stringTypes], + timestamptz: ['date', 'time', 'timetz', 'timestamp', 'timestamptz', ...stringTypes], + bool: ['bool', ...stringTypes], + } + return selectedTableColumns + .filter((c) => Object.keys(columnTypeMap).includes(c.name)) + .filter((c) => !lookup[columnTypeMap[c.name]].includes(c.format)) + .map((c) => ({ name: c.name, format: columnTypeMap[c.name], expectedFormat: c.format })) + } + + const incompatibleTypeColumns = checkInferredColumnTypes(selectedColumnTypeMap) + const isTypeCompatible = selectedTable !== undefined ? selectedColumnTypeMap.length === 0 : true + const onProgressUpdate = (progress: number) => { setParseProgress(progress) } @@ -113,6 +139,7 @@ const SpreadsheetImport = ({ setErrors(errors) setSelectedHeaders(headers) + setSelectedColumnTypeMap(columnTypeMap) setSpreadsheetData({ headers, rows: previewRows, rowCount, columnTypeMap }) } event.target.value = '' @@ -134,8 +161,10 @@ const SpreadsheetImport = ({ `Some issues have been detected on ${errors.length} rows. More details below the content preview.` ) } + setErrors(errors) setSelectedHeaders(headers) + setSelectedColumnTypeMap(columnTypeMap) setSpreadsheetData({ headers, rows, rowCount: rows.length, columnTypeMap }) } else { setSpreadsheetData(EMPTY_SPREADSHEET_DATA) @@ -162,7 +191,7 @@ const SpreadsheetImport = ({ } else if (selectedHeaders.length === 0) { toast.error('Please select at least one header from your CSV') resolve() - } else if (!isCompatible) { + } else if (!(isCompatible && isTypeCompatible)) { toast.error( 'The data that you are trying to import is incompatible with your table structure' ) @@ -233,6 +262,7 @@ const SpreadsheetImport = ({ errors={errors} selectedHeaders={selectedHeaders} incompatibleHeaders={incompatibleHeaders} + incompatibeTypeColumns={incompatibleTypeColumns} /> diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImportPreview.tsx b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImportPreview.tsx index e275feb3196..b274345020c 100644 --- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImportPreview.tsx +++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImportPreview.tsx @@ -24,6 +24,7 @@ interface SpreadsheetImportPreviewProps { errors?: any[] selectedHeaders: string[] incompatibleHeaders: string[] + incompatibeTypeColumns: any[] } const SpreadsheetImportPreview = ({ @@ -32,6 +33,7 @@ const SpreadsheetImportPreview = ({ errors = [], selectedHeaders, incompatibleHeaders, + incompatibeTypeColumns, }: SpreadsheetImportPreviewProps) => { const [expandPreview, setExpandPreview] = useState(false) const [expandedErrors, setExpandedErrors] = useState([]) @@ -43,6 +45,7 @@ const SpreadsheetImportPreview = ({ const previewRows = rows.slice(0, MAX_ROWS) const isCompatible = selectedTable !== undefined ? incompatibleHeaders.length === 0 : true + const isTypeCompatible = selectedTable !== undefined ? incompatibeTypeColumns.length === 0 : true useEffect(() => { setExpandPreview(true) @@ -64,6 +67,7 @@ const SpreadsheetImportPreview = ({

Preview data to be imported

{!isCompatible && Data incompatible} + {!isTypeCompatible && Data type incompatible} {errors.length > 0 && {errors.length} issues found}