feat(studio): check column types in imported spreadsheet (#17072)

* feat(studio): check column types in imported spreadsheet

* refactor: more checks on check inferred column type

* fix: spreadsheet import type check not working because selectedtable is undefined

* rebuild

---------

Co-authored-by: Ryan Martin <rmrt1n@users.noreply.github.com>
Co-authored-by: Terry Sutton <saltcod@gmail.com>

apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImport.tsx

@@ -60,13 +60,39 @@ const SpreadsheetImport = ({
   })
   const [errors, setErrors] = useState<any>([])
   const [selectedHeaders, setSelectedHeaders] = useState<string[]>([])
+  const [selectedColumnTypeMap, setSelectedColumnTypeMap] = useState<any>([])
 
-  const selectedTableColumns = (selectedTable?.columns ?? []).map((column) => column.name)
+  const selectedTableColumns = (selectedTable?.columns ?? []).map((column) => ({
+    name: column.name,
+    format: column.format,
+  }))
   const incompatibleHeaders = selectedHeaders.filter(
-    (header) => !selectedTableColumns.includes(header)
+    (header) => !selectedTableColumns.some((column) => column.name === header)
   )
   const isCompatible = selectedTable !== undefined ? incompatibleHeaders.length === 0 : true
 
+  const checkInferredColumnTypes = (columnTypeMap: any) => {
+    // all other types can in theory be casted into a string
+    // e.g. a column with type of text can still contain the value "1", but inferColumnType() will
+    // still infer it as an int8
+    const stringTypes = ['text', 'varchar']
+    const lookup: { [key: string]: string[] } = {
+      int8: ['int2', 'int4', 'int8', 'float4', 'float8', 'numeric', ...stringTypes],
+      float8: ['float4', 'float8', 'numeric', ...stringTypes],
+      jsonb: ['json', 'jsonb', ...stringTypes],
+      text: ['uuid', ...stringTypes],
+      timestamptz: ['date', 'time', 'timetz', 'timestamp', 'timestamptz', ...stringTypes],
+      bool: ['bool', ...stringTypes],
+    }
+    return selectedTableColumns
+      .filter((c) => Object.keys(columnTypeMap).includes(c.name))
+      .filter((c) => !lookup[columnTypeMap[c.name]].includes(c.format))
+      .map((c) => ({ name: c.name, format: columnTypeMap[c.name], expectedFormat: c.format }))
+  }
+
+  const incompatibleTypeColumns = checkInferredColumnTypes(selectedColumnTypeMap)
+  const isTypeCompatible = selectedTable !== undefined ? selectedColumnTypeMap.length === 0 : true
+
   const onProgressUpdate = (progress: number) => {
     setParseProgress(progress)
   }
@@ -113,6 +139,7 @@ const SpreadsheetImport = ({
 
       setErrors(errors)
       setSelectedHeaders(headers)
+      setSelectedColumnTypeMap(columnTypeMap)
       setSpreadsheetData({ headers, rows: previewRows, rowCount, columnTypeMap })
     }
     event.target.value = ''
@@ -134,8 +161,10 @@ const SpreadsheetImport = ({
           `Some issues have been detected on ${errors.length} rows. More details below the content preview.`
         )
       }
+
       setErrors(errors)
       setSelectedHeaders(headers)
+      setSelectedColumnTypeMap(columnTypeMap)
       setSpreadsheetData({ headers, rows, rowCount: rows.length, columnTypeMap })
     } else {
       setSpreadsheetData(EMPTY_SPREADSHEET_DATA)
@@ -162,7 +191,7 @@ const SpreadsheetImport = ({
     } else if (selectedHeaders.length === 0) {
       toast.error('Please select at least one header from your CSV')
       resolve()
-    } else if (!isCompatible) {
+    } else if (!(isCompatible && isTypeCompatible)) {
       toast.error(
         'The data that you are trying to import is incompatible with your table structure'
       )
@@ -233,6 +262,7 @@ const SpreadsheetImport = ({
             errors={errors}
             selectedHeaders={selectedHeaders}
             incompatibleHeaders={incompatibleHeaders}
+            incompatibeTypeColumns={incompatibleTypeColumns}
           />
           <SidePanel.Separator />
         </>

apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/SpreadsheetImport/SpreadsheetImportPreview.tsx

@@ -24,6 +24,7 @@ interface SpreadsheetImportPreviewProps {
   errors?: any[]
   selectedHeaders: string[]
   incompatibleHeaders: string[]
+  incompatibeTypeColumns: any[]
 }
 
 const SpreadsheetImportPreview = ({
@@ -32,6 +33,7 @@ const SpreadsheetImportPreview = ({
   errors = [],
   selectedHeaders,
   incompatibleHeaders,
+  incompatibeTypeColumns,
 }: SpreadsheetImportPreviewProps) => {
   const [expandPreview, setExpandPreview] = useState(false)
   const [expandedErrors, setExpandedErrors] = useState<string[]>([])
@@ -43,6 +45,7 @@ const SpreadsheetImportPreview = ({
   const previewRows = rows.slice(0, MAX_ROWS)
 
   const isCompatible = selectedTable !== undefined ? incompatibleHeaders.length === 0 : true
+  const isTypeCompatible = selectedTable !== undefined ? incompatibeTypeColumns.length === 0 : true
 
   useEffect(() => {
     setExpandPreview(true)
@@ -64,6 +67,7 @@ const SpreadsheetImportPreview = ({
             <div className="flex items-center space-x-2">
               <p className="text-sm">Preview data to be imported</p>
               {!isCompatible && <Badge variant="destructive">Data incompatible</Badge>}
+              {!isTypeCompatible && <Badge variant="destructive">Data type incompatible</Badge>}
               {errors.length > 0 && <Badge variant="warning">{errors.length} issues found</Badge>}
             </div>
             <Button
@@ -113,11 +117,11 @@ const SpreadsheetImportPreview = ({
               </div>
             )}
           </div>
-          {(!isCompatible || errors.length > 0) && (
+          {(!isCompatible || !isTypeCompatible || errors.length > 0) && (
             <div className="space-y-2 my-4">
               <div className="flex flex-col space-y-1">
                 <p className="text-sm">Issues found in spreadsheet</p>
-                {isCompatible && (
+                {isCompatible && isTypeCompatible && (
                   <p className="text-sm text-foreground-light">
                     {selectedTable !== undefined
                       ? 'This CSV can still be imported into your table despite issues in the following rows.'
@@ -143,6 +147,32 @@ const SpreadsheetImportPreview = ({
                     </div>
                   </div>
                 )}
+                {!isTypeCompatible && (
+                  <div className="space-y-2">
+                    <div className="flex items-start space-x-2">
+                      <div className="w-[14px] h-[14px] flex items-center justify-center translate-y-[3px]">
+                        <div className="w-[6px] h-[6px] rounded-full bg-scale-1000" />
+                      </div>
+                      <p className="text-sm">
+                        This CSV <span className="text-red-900">cannot</span> be imported into your
+                        table due to incompatible types in these columns:
+                        <br />
+                        {incompatibeTypeColumns.map((c) => (
+                          <div className="flex items-start space-x-2">
+                            <div className="w-[14px] h-[14px] flex items-center justify-center translate-y-[3px]">
+                              <div className="w-[6px] h-[6px] rounded-full bg-scale-1000" />
+                            </div>
+                            <p className="text-sm">
+                              "{c.name}": expected type of{' '}
+                              <span className="font-bold">{c.expectedFormat}</span> but found{' '}
+                              <span className="font-bold">{c.format}</span>
+                            </p>
+                          </div>
+                        ))}
+                      </p>
+                    </div>
+                  </div>
+                )}
                 {errors.map((error: any, idx: number) => {
                   const key = `import-error-${idx}`
                   const isExpanded = expandedErrors.includes(key)