feat: add full text search to docs (#19112)

Feature

Adds a full-text search endpoint for docs search, to provide more comprehensive and faster results. Each docs search now performs both FTS and embeddings-based search, and displays results as soon as the first set of results is returned.

Refactor

Slight refactor of the embeddings search to make only one trip to the database, and transfer less content in the JSON. This now uses a different function than the RAG step for Supabase AI.

Bug fix

Fixes a bug where some search results were being filtered out by the cmdk menu.

Removes an obsolete guide. Content is now elsewhere and it messes up search results.

apps/docs/pages/guides/database.mdx

@@ -1,97 +0,0 @@
-import Layout from '~/layouts/DefaultGuideLayout'
-
-export const meta = {
-  id: 'database',
-  title: 'Database',
-  description: 'Using Supabase to manage your data.',
-  subtitle: 'Using Supabase to manage your data.',
-  sidebar_label: 'Overview',
-}
-
-Every Supabase project comes with a full [Postgres](https://www.postgresql.org/) database, a free and open source database which is considered one of the world's most stable and advanced databases.
-
-## Postgres or PostgreSQL?
-
-PostgreSQL the database was derived from the POSTGRES Project, a package written at the University of California at Berkeley in 1986. This package included a query language called "PostQUEL".
-
-In 1994, Postgres95 was built on top of POSTGRES code, adding an SQL language interpreter as a replacement for PostQUEL. Eventually, Postgres95 was renamed to PostgreSQL to reflect the SQL query capability.
-
-After this, many people referred to it as Postgres since it's less prone to confusion. Supabase is all about simplicity, so we also refer to it as Postgres.
-
-## Supabase Studio
-
-Postgres is known to be one of the most extensible databases available, so you can use it to build almost any application. The Supabase platform makes this easier with a simple-to-use dashboard called "Supabase Studio".
-
-### Table View
-
-You don't have to be a database expert to start using Supabase. Our table view makes Postgres as easy to use as a spreadsheet.
-
-![Table View.](/docs/img/table-view.png)
-
-### Relationships
-
-Dig into the relationships within your data.
-
-<video width="99%" loop muted playsInline controls={true}>
-  <source
-    src="https://xguihxuzqibwxjnimxev.supabase.co/storage/v1/object/public/videos/docs/relational-drilldown-zoom.mp4"
-    type="video/mp4"
-  />
-</video>
-
-### Clone tables
-
-You can duplicate your tables, just like you would inside a spreadsheet.
-
-<video width="99%" muted playsInline controls={true}>
-  <source
-    src="https://xguihxuzqibwxjnimxev.supabase.co/storage/v1/object/public/videos/docs/duplicate-tables.mp4"
-    type="video/mp4"
-  />
-</video>
-
-### The SQL Editor
-
-Supabase comes with a SQL Editor. You can also save your favorite queries to run later!
-
-<video width="99%" muted playsInline controls={true}>
-  <source
-    src="https://xguihxuzqibwxjnimxev.supabase.co/storage/v1/object/public/videos/docs/favorites.mp4"
-    type="video/mp4"
-  />
-</video>
-
-### Additional features
-
-- Supabase extends Postgres with realtime functionality using our [Realtime Server](https://github.com/supabase/realtime).
-- Every project is a full Postgres database, with `postgres` level access.
-- Supabase manages your database backups.
-- Import data directly from a CSV or excel spreadsheet.
-
-<Admonition type="note">
-
-Database backups **do not** include objects stored via the Storage API, as the database only includes metadata about these objects. Restoring an old backup does not restore objects that have been deleted since then.
-
-</Admonition>
-
-### Extensions
-
-To expand the functionality of your Postgres database, you can use extensions. You can enable Postgres extensions with the click of a button within the Supabase dashboard.
-
-<video width="99%" muted playsInline controls={true}>
-  <source
-    src="https://xguihxuzqibwxjnimxev.supabase.co/storage/v1/object/public/videos/docs/toggle-extensions.mp4"
-    type="video/mp4"
-  />
-</video>
-
-[Learn more](/docs/guides/database/extensions) about all the extensions provided on Supabase.
-
-## Next steps
-
-- Read more about [Postgres](https://www.postgresql.org/about/)
-- Sign in: [supabase.com/dashboard](https://supabase.com/dashboard)
-
-export const Page = ({ children }) => <Layout meta={meta} children={children} />
-
-export default Page

packages/ui/src/components/Command/CommandMenu.tsx

@@ -65,9 +65,18 @@ interface CommandMenuProps {
 const CommandMenu = ({ projectRef }: CommandMenuProps) => {
   const router = useRouter()
 
-  const commandInputRef = useRef<ElementRef<typeof CommandInput>>(null)
-  const { isOpen, setIsOpen, search, setSearch, pages, setPages, currentPage, site, project } =
-    useCommandMenu()
+  const {
+    isOpen,
+    setIsOpen,
+    search,
+    setSearch,
+    pages,
+    setPages,
+    currentPage,
+    site,
+    project,
+    inputRef: commandInputRef,
+  } = useCommandMenu()
   const showCommandInput = !currentPage || !CHAT_ROUTES.includes(currentPage)
 
   // This function has been added to prevent the use of double quotes in the search docs input due to an issue with the cmdk-supabase module. This function can be removed when we transition to using cmdk.

packages/ui/src/components/Command/CommandMenuProvider.tsx

@@ -1,5 +1,14 @@
 import dynamic from 'next/dynamic'
-import { createContext, PropsWithChildren, useContext, useEffect, useState } from 'react'
+import {
+  createContext,
+  ElementRef,
+  PropsWithChildren,
+  useContext,
+  useEffect,
+  useRef,
+  useState,
+} from 'react'
+import { CommandInput } from './Command.utils'
 
 // `CommandMenu` is heavy - code split to reduce app bundle size
 const CommandMenu = dynamic(() => import('./CommandMenu'), {
@@ -16,6 +25,7 @@ export interface CommandMenuContextValue {
   pages: string[]
   setPages: React.Dispatch<React.SetStateAction<string[]>>
   currentPage?: string
+  inputRef: React.RefObject<HTMLInputElement>
   site: 'studio' | 'docs' | 'website'
 
   /**
@@ -81,6 +91,7 @@ const CommandMenuProvider = ({
   const [isLoading, setIsLoading] = useState(false)
   const [search, setSearch] = useState('')
   const [pages, setPages] = useState<string[]>([])
+  const inputRef = useRef<ElementRef<typeof CommandInput>>(null)
   const currentPage = pages[pages.length - 1]
 
   const project = projectRef !== undefined ? { ref: projectRef, apiKeys, apiUrl } : undefined
@@ -104,6 +115,7 @@ const CommandMenuProvider = ({
         metadata,
         isOptedInToAI,
         saveGeneratedSQL,
+        inputRef,
       }}
     >
       {children}

packages/ui/src/components/Command/DocsSearch.tsx

@@ -1,5 +1,6 @@
 import * as React from 'react'
-import { useCallback, useEffect, useMemo, useState } from 'react'
+import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react'
+import { compact, debounce, uniqBy } from 'lodash'
 
 import { useSupabaseClient } from '@supabase/auth-helpers-react'
 import {
@@ -15,7 +16,7 @@ import {
 } from 'ui'
 import { CommandGroup, CommandItem, CommandLabel, TextHighlighter } from './Command.utils'
 
-import { debounce } from 'lodash'
+const NUMBER_SOURCES = 2
 
 const questions = [
   'How do I get started with Supabase?',
@@ -32,20 +33,18 @@ export enum PageType {
   GithubDiscussion = 'github-discussions',
 }
 
-export interface PageSection {
-  slug?: string
-  heading?: string
+interface PageSection {
+  heading: string
+  slug: string
 }
 
-export interface PageMetadata {
-  title: string
-  description?: string
-}
-
-export interface PageResult {
-  type: PageType
+export interface Page {
+  id: number
   path: string
-  meta: PageMetadata
+  type: PageType
+  title: string
+  subtitle: string | null
+  description: string | null
   sections: PageSection[]
 }
 
@@ -64,68 +63,265 @@ const getDocsUrl = () => {
   return isLocal ? 'http://localhost:3001/docs' : 'https://supabase.com/docs'
 }
 
+type SearchState =
+  | {
+      status: 'initial'
+      key: number
+    }
+  | {
+      status: 'loading'
+      key: number
+      staleResults: Page[]
+    }
+  | {
+      status: 'partialResults'
+      key: number
+      results: Page[]
+    }
+  | {
+      status: 'fullResults'
+      key: number
+      results: Page[]
+    }
+  | {
+      status: 'noResults'
+      key: number
+    }
+  | {
+      status: 'error'
+      key: number
+      message: string
+    }
+
+type Action =
+  | {
+      type: 'resultsReturned'
+      key: number
+      sourcesLoaded: number
+      results: unknown[]
+    }
+  | {
+      type: 'newSearchDispatched'
+      key: number
+    }
+  | {
+      type: 'reset'
+      key: number
+    }
+  | {
+      type: 'errored'
+      key: number
+      sourcesLoaded: number
+      message: string
+    }
+
+function reshapeResults(result: unknown): Page | null {
+  if (typeof result !== 'object' || result === null) {
+    return null
+  }
+  if (!('id' in result && 'path' in result && 'type' in result && 'title' in result)) {
+    return null
+  }
+
+  const sections: PageSection[] = []
+  if (
+    'headings' in result &&
+    Array.isArray(result.headings) &&
+    'slugs' in result &&
+    Array.isArray(result.slugs) &&
+    result.headings.length === result.slugs.length
+  ) {
+    result.headings.forEach((heading, idx) => {
+      const slug = (result.slugs as Array<string>)[idx]
+      if (heading && slug) {
+        sections.push({ heading, slug })
+      }
+    })
+  }
+
+  return {
+    id: result.id as number,
+    path: result.path as string,
+    type: result.type as PageType,
+    title: result.title as string,
+    subtitle: 'subtitle' in result ? (result.subtitle as string) : null,
+    description: 'description' in result ? (result.description as string) : null,
+    sections,
+  }
+}
+
+function reducer(state: SearchState, action: Action): SearchState {
+  // Ignore responses from outdated async functions
+  if (state.key > action.key) {
+    return state
+  }
+  switch (action.type) {
+    case 'resultsReturned':
+      const allSourcesLoaded = action.sourcesLoaded === NUMBER_SOURCES
+      const newResults = compact(action.results.map(reshapeResults))
+      // If the new responses are from the same request as the current responses,
+      // combine the responses.
+      // If the new responses are from a fresher request, replace the current responses.
+      const allResults =
+        state.status === 'partialResults' && state.key === action.key
+          ? uniqBy(state.results.concat(newResults), (res) => res.id)
+          : newResults
+      if (!allResults.length) {
+        return allSourcesLoaded
+          ? {
+              status: 'noResults',
+              key: action.key,
+            }
+          : {
+              status: 'loading',
+              key: action.key,
+              staleResults: [],
+            }
+      }
+      return allSourcesLoaded
+        ? {
+            status: 'fullResults',
+            key: action.key,
+            results: allResults,
+          }
+        : {
+            status: 'partialResults',
+            key: action.key,
+            results: allResults,
+          }
+    case 'newSearchDispatched':
+      return {
+        status: 'loading',
+        key: action.key,
+        staleResults: 'results' in state ? state.results : [],
+      }
+    case 'reset':
+      return {
+        status: 'initial',
+        key: action.key,
+      }
+    case 'errored':
+      // At least one search has failed and all non-failing searches have come back empty
+      if (action.sourcesLoaded === NUMBER_SOURCES && !('results' in state)) {
+        return {
+          status: 'error',
+          key: action.key,
+          message: action.message,
+        }
+      }
+      return state
+    default:
+      return state
+  }
+}
+
 const DocsSearch = () => {
-  const [results, setResults] = useState<PageResult[]>()
-  const [hasSearchError, setHasSearchError] = useState(false)
+  const [state, dispatch] = useReducer(reducer, { status: 'initial', key: 0 })
   const supabaseClient = useSupabaseClient()
-  const { isLoading, setIsLoading, search, setSearch } = useCommandMenu()
+  const { isLoading, setIsLoading, search, setSearch, inputRef } = useCommandMenu()
+  const key = useRef(0)
+  const initialLoad = useRef(true)
+
+  const hasResults =
+    state.status === 'fullResults' ||
+    state.status === 'partialResults' ||
+    (state.status === 'loading' && state.staleResults.length > 0)
 
   const handleSearch = useCallback(
     async (query: string) => {
-      setHasSearchError(false)
       setIsLoading(true)
 
-      const { error, data: pageResults } = await supabaseClient.functions.invoke<PageResult[]>(
-        'search-v2',
-        {
-          body: { query },
-        }
-      )
+      key.current += 1
+      const localKey = key.current
+      dispatch({ type: 'newSearchDispatched', key: localKey })
 
-      setIsLoading(false)
+      let sourcesLoaded = 0
 
-      if (error) {
-        setIsLoading(false)
-
-        setHasSearchError(true)
-        console.error(error)
-        return
-      }
-
-      if (!Array.isArray(pageResults)) {
-        setIsLoading(false)
-        setHasSearchError(true)
-        console.error('Malformed response')
-        return
-      }
-
-      setResults(pageResults)
+      const sources = ['search-fts', 'search-embeddings']
+      sources.forEach((source) => {
+        supabaseClient.functions
+          .invoke(source, { body: { query } })
+          .then(({ data: results, error }) => {
+            sourcesLoaded += 1
+            if (error) {
+              dispatch({
+                type: 'errored',
+                key: localKey,
+                sourcesLoaded,
+                message: error.message ?? '',
+              })
+            } else {
+              dispatch({
+                type: 'resultsReturned',
+                key: localKey,
+                sourcesLoaded,
+                results,
+              })
+            }
+            if (sourcesLoaded === NUMBER_SOURCES) {
+              setIsLoading(false)
+            }
+          })
+      })
     },
     [supabaseClient]
   )
 
   function handleResetPrompt() {
     setSearch('')
-    setResults(undefined)
-    setHasSearchError(false)
+
+    key.current += 1
+    dispatch({
+      type: 'reset',
+      key: key.current,
+    })
   }
 
   const debouncedSearch = useMemo(() => debounce(handleSearch, 1000), [handleSearch])
 
-  // Search initial query immediately (note - empty useEffect deps)
   useEffect(() => {
+    initialLoad.current = false
+    // search immediately if there is a search term on initial load
     if (search) {
       handleSearch(search)
     }
   }, [])
 
-  // TODO: can we do this w/o useEffect if query comes from context?
   useEffect(() => {
-    if (search) {
+    if (!search) {
+      // Clear search results if user deletes query
+      // and cancel any pending debounced searches
+      debouncedSearch.cancel()
+      key.current += 1
+      dispatch({ type: 'reset', key: key.current })
+    } else if (!initialLoad.current) {
       debouncedSearch(search)
     }
   }, [search])
 
+  // Immediately run search if user presses enter
+  // and abort any debounced searches that are waiting
+  useEffect(() => {
+    const handleEnter = (event: KeyboardEvent) => {
+      if (
+        event.key === 'Enter' &&
+        document.activeElement === inputRef.current &&
+        search &&
+        // If there are results, cmdk menu will trigger navigation to the highlighted
+        // result on Enter, even though the active element is the input
+        !hasResults
+      ) {
+        event.preventDefault()
+        debouncedSearch.cancel()
+        handleSearch(search)
+      }
+    }
+
+    inputRef.current?.addEventListener('keydown', handleEnter)
+
+    return () => inputRef.current?.removeEventListener('keydown', handleEnter)
+  }, [search, hasResults])
+
   const ChevronArrow = () => (
     <IconChevronRight
       strokeWidth={1.5}
@@ -164,19 +360,19 @@ const DocsSearch = () => {
 
   return (
     <>
-      {results &&
-        results.length > 0 &&
-        results.map((page, i) => {
-          const pageSections = page.sections.filter((section) => !!section.heading)
+      {hasResults &&
+        ('results' in state ? state.results : state.staleResults).map((page, i) => {
           return (
             <CommandGroup
               heading=""
-              key={`${page.meta.title}-group-index-${i}`}
-              value={`${page.meta.title}-group-index-${i}`}
+              key={`${page.title}-group-index-${i}`}
+              // Adding the search term here is a hack to prevent the cmdk menu
+              // filter from filtering out search results
+              value={`${search}-${page.title}-group-index-${i}`}
             >
               <CommandItem
-                key={`${page.meta.title}-item-index-${i}`}
-                value={`${removeDoubleQuotes(page.meta.title)}-item-index-${i}`}
+                key={`${page.title}-item-index-${i}`}
+                value={`${search}-${removeDoubleQuotes(page.title)}-item-index-${i}`}
                 type="block-link"
                 onSelect={() => {
                   openLink(page.type, formatPageUrl(page))
@@ -186,11 +382,11 @@ const DocsSearch = () => {
                   <IconContainer>{getPageIcon(page)}</IconContainer>
                   <div className="flex flex-col gap-0">
                     <CommandLabel>
-                      <TextHighlighter text={page.meta.title} query={search} />
+                      <TextHighlighter text={page.title} query={search} />
                     </CommandLabel>
-                    {page.meta.description && (
+                    {page.description && (
                       <div className="text-xs text-foreground-muted">
-                        <TextHighlighter text={page.meta.description} query={search} />
+                        <TextHighlighter text={page.description} query={search} />
                       </div>
                     )}
                   </div>
@@ -198,16 +394,16 @@ const DocsSearch = () => {
 
                 <ChevronArrow />
               </CommandItem>
-              {pageSections.length > 0 && (
+              {page.sections.length > 0 && (
                 <div className="border-l border-default ml-3 pt-3">
-                  {pageSections.map((section, i) => (
+                  {page.sections.map((section, i) => (
                     <CommandItem
                       className="ml-3 mb-3"
                       onSelect={() => {
                         openLink(page.type, formatSectionUrl(page, section))
                       }}
-                      key={`${page.meta.title}__${section.heading}-item-index-${i}`}
-                      value={`${removeDoubleQuotes(page.meta.title)}__${removeDoubleQuotes(
+                      key={`${page.title}__${section.heading}-item-index-${i}`}
+                      value={`${search}-${removeDoubleQuotes(page.title)}__${removeDoubleQuotes(
                         section.heading ?? ''
                       )}-item-index-${i}`}
                       type="block-link"
@@ -218,7 +414,7 @@ const DocsSearch = () => {
                           <cite>
                             <TextHighlighter
                               className="not-italic text-xs rounded-full px-2 py-1 bg-overlay-hover text-foreground"
-                              text={page.meta.title}
+                              text={page.title}
                               query={search}
                             />
                           </cite>
@@ -237,7 +433,7 @@ const DocsSearch = () => {
             </CommandGroup>
           )
         })}
-      {!results && !hasSearchError && !isLoading && (
+      {state.status === 'initial' && (
         <CommandGroup>
           {questions.map((question) => {
             const key = question.replace(/\s+/g, '_')
@@ -260,12 +456,12 @@ const DocsSearch = () => {
           })}
         </CommandGroup>
       )}
-      {isLoading && !results && (
+      {state.status === 'loading' && state.staleResults.length === 0 && (
         <div className="p-6 grid gap-6 my-4">
           <p className="text-lg text-foreground-muted text-center">Searching for results</p>
         </div>
       )}
-      {results && results.length === 0 && (
+      {state.status === 'noResults' && (
         <div className="p-6 flex flex-col items-center gap-6 mt-4 text-foreground-light">
           <IconAlertTriangle strokeWidth={1.5} size={40} />
           <p className="text-lg text-center">No results found.</p>
@@ -274,7 +470,7 @@ const DocsSearch = () => {
           </Button>
         </div>
       )}
-      {hasSearchError && (
+      {state.status === 'error' && (
         <div className="p-6 flex flex-col items-center gap-6 mt-4">
           <IconAlertTriangle strokeWidth={1.5} size={40} />
           <p className="text-lg text-center">
@@ -292,7 +488,7 @@ const DocsSearch = () => {
 
 export default DocsSearch
 
-export function formatPageUrl(page: PageResult) {
+export function formatPageUrl(page: Page) {
   const docsUrl = getDocsUrl()
   switch (page.type) {
     case PageType.Markdown:
@@ -305,7 +501,7 @@ export function formatPageUrl(page: PageResult) {
   }
 }
 
-export function formatSectionUrl(page: PageResult, section: PageSection) {
+export function formatSectionUrl(page: Page, section: PageSection) {
   switch (page.type) {
     case PageType.Markdown:
     case PageType.GithubDiscussion:
@@ -317,7 +513,7 @@ export function formatSectionUrl(page: PageResult, section: PageSection) {
   }
 }
 
-export function getPageIcon(page: PageResult) {
+export function getPageIcon(page: Page) {
   switch (page.type) {
     case PageType.Markdown:
     case PageType.Reference:
@@ -329,7 +525,7 @@ export function getPageIcon(page: PageResult) {
   }
 }
 
-export function getPageSectionIcon(page: PageResult) {
+export function getPageSectionIcon(page: Page) {
   switch (page.type) {
     case PageType.Markdown:
     case PageType.Reference:

supabase/functions/common/database-types.ts

@@ -3,7 +3,7 @@ export type Json =
   | number
   | boolean
   | null
-  | { [key: string]: Json }
+  | { [key: string]: Json | undefined }
   | Json[]
 
 export interface Database {
@@ -38,35 +38,50 @@ export interface Database {
         Row: {
           checksum: string | null
           id: number
+          last_refresh: string | null
           meta: Json | null
           parent_page_id: number | null
           path: string
           source: string | null
           type: string | null
+          version: string | null
         }
         Insert: {
           checksum?: string | null
           id?: number
+          last_refresh?: string | null
           meta?: Json | null
           parent_page_id?: number | null
           path: string
           source?: string | null
           type?: string | null
+          version?: string | null
         }
         Update: {
           checksum?: string | null
           id?: number
+          last_refresh?: string | null
           meta?: Json | null
           parent_page_id?: number | null
           path?: string
           source?: string | null
           type?: string | null
+          version?: string | null
         }
+        Relationships: [
+          {
+            foreignKeyName: "page_parent_page_id_fkey"
+            columns: ["parent_page_id"]
+            referencedRelation: "page"
+            referencedColumns: ["id"]
+          }
+        ]
       }
       page_section: {
         Row: {
           content: string | null
-          embedding: unknown | null
+          embedding: string | null
+          fts_tokens: unknown | null
           heading: string | null
           id: number
           page_id: number
@@ -75,7 +90,8 @@ export interface Database {
         }
         Insert: {
           content?: string | null
-          embedding?: unknown | null
+          embedding?: string | null
+          fts_tokens?: unknown | null
           heading?: string | null
           id?: number
           page_id: number
@@ -84,19 +100,59 @@ export interface Database {
         }
         Update: {
           content?: string | null
-          embedding?: unknown | null
+          embedding?: string | null
+          fts_tokens?: unknown | null
           heading?: string | null
           id?: number
           page_id?: number
           slug?: string | null
           token_count?: number | null
         }
+        Relationships: [
+          {
+            foreignKeyName: "page_section_page_id_fkey"
+            columns: ["page_id"]
+            referencedRelation: "page"
+            referencedColumns: ["id"]
+          }
+        ]
       }
     }
     Views: {
       [_ in never]: never
     }
     Functions: {
+      docs_search_embeddings: {
+        Args: {
+          embedding: string
+          match_threshold: number
+        }
+        Returns: {
+          id: number
+          path: string
+          type: string
+          title: string
+          subtitle: string
+          description: string
+          headings: string[]
+          slugs: string[]
+        }[]
+      }
+      docs_search_fts: {
+        Args: {
+          query: string
+        }
+        Returns: {
+          id: number
+          path: string
+          type: string
+          title: string
+          subtitle: string
+          description: string
+          headings: string[]
+          slugs: string[]
+        }[]
+      }
       get_page_parents: {
         Args: {
           page_id: number
@@ -108,6 +164,12 @@ export interface Database {
           meta: Json
         }[]
       }
+      hnswhandler: {
+        Args: {
+          "": unknown
+        }
+        Returns: unknown
+      }
       ivfflathandler: {
         Args: {
           "": unknown
@@ -116,7 +178,7 @@ export interface Database {
       }
       match_page_sections: {
         Args: {
-          embedding: unknown
+          embedding: string
           match_threshold: number
           match_count: number
           min_content_length: number
@@ -132,13 +194,14 @@ export interface Database {
       }
       match_page_sections_v2: {
         Args: {
-          embedding: unknown
+          embedding: string
           match_threshold: number
           min_content_length: number
         }
         Returns: {
           content: string | null
-          embedding: unknown | null
+          embedding: string | null
+          fts_tokens: unknown | null
           heading: string | null
           id: number
           page_id: number
@@ -150,29 +213,29 @@ export interface Database {
         Args: {
           "": number[]
         }
-        Returns: unknown
+        Returns: string
       }
       vector_dims: {
         Args: {
-          "": unknown
+          "": string
         }
         Returns: number
       }
       vector_norm: {
         Args: {
-          "": unknown
+          "": string
         }
         Returns: number
       }
       vector_out: {
         Args: {
-          "": unknown
+          "": string
         }
         Returns: unknown
       }
       vector_send: {
         Args: {
-          "": unknown
+          "": string
         }
         Returns: string
       }
@@ -226,6 +289,14 @@ export interface Database {
           public?: boolean | null
           updated_at?: string | null
         }
+        Relationships: [
+          {
+            foreignKeyName: "buckets_owner_fkey"
+            columns: ["owner"]
+            referencedRelation: "users"
+            referencedColumns: ["id"]
+          }
+        ]
       }
       migrations: {
         Row: {
@@ -246,6 +317,7 @@ export interface Database {
           id?: number
           name?: string
         }
+        Relationships: []
       }
       objects: {
         Row: {
@@ -258,6 +330,7 @@ export interface Database {
           owner: string | null
           path_tokens: string[] | null
           updated_at: string | null
+          version: string | null
         }
         Insert: {
           bucket_id?: string | null
@@ -269,6 +342,7 @@ export interface Database {
           owner?: string | null
           path_tokens?: string[] | null
           updated_at?: string | null
+          version?: string | null
         }
         Update: {
           bucket_id?: string | null
@@ -280,13 +354,31 @@ export interface Database {
           owner?: string | null
           path_tokens?: string[] | null
           updated_at?: string | null
+          version?: string | null
         }
+        Relationships: [
+          {
+            foreignKeyName: "objects_bucketId_fkey"
+            columns: ["bucket_id"]
+            referencedRelation: "buckets"
+            referencedColumns: ["id"]
+          }
+        ]
       }
     }
     Views: {
       [_ in never]: never
     }
     Functions: {
+      can_insert_object: {
+        Args: {
+          bucketid: string
+          name: string
+          owner: string
+          metadata: Json
+        }
+        Returns: undefined
+      }
       extension: {
         Args: {
           name: string
@@ -303,7 +395,7 @@ export interface Database {
         Args: {
           name: string
         }
-        Returns: string[]
+        Returns: unknown
       }
       get_size_by_bucket: {
         Args: Record<PropertyKey, never>

supabase/functions/search-embeddings/index.ts

@@ -0,0 +1,125 @@
+import 'https://deno.land/x/xhr@0.2.1/mod.ts'
+import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.8.0'
+import { Configuration, OpenAIApi } from 'https://esm.sh/openai@3.1.0'
+import { Database } from '../common/database-types.ts'
+import { ApplicationError, UserError } from '../common/errors.ts'
+
+const openAiKey = Deno.env.get('OPENAI_KEY')
+const supabaseUrl = Deno.env.get('SUPABASE_URL')
+const supabaseServiceKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY')
+
+export const corsHeaders = {
+  'Access-Control-Allow-Origin': '*',
+  'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
+}
+
+Deno.serve(async (req) => {
+  try {
+    // Handle CORS
+    if (req.method === 'OPTIONS') {
+      return new Response('ok', { headers: corsHeaders })
+    }
+
+    if (!openAiKey) {
+      throw new ApplicationError('Missing environment variable OPENAI_KEY')
+    }
+
+    if (!supabaseUrl) {
+      throw new ApplicationError('Missing environment variable SUPABASE_URL')
+    }
+
+    if (!supabaseServiceKey) {
+      throw new ApplicationError('Missing environment variable SUPABASE_SERVICE_ROLE_KEY')
+    }
+
+    const requestData = await req.json()
+
+    if (!requestData) {
+      throw new UserError('Missing request data')
+    }
+
+    const { query } = requestData
+
+    if (!query) {
+      throw new UserError('Missing query in request data')
+    }
+
+    // Intentionally log the query
+    console.log({ query })
+
+    const sanitizedQuery = query.trim()
+
+    const supabaseClient = createClient<Database>(supabaseUrl, supabaseServiceKey)
+
+    const configuration = new Configuration({ apiKey: openAiKey })
+    const openai = new OpenAIApi(configuration)
+
+    // Moderate the content to comply with OpenAI T&C
+    const moderationResponse = await openai.createModeration({ input: sanitizedQuery })
+
+    const [results] = moderationResponse.data.results
+
+    if (results.flagged) {
+      throw new UserError('Flagged content', {
+        flagged: true,
+        categories: results.categories,
+      })
+    }
+
+    const embeddingResponse = await openai.createEmbedding({
+      model: 'text-embedding-ada-002',
+      input: sanitizedQuery.replaceAll('\n', ' '),
+    })
+
+    if (embeddingResponse.status !== 200) {
+      throw new ApplicationError('Failed to create embedding for question', embeddingResponse)
+    }
+
+    const [{ embedding }] = embeddingResponse.data.data
+    const { error: matchError, data: pages } = await supabaseClient.rpc('docs_search_embeddings', {
+      embedding,
+      match_threshold: 0.78,
+    })
+
+    if (matchError) {
+      throw new ApplicationError('Failed to match page sections', matchError)
+    }
+
+    return new Response(JSON.stringify(pages), {
+      headers: {
+        ...corsHeaders,
+        'Content-Type': 'application/json',
+      },
+    })
+  } catch (err: unknown) {
+    if (err instanceof UserError) {
+      return new Response(
+        JSON.stringify({
+          error: err.message,
+          data: err.data,
+        }),
+        {
+          status: 400,
+          headers: { ...corsHeaders, 'Content-Type': 'application/json' },
+        }
+      )
+    } else if (err instanceof ApplicationError) {
+      // Print out application errors with their additional data
+      console.error(`${err.message}: ${JSON.stringify(err.data)}`)
+    } else {
+      // Print out unexpected errors as is to help with debugging
+      console.error(err)
+    }
+
+    // TODO: include more response info in debug environments
+    return new Response(
+      JSON.stringify({
+        error: 'There was an error processing your request',
+      }),
+      {
+        status: 500,
+        headers: { ...corsHeaders, 'Content-Type': 'application/json' },
+      }
+    )
+  }
+})

supabase/functions/search-fts/index.ts

@@ -0,0 +1,102 @@
+import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.8.0'
+import { Database } from '../common/database-types.ts'
+import { ApplicationError, UserError } from '../common/errors.ts'
+
+const isDev = Deno.env.get('DENO_ENV') === 'development'
+
+const supabaseUrl = Deno.env.get('SUPABASE_URL')
+const supabaseServiceKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY')
+
+function getCorsHeaders(req) {
+  const receivedOrigin = req.headers.get('Origin')
+  const resolvedOrigin = /^https:\/\/[^.\/]+-supabase.vercel.app$/.test(receivedOrigin)
+    ? receivedOrigin // Allow preview sites on Vercel to access function
+    : 'https://supabase.com'
+
+  return {
+    'Access-Control-Allow-Origin': isDev ? '*' : resolvedOrigin,
+    'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
+  }
+}
+
+Deno.serve(async (req) => {
+  try {
+    // Handle CORS
+    if (req.method === 'OPTIONS') {
+      return new Response('ok', { headers: getCorsHeaders(req) })
+    }
+
+    if (!supabaseUrl) {
+      throw new ApplicationError('Missing environment variable SUPABASE_URL')
+    }
+
+    if (!supabaseServiceKey) {
+      throw new ApplicationError('Missing environment variable SUPABASE_SERVICE_ROLE_KEY')
+    }
+    const requestData = await req.json()
+
+    if (!requestData) {
+      throw new UserError('Missing request data')
+    }
+
+    const { query } = requestData
+
+    if (!query) {
+      throw new UserError('Missing query in request data')
+    }
+
+    // Intentionally log the query
+    console.log({ query })
+
+    const sanitizedQuery = query.trim()
+
+    const supabaseClient = createClient<Database>(supabaseUrl, supabaseServiceKey)
+
+    const { data: searchData, error: searchError } = await supabaseClient.rpc('docs_search_fts', {
+      query: sanitizedQuery,
+    })
+
+    if (searchError) {
+      throw new ApplicationError('Failed to find search results', searchError)
+    }
+
+    return new Response(JSON.stringify(searchData), {
+      headers: {
+        ...getCorsHeaders(req),
+        'Content-Type': 'application/json',
+      },
+    })
+  } catch (err) {
+    console.error(err)
+
+    if (err instanceof UserError) {
+      return new Response(
+        JSON.stringify({
+          error: err.message,
+          data: err.data,
+        }),
+        {
+          status: 400,
+          headers: { ...getCorsHeaders(req), 'Content-Type': 'application/json' },
+        }
+      )
+    } else if (err instanceof ApplicationError) {
+      // Print out application errors with their additional data
+      console.error(`${err.message}: ${JSON.stringify(err.data)}`)
+    } else {
+      // Print out unexpected errors as is to help with debugging
+      console.error(err)
+    }
+
+    // TODO: include more response info in debug environments
+    return new Response(
+      JSON.stringify({
+        error: 'There was an error processing your request',
+      }),
+      {
+        status: 500,
+        headers: { ...getCorsHeaders(req), 'Content-Type': 'application/json' },
+      }
+    )
+  }
+})

supabase/migrations/20231121164837_modify_search_functions.sql

@@ -0,0 +1,88 @@
+alter table page_section
+add column fts_tokens tsvector generated always as (to_tsvector('english', content)) stored;
+
+create index fts_search_index on page_section using gin(fts_tokens);
+
+create or replace function docs_search_fts(query text)
+returns table (
+	id int8,
+	path text,
+	type text,
+	title text,
+	subtitle text,
+	description text,
+	headings text[],
+	slugs text[]
+)
+language plpgsql
+as $$
+#variable_conflict use_variable
+begin
+	return query
+	with match as (
+		select *
+		from page_section
+		where fts_tokens @@ websearch_to_tsquery(query)
+		limit 10
+	)
+	select
+		page.id,
+		page.path,
+		page.type,
+		page.meta ->> 'title' as title,
+		page.meta ->> 'subtitle' as title,
+		page.meta ->> 'description' as description,
+		array_agg(match.heading) as headings,
+		array_agg(match.slug) as slugs
+	from page
+	join match on match.page_id = page.id
+	group by page.id;
+end;
+$$;
+
+create or replace function docs_search_embeddings(
+	embedding vector(1536),
+	match_threshold float
+)
+returns table (
+	id int8,
+	path text,
+	type text,
+	title text,
+	subtitle text,
+	description text,
+	headings text[],
+	slugs text[]
+)
+language plpgsql
+as $$
+#variable_conflict use_variable
+begin
+	return query
+	with match as(
+		select *
+		from page_section
+		-- The dot product is negative because of a Postgres limitation, so we negate it
+		where (page_section.embedding <#> embedding) * -1 > match_threshold	
+		-- OpenAI embeddings are normalized to length 1, so
+		-- cosine similarity and dot product will produce the same results.
+		-- Using dot product which can be computed slightly faster.
+		--
+		-- For the different syntaxes, see https://github.com/pgvector/pgvector
+		order by page_section.embedding <#> embedding
+		limit 10
+	)
+	select
+		page.id,
+		page.path,
+		page.type,
+		page.meta ->> 'title' as title,
+		page.meta ->> 'subtitle' as title,
+		page.meta ->> 'description' as description,
+		array_agg(match.heading) as headings,
+		array_agg(match.slug) as slugs
+	from page
+	join match on match.page_id = page.id
+	group by page.id;
+end;
+$$;