From ba3ba8143465402a03043aedce91d043038ae860 Mon Sep 17 00:00:00 2001 From: Charis <26616127+charislam@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:32:22 -0500 Subject: [PATCH] feat: add full text search to docs (#19112) Feature Adds a full-text search endpoint for docs search, to provide more comprehensive and faster results. Each docs search now performs both FTS and embeddings-based search, and displays results as soon as the first set of results is returned. Refactor Slight refactor of the embeddings search to make only one trip to the database, and transfer less content in the JSON. This now uses a different function than the RAG step for Supabase AI. Bug fix Fixes a bug where some search results were being filtered out by the cmdk menu. Removes an obsolete guide. Content is now elsewhere and it messes up search results. --- apps/docs/pages/guides/database.mdx | 97 ----- .../ui/src/components/Command/CommandMenu.tsx | 15 +- .../Command/CommandMenuProvider.tsx | 14 +- .../ui/src/components/Command/DocsSearch.tsx | 334 ++++++++++++++---- supabase/functions/common/database-types.ts | 118 ++++++- supabase/functions/search-embeddings/index.ts | 125 +++++++ supabase/functions/search-fts/index.ts | 102 ++++++ ...20231121164837_modify_search_functions.sql | 88 +++++ 8 files changed, 710 insertions(+), 183 deletions(-) delete mode 100644 apps/docs/pages/guides/database.mdx create mode 100644 supabase/functions/search-embeddings/index.ts create mode 100644 supabase/functions/search-fts/index.ts create mode 100644 supabase/migrations/20231121164837_modify_search_functions.sql diff --git a/apps/docs/pages/guides/database.mdx b/apps/docs/pages/guides/database.mdx deleted file mode 100644 index b0c0c516538..00000000000 --- a/apps/docs/pages/guides/database.mdx +++ /dev/null @@ -1,97 +0,0 @@ -import Layout from '~/layouts/DefaultGuideLayout' - -export const meta = { - id: 'database', - title: 'Database', - description: 'Using Supabase to manage your data.', - subtitle: 'Using Supabase to manage your data.', - sidebar_label: 'Overview', -} - -Every Supabase project comes with a full [Postgres](https://www.postgresql.org/) database, a free and open source database which is considered one of the world's most stable and advanced databases. - -## Postgres or PostgreSQL? - -PostgreSQL the database was derived from the POSTGRES Project, a package written at the University of California at Berkeley in 1986. This package included a query language called "PostQUEL". - -In 1994, Postgres95 was built on top of POSTGRES code, adding an SQL language interpreter as a replacement for PostQUEL. Eventually, Postgres95 was renamed to PostgreSQL to reflect the SQL query capability. - -After this, many people referred to it as Postgres since it's less prone to confusion. Supabase is all about simplicity, so we also refer to it as Postgres. - -## Supabase Studio - -Postgres is known to be one of the most extensible databases available, so you can use it to build almost any application. The Supabase platform makes this easier with a simple-to-use dashboard called "Supabase Studio". - -### Table View - -You don't have to be a database expert to start using Supabase. Our table view makes Postgres as easy to use as a spreadsheet. - -![Table View.](/docs/img/table-view.png) - -### Relationships - -Dig into the relationships within your data. - - - -### Clone tables - -You can duplicate your tables, just like you would inside a spreadsheet. - - - -### The SQL Editor - -Supabase comes with a SQL Editor. You can also save your favorite queries to run later! - - - -### Additional features - -- Supabase extends Postgres with realtime functionality using our [Realtime Server](https://github.com/supabase/realtime). -- Every project is a full Postgres database, with `postgres` level access. -- Supabase manages your database backups. -- Import data directly from a CSV or excel spreadsheet. - - - -Database backups **do not** include objects stored via the Storage API, as the database only includes metadata about these objects. Restoring an old backup does not restore objects that have been deleted since then. - - - -### Extensions - -To expand the functionality of your Postgres database, you can use extensions. You can enable Postgres extensions with the click of a button within the Supabase dashboard. - - - -[Learn more](/docs/guides/database/extensions) about all the extensions provided on Supabase. - -## Next steps - -- Read more about [Postgres](https://www.postgresql.org/about/) -- Sign in: [supabase.com/dashboard](https://supabase.com/dashboard) - -export const Page = ({ children }) => - -export default Page diff --git a/packages/ui/src/components/Command/CommandMenu.tsx b/packages/ui/src/components/Command/CommandMenu.tsx index f0c65dcaa95..98b296b2002 100644 --- a/packages/ui/src/components/Command/CommandMenu.tsx +++ b/packages/ui/src/components/Command/CommandMenu.tsx @@ -65,9 +65,18 @@ interface CommandMenuProps { const CommandMenu = ({ projectRef }: CommandMenuProps) => { const router = useRouter() - const commandInputRef = useRef>(null) - const { isOpen, setIsOpen, search, setSearch, pages, setPages, currentPage, site, project } = - useCommandMenu() + const { + isOpen, + setIsOpen, + search, + setSearch, + pages, + setPages, + currentPage, + site, + project, + inputRef: commandInputRef, + } = useCommandMenu() const showCommandInput = !currentPage || !CHAT_ROUTES.includes(currentPage) // This function has been added to prevent the use of double quotes in the search docs input due to an issue with the cmdk-supabase module. This function can be removed when we transition to using cmdk. diff --git a/packages/ui/src/components/Command/CommandMenuProvider.tsx b/packages/ui/src/components/Command/CommandMenuProvider.tsx index f86a33279f1..61e9f0f34e2 100644 --- a/packages/ui/src/components/Command/CommandMenuProvider.tsx +++ b/packages/ui/src/components/Command/CommandMenuProvider.tsx @@ -1,5 +1,14 @@ import dynamic from 'next/dynamic' -import { createContext, PropsWithChildren, useContext, useEffect, useState } from 'react' +import { + createContext, + ElementRef, + PropsWithChildren, + useContext, + useEffect, + useRef, + useState, +} from 'react' +import { CommandInput } from './Command.utils' // `CommandMenu` is heavy - code split to reduce app bundle size const CommandMenu = dynamic(() => import('./CommandMenu'), { @@ -16,6 +25,7 @@ export interface CommandMenuContextValue { pages: string[] setPages: React.Dispatch> currentPage?: string + inputRef: React.RefObject site: 'studio' | 'docs' | 'website' /** @@ -81,6 +91,7 @@ const CommandMenuProvider = ({ const [isLoading, setIsLoading] = useState(false) const [search, setSearch] = useState('') const [pages, setPages] = useState([]) + const inputRef = useRef>(null) const currentPage = pages[pages.length - 1] const project = projectRef !== undefined ? { ref: projectRef, apiKeys, apiUrl } : undefined @@ -104,6 +115,7 @@ const CommandMenuProvider = ({ metadata, isOptedInToAI, saveGeneratedSQL, + inputRef, }} > {children} diff --git a/packages/ui/src/components/Command/DocsSearch.tsx b/packages/ui/src/components/Command/DocsSearch.tsx index 14223bea2ef..0455a23359e 100644 --- a/packages/ui/src/components/Command/DocsSearch.tsx +++ b/packages/ui/src/components/Command/DocsSearch.tsx @@ -1,5 +1,6 @@ import * as React from 'react' -import { useCallback, useEffect, useMemo, useState } from 'react' +import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react' +import { compact, debounce, uniqBy } from 'lodash' import { useSupabaseClient } from '@supabase/auth-helpers-react' import { @@ -15,7 +16,7 @@ import { } from 'ui' import { CommandGroup, CommandItem, CommandLabel, TextHighlighter } from './Command.utils' -import { debounce } from 'lodash' +const NUMBER_SOURCES = 2 const questions = [ 'How do I get started with Supabase?', @@ -32,20 +33,18 @@ export enum PageType { GithubDiscussion = 'github-discussions', } -export interface PageSection { - slug?: string - heading?: string +interface PageSection { + heading: string + slug: string } -export interface PageMetadata { - title: string - description?: string -} - -export interface PageResult { - type: PageType +export interface Page { + id: number path: string - meta: PageMetadata + type: PageType + title: string + subtitle: string | null + description: string | null sections: PageSection[] } @@ -64,68 +63,265 @@ const getDocsUrl = () => { return isLocal ? 'http://localhost:3001/docs' : 'https://supabase.com/docs' } +type SearchState = + | { + status: 'initial' + key: number + } + | { + status: 'loading' + key: number + staleResults: Page[] + } + | { + status: 'partialResults' + key: number + results: Page[] + } + | { + status: 'fullResults' + key: number + results: Page[] + } + | { + status: 'noResults' + key: number + } + | { + status: 'error' + key: number + message: string + } + +type Action = + | { + type: 'resultsReturned' + key: number + sourcesLoaded: number + results: unknown[] + } + | { + type: 'newSearchDispatched' + key: number + } + | { + type: 'reset' + key: number + } + | { + type: 'errored' + key: number + sourcesLoaded: number + message: string + } + +function reshapeResults(result: unknown): Page | null { + if (typeof result !== 'object' || result === null) { + return null + } + if (!('id' in result && 'path' in result && 'type' in result && 'title' in result)) { + return null + } + + const sections: PageSection[] = [] + if ( + 'headings' in result && + Array.isArray(result.headings) && + 'slugs' in result && + Array.isArray(result.slugs) && + result.headings.length === result.slugs.length + ) { + result.headings.forEach((heading, idx) => { + const slug = (result.slugs as Array)[idx] + if (heading && slug) { + sections.push({ heading, slug }) + } + }) + } + + return { + id: result.id as number, + path: result.path as string, + type: result.type as PageType, + title: result.title as string, + subtitle: 'subtitle' in result ? (result.subtitle as string) : null, + description: 'description' in result ? (result.description as string) : null, + sections, + } +} + +function reducer(state: SearchState, action: Action): SearchState { + // Ignore responses from outdated async functions + if (state.key > action.key) { + return state + } + switch (action.type) { + case 'resultsReturned': + const allSourcesLoaded = action.sourcesLoaded === NUMBER_SOURCES + const newResults = compact(action.results.map(reshapeResults)) + // If the new responses are from the same request as the current responses, + // combine the responses. + // If the new responses are from a fresher request, replace the current responses. + const allResults = + state.status === 'partialResults' && state.key === action.key + ? uniqBy(state.results.concat(newResults), (res) => res.id) + : newResults + if (!allResults.length) { + return allSourcesLoaded + ? { + status: 'noResults', + key: action.key, + } + : { + status: 'loading', + key: action.key, + staleResults: [], + } + } + return allSourcesLoaded + ? { + status: 'fullResults', + key: action.key, + results: allResults, + } + : { + status: 'partialResults', + key: action.key, + results: allResults, + } + case 'newSearchDispatched': + return { + status: 'loading', + key: action.key, + staleResults: 'results' in state ? state.results : [], + } + case 'reset': + return { + status: 'initial', + key: action.key, + } + case 'errored': + // At least one search has failed and all non-failing searches have come back empty + if (action.sourcesLoaded === NUMBER_SOURCES && !('results' in state)) { + return { + status: 'error', + key: action.key, + message: action.message, + } + } + return state + default: + return state + } +} + const DocsSearch = () => { - const [results, setResults] = useState() - const [hasSearchError, setHasSearchError] = useState(false) + const [state, dispatch] = useReducer(reducer, { status: 'initial', key: 0 }) const supabaseClient = useSupabaseClient() - const { isLoading, setIsLoading, search, setSearch } = useCommandMenu() + const { isLoading, setIsLoading, search, setSearch, inputRef } = useCommandMenu() + const key = useRef(0) + const initialLoad = useRef(true) + + const hasResults = + state.status === 'fullResults' || + state.status === 'partialResults' || + (state.status === 'loading' && state.staleResults.length > 0) const handleSearch = useCallback( async (query: string) => { - setHasSearchError(false) setIsLoading(true) - const { error, data: pageResults } = await supabaseClient.functions.invoke( - 'search-v2', - { - body: { query }, - } - ) + key.current += 1 + const localKey = key.current + dispatch({ type: 'newSearchDispatched', key: localKey }) - setIsLoading(false) + let sourcesLoaded = 0 - if (error) { - setIsLoading(false) - - setHasSearchError(true) - console.error(error) - return - } - - if (!Array.isArray(pageResults)) { - setIsLoading(false) - setHasSearchError(true) - console.error('Malformed response') - return - } - - setResults(pageResults) + const sources = ['search-fts', 'search-embeddings'] + sources.forEach((source) => { + supabaseClient.functions + .invoke(source, { body: { query } }) + .then(({ data: results, error }) => { + sourcesLoaded += 1 + if (error) { + dispatch({ + type: 'errored', + key: localKey, + sourcesLoaded, + message: error.message ?? '', + }) + } else { + dispatch({ + type: 'resultsReturned', + key: localKey, + sourcesLoaded, + results, + }) + } + if (sourcesLoaded === NUMBER_SOURCES) { + setIsLoading(false) + } + }) + }) }, [supabaseClient] ) function handleResetPrompt() { setSearch('') - setResults(undefined) - setHasSearchError(false) + + key.current += 1 + dispatch({ + type: 'reset', + key: key.current, + }) } const debouncedSearch = useMemo(() => debounce(handleSearch, 1000), [handleSearch]) - // Search initial query immediately (note - empty useEffect deps) useEffect(() => { + initialLoad.current = false + // search immediately if there is a search term on initial load if (search) { handleSearch(search) } }, []) - // TODO: can we do this w/o useEffect if query comes from context? useEffect(() => { - if (search) { + if (!search) { + // Clear search results if user deletes query + // and cancel any pending debounced searches + debouncedSearch.cancel() + key.current += 1 + dispatch({ type: 'reset', key: key.current }) + } else if (!initialLoad.current) { debouncedSearch(search) } }, [search]) + // Immediately run search if user presses enter + // and abort any debounced searches that are waiting + useEffect(() => { + const handleEnter = (event: KeyboardEvent) => { + if ( + event.key === 'Enter' && + document.activeElement === inputRef.current && + search && + // If there are results, cmdk menu will trigger navigation to the highlighted + // result on Enter, even though the active element is the input + !hasResults + ) { + event.preventDefault() + debouncedSearch.cancel() + handleSearch(search) + } + } + + inputRef.current?.addEventListener('keydown', handleEnter) + + return () => inputRef.current?.removeEventListener('keydown', handleEnter) + }, [search, hasResults]) + const ChevronArrow = () => ( { return ( <> - {results && - results.length > 0 && - results.map((page, i) => { - const pageSections = page.sections.filter((section) => !!section.heading) + {hasResults && + ('results' in state ? state.results : state.staleResults).map((page, i) => { return ( { openLink(page.type, formatPageUrl(page)) @@ -186,11 +382,11 @@ const DocsSearch = () => { {getPageIcon(page)}
- + - {page.meta.description && ( + {page.description && (
- +
)}
@@ -198,16 +394,16 @@ const DocsSearch = () => { - {pageSections.length > 0 && ( + {page.sections.length > 0 && (
- {pageSections.map((section, i) => ( + {page.sections.map((section, i) => ( { openLink(page.type, formatSectionUrl(page, section)) }} - key={`${page.meta.title}__${section.heading}-item-index-${i}`} - value={`${removeDoubleQuotes(page.meta.title)}__${removeDoubleQuotes( + key={`${page.title}__${section.heading}-item-index-${i}`} + value={`${search}-${removeDoubleQuotes(page.title)}__${removeDoubleQuotes( section.heading ?? '' )}-item-index-${i}`} type="block-link" @@ -218,7 +414,7 @@ const DocsSearch = () => { @@ -237,7 +433,7 @@ const DocsSearch = () => { ) })} - {!results && !hasSearchError && !isLoading && ( + {state.status === 'initial' && ( {questions.map((question) => { const key = question.replace(/\s+/g, '_') @@ -260,12 +456,12 @@ const DocsSearch = () => { })} )} - {isLoading && !results && ( + {state.status === 'loading' && state.staleResults.length === 0 && (

Searching for results

)} - {results && results.length === 0 && ( + {state.status === 'noResults' && (

No results found.

@@ -274,7 +470,7 @@ const DocsSearch = () => {
)} - {hasSearchError && ( + {state.status === 'error' && (

@@ -292,7 +488,7 @@ const DocsSearch = () => { export default DocsSearch -export function formatPageUrl(page: PageResult) { +export function formatPageUrl(page: Page) { const docsUrl = getDocsUrl() switch (page.type) { case PageType.Markdown: @@ -305,7 +501,7 @@ export function formatPageUrl(page: PageResult) { } } -export function formatSectionUrl(page: PageResult, section: PageSection) { +export function formatSectionUrl(page: Page, section: PageSection) { switch (page.type) { case PageType.Markdown: case PageType.GithubDiscussion: @@ -317,7 +513,7 @@ export function formatSectionUrl(page: PageResult, section: PageSection) { } } -export function getPageIcon(page: PageResult) { +export function getPageIcon(page: Page) { switch (page.type) { case PageType.Markdown: case PageType.Reference: @@ -329,7 +525,7 @@ export function getPageIcon(page: PageResult) { } } -export function getPageSectionIcon(page: PageResult) { +export function getPageSectionIcon(page: Page) { switch (page.type) { case PageType.Markdown: case PageType.Reference: diff --git a/supabase/functions/common/database-types.ts b/supabase/functions/common/database-types.ts index 7024bb35adb..80190d89f83 100644 --- a/supabase/functions/common/database-types.ts +++ b/supabase/functions/common/database-types.ts @@ -3,7 +3,7 @@ export type Json = | number | boolean | null - | { [key: string]: Json } + | { [key: string]: Json | undefined } | Json[] export interface Database { @@ -38,35 +38,50 @@ export interface Database { Row: { checksum: string | null id: number + last_refresh: string | null meta: Json | null parent_page_id: number | null path: string source: string | null type: string | null + version: string | null } Insert: { checksum?: string | null id?: number + last_refresh?: string | null meta?: Json | null parent_page_id?: number | null path: string source?: string | null type?: string | null + version?: string | null } Update: { checksum?: string | null id?: number + last_refresh?: string | null meta?: Json | null parent_page_id?: number | null path?: string source?: string | null type?: string | null + version?: string | null } + Relationships: [ + { + foreignKeyName: "page_parent_page_id_fkey" + columns: ["parent_page_id"] + referencedRelation: "page" + referencedColumns: ["id"] + } + ] } page_section: { Row: { content: string | null - embedding: unknown | null + embedding: string | null + fts_tokens: unknown | null heading: string | null id: number page_id: number @@ -75,7 +90,8 @@ export interface Database { } Insert: { content?: string | null - embedding?: unknown | null + embedding?: string | null + fts_tokens?: unknown | null heading?: string | null id?: number page_id: number @@ -84,19 +100,59 @@ export interface Database { } Update: { content?: string | null - embedding?: unknown | null + embedding?: string | null + fts_tokens?: unknown | null heading?: string | null id?: number page_id?: number slug?: string | null token_count?: number | null } + Relationships: [ + { + foreignKeyName: "page_section_page_id_fkey" + columns: ["page_id"] + referencedRelation: "page" + referencedColumns: ["id"] + } + ] } } Views: { [_ in never]: never } Functions: { + docs_search_embeddings: { + Args: { + embedding: string + match_threshold: number + } + Returns: { + id: number + path: string + type: string + title: string + subtitle: string + description: string + headings: string[] + slugs: string[] + }[] + } + docs_search_fts: { + Args: { + query: string + } + Returns: { + id: number + path: string + type: string + title: string + subtitle: string + description: string + headings: string[] + slugs: string[] + }[] + } get_page_parents: { Args: { page_id: number @@ -108,6 +164,12 @@ export interface Database { meta: Json }[] } + hnswhandler: { + Args: { + "": unknown + } + Returns: unknown + } ivfflathandler: { Args: { "": unknown @@ -116,7 +178,7 @@ export interface Database { } match_page_sections: { Args: { - embedding: unknown + embedding: string match_threshold: number match_count: number min_content_length: number @@ -132,13 +194,14 @@ export interface Database { } match_page_sections_v2: { Args: { - embedding: unknown + embedding: string match_threshold: number min_content_length: number } Returns: { content: string | null - embedding: unknown | null + embedding: string | null + fts_tokens: unknown | null heading: string | null id: number page_id: number @@ -150,29 +213,29 @@ export interface Database { Args: { "": number[] } - Returns: unknown + Returns: string } vector_dims: { Args: { - "": unknown + "": string } Returns: number } vector_norm: { Args: { - "": unknown + "": string } Returns: number } vector_out: { Args: { - "": unknown + "": string } Returns: unknown } vector_send: { Args: { - "": unknown + "": string } Returns: string } @@ -226,6 +289,14 @@ export interface Database { public?: boolean | null updated_at?: string | null } + Relationships: [ + { + foreignKeyName: "buckets_owner_fkey" + columns: ["owner"] + referencedRelation: "users" + referencedColumns: ["id"] + } + ] } migrations: { Row: { @@ -246,6 +317,7 @@ export interface Database { id?: number name?: string } + Relationships: [] } objects: { Row: { @@ -258,6 +330,7 @@ export interface Database { owner: string | null path_tokens: string[] | null updated_at: string | null + version: string | null } Insert: { bucket_id?: string | null @@ -269,6 +342,7 @@ export interface Database { owner?: string | null path_tokens?: string[] | null updated_at?: string | null + version?: string | null } Update: { bucket_id?: string | null @@ -280,13 +354,31 @@ export interface Database { owner?: string | null path_tokens?: string[] | null updated_at?: string | null + version?: string | null } + Relationships: [ + { + foreignKeyName: "objects_bucketId_fkey" + columns: ["bucket_id"] + referencedRelation: "buckets" + referencedColumns: ["id"] + } + ] } } Views: { [_ in never]: never } Functions: { + can_insert_object: { + Args: { + bucketid: string + name: string + owner: string + metadata: Json + } + Returns: undefined + } extension: { Args: { name: string @@ -303,7 +395,7 @@ export interface Database { Args: { name: string } - Returns: string[] + Returns: unknown } get_size_by_bucket: { Args: Record diff --git a/supabase/functions/search-embeddings/index.ts b/supabase/functions/search-embeddings/index.ts new file mode 100644 index 00000000000..1218a8c07f3 --- /dev/null +++ b/supabase/functions/search-embeddings/index.ts @@ -0,0 +1,125 @@ +import 'https://deno.land/x/xhr@0.2.1/mod.ts' +import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.8.0' +import { Configuration, OpenAIApi } from 'https://esm.sh/openai@3.1.0' +import { Database } from '../common/database-types.ts' +import { ApplicationError, UserError } from '../common/errors.ts' + +const openAiKey = Deno.env.get('OPENAI_KEY') +const supabaseUrl = Deno.env.get('SUPABASE_URL') +const supabaseServiceKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY') + +export const corsHeaders = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type', +} + +Deno.serve(async (req) => { + try { + // Handle CORS + if (req.method === 'OPTIONS') { + return new Response('ok', { headers: corsHeaders }) + } + + if (!openAiKey) { + throw new ApplicationError('Missing environment variable OPENAI_KEY') + } + + if (!supabaseUrl) { + throw new ApplicationError('Missing environment variable SUPABASE_URL') + } + + if (!supabaseServiceKey) { + throw new ApplicationError('Missing environment variable SUPABASE_SERVICE_ROLE_KEY') + } + + const requestData = await req.json() + + if (!requestData) { + throw new UserError('Missing request data') + } + + const { query } = requestData + + if (!query) { + throw new UserError('Missing query in request data') + } + + // Intentionally log the query + console.log({ query }) + + const sanitizedQuery = query.trim() + + const supabaseClient = createClient(supabaseUrl, supabaseServiceKey) + + const configuration = new Configuration({ apiKey: openAiKey }) + const openai = new OpenAIApi(configuration) + + // Moderate the content to comply with OpenAI T&C + const moderationResponse = await openai.createModeration({ input: sanitizedQuery }) + + const [results] = moderationResponse.data.results + + if (results.flagged) { + throw new UserError('Flagged content', { + flagged: true, + categories: results.categories, + }) + } + + const embeddingResponse = await openai.createEmbedding({ + model: 'text-embedding-ada-002', + input: sanitizedQuery.replaceAll('\n', ' '), + }) + + if (embeddingResponse.status !== 200) { + throw new ApplicationError('Failed to create embedding for question', embeddingResponse) + } + + const [{ embedding }] = embeddingResponse.data.data + const { error: matchError, data: pages } = await supabaseClient.rpc('docs_search_embeddings', { + embedding, + match_threshold: 0.78, + }) + + if (matchError) { + throw new ApplicationError('Failed to match page sections', matchError) + } + + return new Response(JSON.stringify(pages), { + headers: { + ...corsHeaders, + 'Content-Type': 'application/json', + }, + }) + } catch (err: unknown) { + if (err instanceof UserError) { + return new Response( + JSON.stringify({ + error: err.message, + data: err.data, + }), + { + status: 400, + headers: { ...corsHeaders, 'Content-Type': 'application/json' }, + } + ) + } else if (err instanceof ApplicationError) { + // Print out application errors with their additional data + console.error(`${err.message}: ${JSON.stringify(err.data)}`) + } else { + // Print out unexpected errors as is to help with debugging + console.error(err) + } + + // TODO: include more response info in debug environments + return new Response( + JSON.stringify({ + error: 'There was an error processing your request', + }), + { + status: 500, + headers: { ...corsHeaders, 'Content-Type': 'application/json' }, + } + ) + } +}) diff --git a/supabase/functions/search-fts/index.ts b/supabase/functions/search-fts/index.ts new file mode 100644 index 00000000000..def7c8c1f0c --- /dev/null +++ b/supabase/functions/search-fts/index.ts @@ -0,0 +1,102 @@ +import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.8.0' +import { Database } from '../common/database-types.ts' +import { ApplicationError, UserError } from '../common/errors.ts' + +const isDev = Deno.env.get('DENO_ENV') === 'development' + +const supabaseUrl = Deno.env.get('SUPABASE_URL') +const supabaseServiceKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY') + +function getCorsHeaders(req) { + const receivedOrigin = req.headers.get('Origin') + const resolvedOrigin = /^https:\/\/[^.\/]+-supabase.vercel.app$/.test(receivedOrigin) + ? receivedOrigin // Allow preview sites on Vercel to access function + : 'https://supabase.com' + + return { + 'Access-Control-Allow-Origin': isDev ? '*' : resolvedOrigin, + 'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type', + } +} + +Deno.serve(async (req) => { + try { + // Handle CORS + if (req.method === 'OPTIONS') { + return new Response('ok', { headers: getCorsHeaders(req) }) + } + + if (!supabaseUrl) { + throw new ApplicationError('Missing environment variable SUPABASE_URL') + } + + if (!supabaseServiceKey) { + throw new ApplicationError('Missing environment variable SUPABASE_SERVICE_ROLE_KEY') + } + const requestData = await req.json() + + if (!requestData) { + throw new UserError('Missing request data') + } + + const { query } = requestData + + if (!query) { + throw new UserError('Missing query in request data') + } + + // Intentionally log the query + console.log({ query }) + + const sanitizedQuery = query.trim() + + const supabaseClient = createClient(supabaseUrl, supabaseServiceKey) + + const { data: searchData, error: searchError } = await supabaseClient.rpc('docs_search_fts', { + query: sanitizedQuery, + }) + + if (searchError) { + throw new ApplicationError('Failed to find search results', searchError) + } + + return new Response(JSON.stringify(searchData), { + headers: { + ...getCorsHeaders(req), + 'Content-Type': 'application/json', + }, + }) + } catch (err) { + console.error(err) + + if (err instanceof UserError) { + return new Response( + JSON.stringify({ + error: err.message, + data: err.data, + }), + { + status: 400, + headers: { ...getCorsHeaders(req), 'Content-Type': 'application/json' }, + } + ) + } else if (err instanceof ApplicationError) { + // Print out application errors with their additional data + console.error(`${err.message}: ${JSON.stringify(err.data)}`) + } else { + // Print out unexpected errors as is to help with debugging + console.error(err) + } + + // TODO: include more response info in debug environments + return new Response( + JSON.stringify({ + error: 'There was an error processing your request', + }), + { + status: 500, + headers: { ...getCorsHeaders(req), 'Content-Type': 'application/json' }, + } + ) + } +}) diff --git a/supabase/migrations/20231121164837_modify_search_functions.sql b/supabase/migrations/20231121164837_modify_search_functions.sql new file mode 100644 index 00000000000..2f5624fa68e --- /dev/null +++ b/supabase/migrations/20231121164837_modify_search_functions.sql @@ -0,0 +1,88 @@ +alter table page_section +add column fts_tokens tsvector generated always as (to_tsvector('english', content)) stored; + +create index fts_search_index on page_section using gin(fts_tokens); + +create or replace function docs_search_fts(query text) +returns table ( + id int8, + path text, + type text, + title text, + subtitle text, + description text, + headings text[], + slugs text[] +) +language plpgsql +as $$ +#variable_conflict use_variable +begin + return query + with match as ( + select * + from page_section + where fts_tokens @@ websearch_to_tsquery(query) + limit 10 + ) + select + page.id, + page.path, + page.type, + page.meta ->> 'title' as title, + page.meta ->> 'subtitle' as title, + page.meta ->> 'description' as description, + array_agg(match.heading) as headings, + array_agg(match.slug) as slugs + from page + join match on match.page_id = page.id + group by page.id; +end; +$$; + +create or replace function docs_search_embeddings( + embedding vector(1536), + match_threshold float +) +returns table ( + id int8, + path text, + type text, + title text, + subtitle text, + description text, + headings text[], + slugs text[] +) +language plpgsql +as $$ +#variable_conflict use_variable +begin + return query + with match as( + select * + from page_section + -- The dot product is negative because of a Postgres limitation, so we negate it + where (page_section.embedding <#> embedding) * -1 > match_threshold + -- OpenAI embeddings are normalized to length 1, so + -- cosine similarity and dot product will produce the same results. + -- Using dot product which can be computed slightly faster. + -- + -- For the different syntaxes, see https://github.com/pgvector/pgvector + order by page_section.embedding <#> embedding + limit 10 + ) + select + page.id, + page.path, + page.type, + page.meta ->> 'title' as title, + page.meta ->> 'subtitle' as title, + page.meta ->> 'description' as description, + array_agg(match.heading) as headings, + array_agg(match.slug) as slugs + from page + join match on match.page_id = page.id + group by page.id; +end; +$$;