From ce14f305ac6e28d6092f0fbec6717c5e791af95e Mon Sep 17 00:00:00 2001 From: Charis <26616127+charislam@users.noreply.github.com> Date: Mon, 14 Jul 2025 16:29:01 -0400 Subject: [PATCH] feat: hybrid search (#37127) Implement hybrid search for the /docs/api/graphql searchDocs endpoint. Prepend a more descriptive title and introduction to database advisor docs so they rank more highly when directly searched for. --- .../app/api/graphql/tests/searchDocs.test.ts | 8 +-- apps/docs/lib/supabase.ts | 25 ++++++- .../globalSearch/globalSearchModel.ts | 28 ++++++++ .../globalSearch/globalSearchResolver.ts | 2 +- .../search/sources/lint-warnings-guide.ts | 2 +- packages/common/database-types.ts | 21 ++++++ .../20250714120000_hybrid_search.sql | 69 +++++++++++++++++++ 7 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 supabase/migrations/20250714120000_hybrid_search.sql diff --git a/apps/docs/app/api/graphql/tests/searchDocs.test.ts b/apps/docs/app/api/graphql/tests/searchDocs.test.ts index f767021daa2..2f1f9524a92 100644 --- a/apps/docs/app/api/graphql/tests/searchDocs.test.ts +++ b/apps/docs/app/api/graphql/tests/searchDocs.test.ts @@ -13,7 +13,7 @@ vi.mock(import('~/lib/openAi'), () => ({ })) const rpcSpy = vi.fn().mockImplementation((funcName, params) => { - if (funcName === 'search_content') { + if (funcName === 'search_content_hybrid') { const limit = params?.max_result || 2 const mockResults = [ { @@ -88,7 +88,7 @@ describe('/api/graphql searchDocs', () => { expect(json.data).toBeDefined() expect(json.data.searchDocs).toBeDefined() expect(json.data.searchDocs.nodes).toBeInstanceOf(Array) - expect(json.data.searchDocs.nodes).toHaveLength(2) + expect(json.data.searchDocs.nodes).toHaveLength(3) expect(json.data.searchDocs.nodes[0]).toMatchObject({ title: 'Test Guide', href: '/guides/test', @@ -117,7 +117,7 @@ describe('/api/graphql searchDocs', () => { expect(json.data.searchDocs.nodes).toHaveLength(1) expect(json.data.searchDocs.nodes[0].title).toBe('Test Guide') expect(rpcSpy).toHaveBeenCalledWith( - 'search_content', + 'search_content_hybrid', expect.objectContaining({ max_result: 1, }) @@ -146,7 +146,7 @@ describe('/api/graphql searchDocs', () => { expect(json.errors).toBeUndefined() expect(json.data.searchDocs.nodes[0].content).toBe('Test content') expect(rpcSpy).toHaveBeenCalledWith( - 'search_content', + 'search_content_hybrid', expect.objectContaining({ include_full_content: true, }) diff --git a/apps/docs/lib/supabase.ts b/apps/docs/lib/supabase.ts index 65621a38ab3..e7c1768a374 100644 --- a/apps/docs/lib/supabase.ts +++ b/apps/docs/lib/supabase.ts @@ -7,7 +7,10 @@ type Database = { public: { Tables: DatabaseGenerated['public']['Tables'] Views: DatabaseGenerated['public']['Views'] - Functions: Omit & { + Functions: Omit< + DatabaseGenerated['public']['Functions'], + 'search_content' | 'search_content_hybrid' + > & { search_content: { Args: Omit< DatabaseGenerated['public']['Functions']['search_content']['Args'], @@ -28,6 +31,26 @@ type Database = { } > } + search_content_hybrid: { + Args: Omit< + DatabaseGenerated['public']['Functions']['search_content_hybrid']['Args'], + 'query_embedding' + > & { query_embedding: Array } + Returns: Array< + Omit< + DatabaseGenerated['public']['Functions']['search_content_hybrid']['Returns'][number], + 'subsections' | 'metadata' + > & { + metadata: { + subtitle?: string + language?: string + methodName?: string + platform?: string + } + subsections: Array<{ title?: string; href?: string; content?: string }> + } + > + } } Enums: DatabaseGenerated['public']['Enums'] CompositeTypes: DatabaseGenerated['public']['CompositeTypes'] diff --git a/apps/docs/resources/globalSearch/globalSearchModel.ts b/apps/docs/resources/globalSearch/globalSearchModel.ts index 47cdd2f12dd..8c63f28f493 100644 --- a/apps/docs/resources/globalSearch/globalSearchModel.ts +++ b/apps/docs/resources/globalSearch/globalSearchModel.ts @@ -40,6 +40,34 @@ export abstract class SearchResultModel { return matchResult }) } + + static async searchHybrid( + args: RootQueryTypeSearchDocsArgs, + requestedFields: Array + ): Promise> { + const query = args.query.trim() + const includeFullContent = requestedFields.includes('content') + const embeddingResult = await openAI().createContentEmbedding(query) + + return embeddingResult.flatMapAsync(async (embedding) => { + const matchResult = new Result( + await supabase().rpc('search_content_hybrid', { + query_text: query, + query_embedding: embedding, + include_full_content: includeFullContent, + max_result: args.limit ?? 30, + }) + ) + .map((matches) => + matches + .map(createModelFromMatch) + .filter((item): item is SearchResultInterface => item !== null) + ) + .mapError(convertPostgrestToApiError) + + return matchResult + }) + } } function createModelFromMatch({ diff --git a/apps/docs/resources/globalSearch/globalSearchResolver.ts b/apps/docs/resources/globalSearch/globalSearchResolver.ts index 238f567c97d..f730ce99e81 100644 --- a/apps/docs/resources/globalSearch/globalSearchResolver.ts +++ b/apps/docs/resources/globalSearch/globalSearchResolver.ts @@ -53,7 +53,7 @@ async function resolveSearchImpl( ): Promise, ApiErrorGeneric>> { const fieldsInfo = graphQLFields(info) const requestedFields = Object.keys(fieldsInfo.nodes ?? fieldsInfo.edges?.node ?? {}) - return await SearchResultModel.search(args, requestedFields) + return await SearchResultModel.searchHybrid(args, requestedFields) } export const searchRoot = { diff --git a/apps/docs/scripts/search/sources/lint-warnings-guide.ts b/apps/docs/scripts/search/sources/lint-warnings-guide.ts index 61d4f5e4e34..77713504e01 100644 --- a/apps/docs/scripts/search/sources/lint-warnings-guide.ts +++ b/apps/docs/scripts/search/sources/lint-warnings-guide.ts @@ -111,6 +111,6 @@ export class LintWarningsGuideSource extends BaseSource { extractIndexedContent(): string { const sections = this.sections ?? [] const sectionText = sections.map(({ content }) => content).join('\n\n') - return `# ${this.lint.path}\n\n${sectionText}` + return `# Database Advisor: Lint ${this.lint.path}\n\nThis is a database lint rule for Supabase, targeting the lint ID ${this.lint.path}. Lint rules help enforce performance and security best practices for your Supabase database.\n\n${sectionText}` } } diff --git a/packages/common/database-types.ts b/packages/common/database-types.ts index fee0b87aacd..80ad326b921 100644 --- a/packages/common/database-types.ts +++ b/packages/common/database-types.ts @@ -655,6 +655,27 @@ export type Database = { subsections: Json[] }[] } + search_content_hybrid: { + Args: { + query_text: string + query_embedding: string + max_result?: number + full_text_weight?: number + semantic_weight?: number + rrf_k?: number + match_threshold?: number + include_full_content?: boolean + } + Returns: { + id: number + page_title: string + type: string + href: string + content: string + metadata: Json + subsections: Json[] + }[] + } update_last_changed_checksum: { Args: { new_parent_page: string diff --git a/supabase/migrations/20250714120000_hybrid_search.sql b/supabase/migrations/20250714120000_hybrid_search.sql new file mode 100644 index 00000000000..592106ee286 --- /dev/null +++ b/supabase/migrations/20250714120000_hybrid_search.sql @@ -0,0 +1,69 @@ +-- Hybrid search: combines FTS and vector search using reciprocal rank fusion (RRF) +create or replace function search_content_hybrid( + query_text text, + query_embedding vector(1536), + max_result int default 30, + full_text_weight float default 1, + semantic_weight float default 1, + rrf_k int default 50, + match_threshold float default 0.78, + include_full_content boolean default false +) +returns table ( + id bigint, + page_title text, + type text, + href text, + content text, + metadata json, + subsections json[] +) +language sql +set search_path = '' +as $$ +with full_text as ( + select + id, + row_number() over(order by greatest( + least(10 * ts_rank(title_tokens, websearch_to_tsquery(query_text)), 1), + ts_rank(fts_tokens, websearch_to_tsquery(query_text)) + ) desc) as rank_ix + from public.page + where title_tokens @@ websearch_to_tsquery(query_text) or fts_tokens @@ websearch_to_tsquery(query_text) + order by rank_ix + limit least(max_result, 30) * 2 +), +semantic as ( + select + page_id as id, + row_number() over () as rank_ix + from public.match_embedding(query_embedding, match_threshold, max_result * 2) +), +rrf as ( + select + coalesce(full_text.id, semantic.id) as id, + coalesce(1.0 / (rrf_k + full_text.rank_ix), 0.0) * full_text_weight + + coalesce(1.0 / (rrf_k + semantic.rank_ix), 0.0) * semantic_weight as rrf_score + from full_text + full outer join semantic on full_text.id = semantic.id +) +select + page.id, + page.meta ->> 'title' as page_title, + page.type, + public.get_full_content_url(page.type, page.path, null) as href, + case when include_full_content then page.content else null end as content, + page.meta as metadata, + array_agg(json_build_object( + 'title', page_section.heading, + 'href', public.get_full_content_url(page.type, page.path, page_section.slug), + 'content', page_section.content + )) as subsections +from rrf +join public.page on page.id = rrf.id +left join public.page_section on page_section.page_id = page.id +where rrf.rrf_score > 0 +group by page.id +order by max(rrf.rrf_score) desc +limit max_result; +$$; \ No newline at end of file