From 6c4311f72344fc1b2cb47c0199dc3e47ac6d72f2 Mon Sep 17 00:00:00 2001 From: Charis <26616127+charislam@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:27:33 -0500 Subject: [PATCH] fix: improve docs search ux and result rankings (#19450) Improve the way docs are indexed and FTS results are ranked, so results are more relevant. Also improve debouncing and searching UX so it feels a bit faster and you can scan the results more easily. --- .gitignore | 2 + .../scripts/search/generate-embeddings.ts | 54 ++---- apps/docs/scripts/search/sources/base.ts | 15 +- .../search/sources/github-discussion.ts | 21 ++- apps/docs/scripts/search/sources/index.ts | 73 ++++---- apps/docs/scripts/search/sources/markdown.ts | 34 ++-- .../scripts/search/sources/reference-doc.ts | 168 +++++++++++++++--- .../ui/src/components/Command/CommandMenu.tsx | 11 +- .../ui/src/components/Command/DocsSearch.tsx | 81 ++++----- ...0231127222412_search_full_text_for_fts.sql | 68 +++++++ 10 files changed, 364 insertions(+), 163 deletions(-) create mode 100644 supabase/migrations/20231127222412_search_full_text_for_fts.sql diff --git a/.gitignore b/.gitignore index 3c9c2471b71..253f3d887dd 100644 --- a/.gitignore +++ b/.gitignore @@ -82,8 +82,10 @@ typings/ .env # excempt the apps/www .env !apps/www/.env +.env.tmp .env.test .env.local +.env.staging .env.production # parcel-bundler cache (https://parceljs.org/) diff --git a/apps/docs/scripts/search/generate-embeddings.ts b/apps/docs/scripts/search/generate-embeddings.ts index 4a06ac909b8..c9f7e004f75 100644 --- a/apps/docs/scripts/search/generate-embeddings.ts +++ b/apps/docs/scripts/search/generate-embeddings.ts @@ -4,6 +4,7 @@ import { parseArgs } from 'node:util' import { OpenAI } from 'openai' import { v4 as uuidv4 } from 'uuid' import { fetchSources } from './sources' +import { Json, Section } from './sources/base' dotenv.config() @@ -63,15 +64,19 @@ async function generateEmbeddings() { } for (const embeddingSource of embeddingSources) { - const { type, source, path, parentPath } = embeddingSource + const { type, source, path } = embeddingSource try { - const { checksum, meta, sections } = await embeddingSource.load() + const { + checksum, + sections, + meta = {}, + }: { checksum: string; sections: Section[]; meta?: Json } = embeddingSource.process() // Check for existing page in DB and compare checksums const { error: fetchPageError, data: existingPage } = await supabaseClient .from('page') - .select('id, path, checksum, parentPage:parent_page_id(id, path)') + .select('id, path, checksum') .filter('path', 'eq', path) .limit(1) .maybeSingle() @@ -82,34 +87,6 @@ async function generateEmbeddings() { // We use checksum to determine if this page & its sections need to be regenerated if (!shouldRefresh && existingPage?.checksum === checksum) { - const existingParentPage = Array.isArray(existingPage?.parentPage) - ? existingPage?.parentPage[0] - : existingPage?.parentPage - - // If parent page changed, update it - if (existingParentPage?.path !== parentPath) { - console.log(`[${path}] Parent page has changed. Updating to '${parentPath}'...`) - const { error: fetchParentPageError, data: parentPage } = await supabaseClient - .from('page') - .select() - .filter('path', 'eq', parentPath) - .limit(1) - .maybeSingle() - - if (fetchParentPageError) { - throw fetchParentPageError - } - - const { error: updatePageError } = await supabaseClient - .from('page') - .update({ parent_page_id: parentPage?.id }) - .filter('id', 'eq', existingPage.id) - - if (updatePageError) { - throw updatePageError - } - } - // No content/embedding update required on this page // Update other meta info const { error: updatePageError } = await supabaseClient @@ -149,17 +126,6 @@ async function generateEmbeddings() { } } - const { error: fetchParentPageError, data: parentPage } = await supabaseClient - .from('page') - .select() - .filter('path', 'eq', parentPath) - .limit(1) - .maybeSingle() - - if (fetchParentPageError) { - throw fetchParentPageError - } - // Create/update page record. Intentionally clear checksum until we // have successfully generated all page sections. const { error: upsertPageError, data: page } = await supabaseClient @@ -171,7 +137,7 @@ async function generateEmbeddings() { type, source, meta, - parent_page_id: parentPage?.id, + content: embeddingSource.extractIndexedContent(), version: refreshVersion, last_refresh: refreshDate, }, @@ -200,7 +166,7 @@ async function generateEmbeddings() { const [responseData] = embeddingResponse.data - const { error: insertPageSectionError, data: pageSection } = await supabaseClient + const { error: insertPageSectionError } = await supabaseClient .from('page_section') .insert({ page_id: page.id, diff --git a/apps/docs/scripts/search/sources/base.ts b/apps/docs/scripts/search/sources/base.ts index 3a12513ed96..0a0aea98a98 100644 --- a/apps/docs/scripts/search/sources/base.ts +++ b/apps/docs/scripts/search/sources/base.ts @@ -9,12 +9,23 @@ export type Section = { slug?: string } +export abstract class BaseLoader { + type: string + + constructor(public source: string, public path: string) {} + + abstract load(): Promise +} + export abstract class BaseSource { + type: string checksum?: string meta?: Json sections?: Section[] - constructor(public source: string, public path: string, public parentPath?: string) {} + constructor(public source: string, public path: string) {} - abstract load(): Promise<{ checksum: string; meta?: Json; sections: Section[] }> + abstract process(): { checksum: string; meta?: Json; sections: Section[] } + + abstract extractIndexedContent(): string } diff --git a/apps/docs/scripts/search/sources/github-discussion.ts b/apps/docs/scripts/search/sources/github-discussion.ts index a87012a9780..5a27de92c39 100644 --- a/apps/docs/scripts/search/sources/github-discussion.ts +++ b/apps/docs/scripts/search/sources/github-discussion.ts @@ -2,7 +2,7 @@ import { createAppAuth } from '@octokit/auth-app' import { Octokit } from '@octokit/core' import { paginateGraphql } from '@octokit/plugin-paginate-graphql' import { createHash } from 'crypto' -import { BaseSource } from './base' +import { BaseLoader, BaseSource } from './base' export const ExtendedOctokit = Octokit.plugin(paginateGraphql) export type ExtendedOctokit = InstanceType @@ -74,7 +74,7 @@ export async function fetchDiscussions(owner: string, repo: string, categoryId: return discussions } -export class GitHubDiscussionSource extends BaseSource { +export class GitHubDiscussionLoader extends BaseLoader { type = 'github-discussions' as const constructor(source: string, public discussion: Discussion) { @@ -82,6 +82,18 @@ export class GitHubDiscussionSource extends BaseSource { } async load() { + return [new GitHubDiscussionSource(this.source, this.path, this.discussion)] + } +} + +export class GitHubDiscussionSource extends BaseSource { + type = 'github-discussions' as const + + constructor(source: string, path: string, public discussion: Discussion) { + super(source, path) + } + + process() { const { id, title, updatedAt, body, databaseId } = this.discussion const checksum = createHash('sha256').update(updatedAt).digest('base64') @@ -116,4 +128,9 @@ export class GitHubDiscussionSource extends BaseSource { sections, } } + + extractIndexedContent(): string { + const sections = this.sections ?? [] + return sections.map(({ heading, content }) => `${heading}\n\n${content}`).join('\n') + } } diff --git a/apps/docs/scripts/search/sources/index.ts b/apps/docs/scripts/search/sources/index.ts index 349186f6365..b632445ba6a 100644 --- a/apps/docs/scripts/search/sources/index.ts +++ b/apps/docs/scripts/search/sources/index.ts @@ -1,8 +1,15 @@ -import { GitHubDiscussionSource, fetchDiscussions } from './github-discussion' -import { MarkdownSource } from './markdown' import { + GitHubDiscussionLoader, + GitHubDiscussionSource, + fetchDiscussions, +} from './github-discussion' +import { MarkdownLoader, MarkdownSource } from './markdown' +import { + CliReferenceLoader, CliReferenceSource, + ClientLibReferenceLoader, ClientLibReferenceSource, + OpenApiReferenceLoader, OpenApiReferenceSource, } from './reference-doc' import { walk } from './util' @@ -20,74 +27,74 @@ export type SearchSource = * Fetches all the sources we want to index for search */ export async function fetchSources() { - const openApiReferenceSource = new OpenApiReferenceSource( + const openApiReferenceSource = new OpenApiReferenceLoader( 'api', '/reference/api', { title: 'Management API Reference' }, '../../spec/transforms/api_v0_openapi_deparsed.json', '../../spec/common-api-sections.json' - ) + ).load() - const jsLibReferenceSource = new ClientLibReferenceSource( + const jsLibReferenceSource = new ClientLibReferenceLoader( 'js-lib', '/reference/javascript', { title: 'JavaScript Reference' }, '../../spec/supabase_js_v2.yml', '../../spec/common-client-libs-sections.json' - ) + ).load() - const dartLibReferenceSource = new ClientLibReferenceSource( + const dartLibReferenceSource = new ClientLibReferenceLoader( 'dart-lib', '/reference/dart', { title: 'Dart Reference' }, '../../spec/supabase_dart_v1.yml', '../../spec/common-client-libs-sections.json' - ) + ).load() - const pythonLibReferenceSource = new ClientLibReferenceSource( + const pythonLibReferenceSource = new ClientLibReferenceLoader( 'python-lib', '/reference/python', { title: 'Python Reference' }, '../../spec/supabase_py_v2.yml', '../../spec/common-client-libs-sections.json' - ) + ).load() - const cSharpLibReferenceSource = new ClientLibReferenceSource( + const cSharpLibReferenceSource = new ClientLibReferenceLoader( 'csharp-lib', '/reference/csharp', { title: 'C# Reference' }, '../../spec/supabase_csharp_v0.yml', '../../spec/common-client-libs-sections.json' - ) + ).load() - const swiftLibReferenceSource = new ClientLibReferenceSource( + const swiftLibReferenceSource = new ClientLibReferenceLoader( 'swift-lib', '/reference/swift', { title: 'Swift Reference' }, '../../spec/supabase_swift_v1.yml', '../../spec/common-client-libs-sections.json' - ) + ).load() - const ktLibReferenceSource = new ClientLibReferenceSource( + const ktLibReferenceSource = new ClientLibReferenceLoader( 'kt-lib', '/reference/kotlin', { title: 'Kotlin Reference' }, '../../spec/supabase_kt_v1.yml', '../../spec/common-client-libs-sections.json' - ) + ).load() - const cliReferenceSource = new CliReferenceSource( + const cliReferenceSource = new CliReferenceLoader( 'cli', '/reference/cli', { title: 'CLI Reference' }, '../../spec/cli_v1_commands.yaml', '../../spec/common-cli-sections.json' - ) + ).load() const guideSources = (await walk('pages')) .filter(({ path }) => /\.mdx?$/.test(path)) .filter(({ path }) => !ignoredFiles.includes(path)) - .map((entry) => new MarkdownSource('guide', entry.path)) + .map((entry) => new MarkdownLoader('guide', entry.path).load()) const githubDiscussionSources = ( await fetchDiscussions( @@ -95,20 +102,22 @@ export async function fetchSources() { 'supabase', 'DIC_kwDODMpXOc4CUvEr' // 'Troubleshooting' category ) - ).map((discussion) => new GitHubDiscussionSource('supabase/supabase', discussion)) + ).map((discussion) => new GitHubDiscussionLoader('supabase/supabase', discussion).load()) - const sources: SearchSource[] = [ - openApiReferenceSource, - jsLibReferenceSource, - dartLibReferenceSource, - pythonLibReferenceSource, - cSharpLibReferenceSource, - swiftLibReferenceSource, - ktLibReferenceSource, - cliReferenceSource, - ...githubDiscussionSources, - ...guideSources, - ] + const sources: SearchSource[] = ( + await Promise.all([ + openApiReferenceSource, + jsLibReferenceSource, + dartLibReferenceSource, + pythonLibReferenceSource, + cSharpLibReferenceSource, + swiftLibReferenceSource, + ktLibReferenceSource, + cliReferenceSource, + ...githubDiscussionSources, + ...guideSources, + ]) + ).flat() return sources } diff --git a/apps/docs/scripts/search/sources/markdown.ts b/apps/docs/scripts/search/sources/markdown.ts index 91beb6cc01a..49c5f42b373 100644 --- a/apps/docs/scripts/search/sources/markdown.ts +++ b/apps/docs/scripts/search/sources/markdown.ts @@ -10,7 +10,7 @@ import { toString } from 'mdast-util-to-string' import { mdxjs } from 'micromark-extension-mdxjs' import { u } from 'unist-builder' import { filter } from 'unist-util-filter' -import { BaseSource, Json, Section } from './base' +import { BaseLoader, BaseSource, Json, Section } from './base' /** * Extracts ES literals from an `estree` `ObjectExpression` @@ -189,29 +189,39 @@ export type ProcessedMdx = { sections: Section[] } -export class MarkdownSource extends BaseSource { +export class MarkdownLoader extends BaseLoader { type = 'markdown' as const - constructor(source: string, public filePath: string, public parentFilePath?: string) { + constructor(source: string, public filePath: string) { const path = filePath.replace(/^pages/, '').replace(/\.mdx?$/, '') - const parentPath = parentFilePath?.replace(/^pages/, '').replace(/\.mdx?$/, '') - - super(source, path, parentPath) + super(source, path) } async load() { const contents = await readFile(this.filePath, 'utf8') + return [new MarkdownSource(this.source, this.path, contents)] + } +} - const { checksum, meta, sections } = processMdxForSearch(contents) +export class MarkdownSource extends BaseSource { + type = 'markdown' as const + + constructor(source: string, path: string, public contents: string) { + super(source, path) + } + + process() { + const { checksum, meta, sections } = processMdxForSearch(this.contents) this.checksum = checksum this.meta = meta this.sections = sections - return { - checksum, - meta, - sections, - } + return { checksum, meta, sections } + } + + extractIndexedContent(): string { + const sections = this.sections ?? [] + return sections.map(({ content }) => content).join('\n\n') } } diff --git a/apps/docs/scripts/search/sources/reference-doc.ts b/apps/docs/scripts/search/sources/reference-doc.ts index 34a907bb64a..789f52d418c 100644 --- a/apps/docs/scripts/search/sources/reference-doc.ts +++ b/apps/docs/scripts/search/sources/reference-doc.ts @@ -4,16 +4,20 @@ import yaml from 'js-yaml' import { OpenAPIV3 } from 'openapi-types' import { ICommonItem, + ICommonSection, IFunctionDefinition, ISpec, } from '../../../components/reference/Reference.types' import { CliCommand, CliSpec } from '../../../generator/types/CliSpec' import { flattenSections } from '../../../lib/helpers' import { enrichedOperation, gen_v3 } from '../../../lib/refGenerator/helpers' -import { BaseSource, Json } from './base' +import { BaseLoader, BaseSource, Json } from './base' -export abstract class ReferenceSource extends BaseSource { +export abstract class ReferenceLoader extends BaseLoader { type = 'reference' as const + sourceConstructor: ( + ...args: ConstructorParameters> + ) => ReferenceSource constructor( source: string, @@ -32,10 +36,6 @@ export abstract class ReferenceSource extends BaseSource { const refSections: ICommonItem[] = JSON.parse(refSectionsContents) const flattenedRefSections = flattenSections(refSections) - const checksum = createHash('sha256') - .update(specContents + refSectionsContents) - .digest('base64') - const specSections = this.getSpecSections(specContents) const sections = flattenedRefSections @@ -46,16 +46,51 @@ export abstract class ReferenceSource extends BaseSource { return } - return { - heading: refSection.title, - slug: refSection.slug, - content: `${this.meta.title} for ${refSection.title}:\n${this.formatSection( - specSection, - refSection - )}`, - } + return this.sourceConstructor( + this.source, + `${this.path}/${refSection.slug}`, + refSection, + specSection, + this.meta + ) }) - .filter((section) => !!section) + .filter(Boolean) + + return sections + } + + abstract getSpecSections(specContents: string): SpecSection[] + abstract matchSpecSection(specSections: SpecSection[], id: string): SpecSection +} + +export abstract class ReferenceSource extends BaseSource { + type = 'reference' as const + + constructor( + source: string, + path: string, + public refSection: ICommonSection, + public specSection: SpecSection, + public meta: Json + ) { + super(source, path) + } + + process() { + const checksum = createHash('sha256') + .update(JSON.stringify(this.refSection) + JSON.stringify(this.specSection)) + .digest('base64') + + const sections = [ + { + heading: this.refSection.title, + slug: this.refSection.slug, + content: `${this.meta.title} for ${this.refSection.title}:\n${this.formatSection( + this.specSection, + this.refSection + )}`, + }, + ] this.checksum = checksum this.sections = sections @@ -63,16 +98,31 @@ export abstract class ReferenceSource extends BaseSource { return { checksum, sections, - meta: this.meta, + meta: { + ...this.meta, + subtitle: this.extractSubtitle(), + title: this.extractTitle(), + }, } } - abstract getSpecSections(specContents: string): SpecSection[] - abstract matchSpecSection(specSections: SpecSection[], id: string): SpecSection abstract formatSection(specSection: SpecSection, refSection: ICommonItem): string + abstract extractTitle(): string + abstract extractSubtitle(): string } -export class OpenApiReferenceSource extends ReferenceSource { +export class OpenApiReferenceLoader extends ReferenceLoader { + constructor( + source: string, + path: string, + meta: Json, + specFilePath: string, + sectionsFilePath: string + ) { + super(source, path, meta, specFilePath, sectionsFilePath) + this.sourceConstructor = (...args) => new OpenApiReferenceSource(...args) + } + getSpecSections(specContents: string): enrichedOperation[] { const spec: OpenAPIV3.Document<{}> = JSON.parse(specContents) @@ -85,7 +135,10 @@ export class OpenApiReferenceSource extends ReferenceSource { matchSpecSection(operations: enrichedOperation[], id: string): enrichedOperation { return operations.find((operation) => operation.operationId === id) } - formatSection(specOperation: enrichedOperation) { +} + +export class OpenApiReferenceSource extends ReferenceSource { + formatSection(specOperation: enrichedOperation, _: ICommonItem) { const { summary, description, operation, path, tags } = specOperation return JSON.stringify({ summary, @@ -95,9 +148,35 @@ export class OpenApiReferenceSource extends ReferenceSource { tags, }) } + + extractSubtitle() { + return typeof this.specSection.description === 'string' ? this.specSection.description : '' + } + + extractTitle() { + return `${this.meta.title}: ${this.specSection.operation}` + } + + extractIndexedContent(): string { + const { summary, description, operation, tags } = this.specSection + return `${this.meta.title}\n\n${summary}\n\n${description}\n\n${operation}\n\n${tags.join( + ', ' + )}` + } } -export class ClientLibReferenceSource extends ReferenceSource { +export class ClientLibReferenceLoader extends ReferenceLoader { + constructor( + source: string, + path: string, + meta: Json, + specFilePath: string, + sectionsFilePath: string + ) { + super(source, path, meta, specFilePath, sectionsFilePath) + this.sourceConstructor = (...args) => new ClientLibReferenceSource(...args) + } + getSpecSections(specContents: string): IFunctionDefinition[] { const spec = yaml.load(specContents) as ISpec @@ -106,6 +185,9 @@ export class ClientLibReferenceSource extends ReferenceSource functionDefinition.id === id) } +} + +export class ClientLibReferenceSource extends ReferenceSource { formatSection(functionDefinition: IFunctionDefinition, refSection: ICommonItem): string { const { title } = refSection const { description, title: functionName } = functionDefinition @@ -116,9 +198,33 @@ export class ClientLibReferenceSource extends ReferenceSource { +export class CliReferenceLoader extends ReferenceLoader { + constructor( + source: string, + path: string, + meta: Json, + specFilePath: string, + sectionsFilePath: string + ) { + super(source, path, meta, specFilePath, sectionsFilePath) + this.sourceConstructor = (...args) => new CliReferenceSource(...args) + } + getSpecSections(specContents: string): CliCommand[] { const spec = yaml.load(specContents) as CliSpec @@ -127,7 +233,10 @@ export class CliReferenceSource extends ReferenceSource { matchSpecSection(cliCommands: CliCommand[], id: string): CliCommand { return cliCommands.find((cliCommand) => cliCommand.id === id) } - formatSection(cliCommand: CliCommand): string { +} + +export class CliReferenceSource extends ReferenceSource { + formatSection(cliCommand: CliCommand, _: ICommonItem): string { const { summary, description, usage } = cliCommand return JSON.stringify({ summary, @@ -135,4 +244,17 @@ export class CliReferenceSource extends ReferenceSource { usage, }) } + + extractSubtitle(): string { + return `${this.meta.title}: ${this.specSection.title}` + } + + extractTitle(): string { + return this.specSection.summary + } + + extractIndexedContent(): string { + const { summary, description, usage } = this.specSection + return `${this.meta.title}\n\n${summary}\n\n${description}\n\n${usage}` + } } diff --git a/packages/ui/src/components/Command/CommandMenu.tsx b/packages/ui/src/components/Command/CommandMenu.tsx index d2dc8bf8dd3..875c1661065 100644 --- a/packages/ui/src/components/Command/CommandMenu.tsx +++ b/packages/ui/src/components/Command/CommandMenu.tsx @@ -85,6 +85,9 @@ const CommandMenu = ({ projectRef }: CommandMenuProps) => { setSearch(newValue) } + const commandListMaxHeight = + currentPage === COMMAND_ROUTES.DOCS_SEARCH ? 'min(600px, 50vh)' : '300px' + return ( <> { onValueChange={handleInputChange} /> )} - + {!currentPage && ( <> diff --git a/packages/ui/src/components/Command/DocsSearch.tsx b/packages/ui/src/components/Command/DocsSearch.tsx index 69258d23503..72ac6392dde 100644 --- a/packages/ui/src/components/Command/DocsSearch.tsx +++ b/packages/ui/src/components/Command/DocsSearch.tsx @@ -21,6 +21,7 @@ import { FORCE_MOUNT_ITEM, TextHighlighter, } from './Command.utils' +import { useRouter } from 'next/router' const NUMBER_SOURCES = 2 @@ -56,21 +57,6 @@ export interface Page { sections: PageSection[] } -function removeDoubleQuotes(inputString: string): string { - // Use the replace method with a regular expression to remove double quotes - return inputString.replace(/"/g, '') -} - -const getDocsUrl = () => { - if (!process.env.NEXT_PUBLIC_SITE_URL || !process.env.NEXT_PUBLIC_LOCAL_SUPABASE) { - return 'https://supabase.com/docs' - } - - const isLocal = - process.env.NEXT_PUBLIC_SITE_URL.includes('localhost') || process.env.NEXT_PUBLIC_LOCAL_SUPABASE - return isLocal ? 'http://localhost:3001/docs' : 'https://supabase.com/docs' -} - type SearchState = | { status: 'initial' @@ -183,7 +169,12 @@ function reducer(state: SearchState, action: Action): SearchState { : { status: 'loading', key: action.key, - staleResults: [], + staleResults: + 'results' in state + ? state.results + : 'staleResults' in state + ? state.staleResults + : [], } } return allSourcesLoaded @@ -201,7 +192,8 @@ function reducer(state: SearchState, action: Action): SearchState { return { status: 'loading', key: action.key, - staleResults: 'results' in state ? state.results : [], + staleResults: + 'results' in state ? state.results : 'staleResults' in state ? state.staleResults : [], } case 'reset': return { @@ -226,9 +218,22 @@ function reducer(state: SearchState, action: Action): SearchState { const DocsSearch = () => { const [state, dispatch] = useReducer(reducer, { status: 'initial', key: 0 }) const supabaseClient = useSupabaseClient() - const { isLoading, setIsLoading, search, setSearch, inputRef } = useCommandMenu() + const { search, setSearch, inputRef } = useCommandMenu() const key = useRef(0) const initialLoad = useRef(true) + const router = useRouter() + + function openLink(pageType: PageType, link: string) { + switch (pageType) { + case PageType.Markdown: + case PageType.Reference: + return router.push(link) + case PageType.GithubDiscussion: + return window.open(link, '_blank') + default: + throw new Error(`Unknown page type '${pageType}'`) + } + } const hasResults = state.status === 'fullResults' || @@ -237,8 +242,6 @@ const DocsSearch = () => { const handleSearch = useCallback( async (query: string) => { - setIsLoading(true) - key.current += 1 const localKey = key.current dispatch({ type: 'newSearchDispatched', key: localKey }) @@ -273,11 +276,6 @@ const DocsSearch = () => { message: error.message ?? '', }) }) - .finally(() => { - if (sourcesLoaded === NUMBER_SOURCES) { - setIsLoading(false) - } - }) }) }, [supabaseClient] @@ -293,7 +291,7 @@ const DocsSearch = () => { }) } - const debouncedSearch = useMemo(() => debounce(handleSearch, 300), [handleSearch]) + const debouncedSearch = useMemo(() => debounce(handleSearch, 150), [handleSearch]) useEffect(() => { if (initialLoad.current) { @@ -311,7 +309,7 @@ const DocsSearch = () => { key.current += 1 dispatch({ type: 'reset', key: key.current }) } - }, [search]) + }, [search, handleSearch, debouncedSearch]) // Immediately run search if user presses enter // and abort any debounced searches that are waiting @@ -379,11 +377,11 @@ const DocsSearch = () => { return ( { @@ -396,9 +394,12 @@ const DocsSearch = () => { - {page.description && ( + {(page.description || page.subtitle) && (
- +
)} @@ -414,7 +415,7 @@ const DocsSearch = () => { onSelect={() => { openLink(page.type, formatSectionUrl(page, section)) }} - key={`${page.title}__${section.heading}-item-index-${i}`} + key={`${page.path}__${section.heading}-item`} value={`${FORCE_MOUNT_ITEM}--${page.title}__${section.heading}-item-index-${i}`} type="block-link" > @@ -449,7 +450,7 @@ const DocsSearch = () => { const key = question.replace(/\s+/g, '_') return ( { if (!search) { handleSearch(question) @@ -499,11 +500,9 @@ const DocsSearch = () => { export default DocsSearch export function formatPageUrl(page: Page) { - const docsUrl = getDocsUrl() switch (page.type) { case PageType.Markdown: case PageType.Reference: - return `${docsUrl}${page.path}` case PageType.GithubDiscussion: return page.path default: @@ -546,15 +545,3 @@ export function getPageSectionIcon(page: Page) { throw new Error(`Unknown page type '${page.type}'`) } } - -export function openLink(pageType: PageType, link: string) { - switch (pageType) { - case PageType.Markdown: - case PageType.Reference: - return window.location.assign(link) - case PageType.GithubDiscussion: - return window.open(link, '_blank') - default: - throw new Error(`Unknown page type '${pageType}'`) - } -} diff --git a/supabase/migrations/20231127222412_search_full_text_for_fts.sql b/supabase/migrations/20231127222412_search_full_text_for_fts.sql new file mode 100644 index 00000000000..2eeb904b36b --- /dev/null +++ b/supabase/migrations/20231127222412_search_full_text_for_fts.sql @@ -0,0 +1,68 @@ +-- remove unused column + +alter table page +drop column parent_page_id; + +-- move indexed content for fts search from page_section to page +-- this should allow better rankings as it gives a better overview of +-- search term frequency on that page + +drop index fts_search_index; + +alter table page_section +drop column fts_tokens; + +alter table page +add column content text; + +alter table page +add column fts_tokens tsvector generated always as (to_tsvector('english', content)) stored; + +create index fts_search_index_page on page using gin(fts_tokens); + +-- also search against the page title if it exists, to give more +-- intuitive search rankings + +alter table page + +add column title_tokens tsvector generated always as (to_tsvector('english', coalesce(meta ->> 'title', ''))) stored; + +create index fts_search_index_title on page using gin(title_tokens); + +-- rank search by best match (title matches tend to rank better than content matches +-- due to underlying ts_rank algorithm + +drop function docs_search_fts; + +create or replace function docs_search_fts(query text) +returns table ( + id int8, + path text, + type text, + title text, + subtitle text, + description text +) +language plpgsql +as $$ +#variable_conflict use_variable +begin + return query + select + page.id, + page.path, + page.type, + page.meta ->> 'title' as title, + page.meta ->> 'subtitle' as subtitle, + page.meta ->> 'description' as description + from page + where title_tokens @@ websearch_to_tsquery(query) or fts_tokens @@ websearch_to_tsquery(query) + order by greatest( + -- Title is more important than body, so use 10 as the weighting factor + -- Cut off at max rank of 1 + least(10 * ts_rank(title_tokens, websearch_to_tsquery(query)), 1), + ts_rank(fts_tokens, websearch_to_tsquery(query)) + ) desc + limit 10; +end; +$$;