diff --git a/apps/docs/DEVELOPERS.md b/apps/docs/DEVELOPERS.md index 2f132d69801..a998eaba7d2 100644 --- a/apps/docs/DEVELOPERS.md +++ b/apps/docs/DEVELOPERS.md @@ -96,6 +96,6 @@ On the Next.JS side of things, these work almost exactly the same as the client #### Search -Search is handled using a Supabase instance. During CI, [a script](https://github.com/supabase/supabase/blob/master/apps/docs/scripts/generate-embeddings.ts) aggregates all content sources (eg. guides, reference docs, etc), indexes them using OpenAI embeddings, and stores them in a Supabase database. +Search is handled using a Supabase instance. During CI, [a script](https://github.com/supabase/supabase/blob/master/apps/docs/scripts/search/generate-embeddings.ts) aggregates all content sources (eg. guides, reference docs, etc), indexes them using OpenAI embeddings, and stores them in a Supabase database. At runtime, an [Edge Function](https://github.com/supabase/supabase/blob/master/supabase/functions) is executed that performs a similarity search between the user's query and the above content sources using [`pgvector`](https://github.com/pgvector/pgvector) embeddings. diff --git a/apps/docs/package.json b/apps/docs/package.json index cfda86a3831..ca12c92170b 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -9,9 +9,9 @@ "start": "next start", "lint": "next lint", "build:sitemap": "node ./internals/generate-sitemap.mjs", - "embeddings": "tsx scripts/generate-embeddings.ts", + "embeddings": "tsx scripts/search/generate-embeddings.ts", "embeddings:refresh": "npm run embeddings -- --refresh", - "postbuild": "ts-node ./scripts/build-search.ts && node ./internals/generate-sitemap.mjs", + "postbuild": "node ./internals/generate-sitemap.mjs", "generate:all": "npm-run-all --parallel gen:api gen:cli gen:gotrue gen:storage gen:supabase-dart:v0 gen:supabase-dart:v1 gen:supabase-csharp:v0 gen:supabase-js:v1 gen:supabase-js:v2 gen:realtime", "gen:api": "npm-run-all gen:api:usage", "gen:api:usage": "ts-node ./generator/index.ts gen --type api --url https://api.supabase.com --input ../../spec/transforms/api_v0_openapi_deparsed.json --output ./docs/reference/api/generated/usage.mdx", diff --git a/apps/docs/scripts/build-search.ts b/apps/docs/scripts/build-search.ts deleted file mode 100644 index 1fdc4e18a88..00000000000 --- a/apps/docs/scripts/build-search.ts +++ /dev/null @@ -1,155 +0,0 @@ -import fs from 'fs' -import crypto from 'crypto' -import path from 'path' -import matter from 'gray-matter' -import dotenv from 'dotenv' -import algoliasearch from 'algoliasearch/lite' -import { isEmpty } from 'lodash' - -// search objects -import { generateClientLibSearchObjects } from './files/client-libs' -import { generateAPISearchObjects } from './files/api' -import { generateCLISearchObjects } from './files/cli' - -const cliObjects = generateCLISearchObjects() -const apiObjects = generateAPISearchObjects() -const clientLibSearchObjects = generateClientLibSearchObjects() - -// @ts-ignore -// The properties of the searchObject that are specifically read -// by DocSearch are "type" and "hierarchy". The rest, though saved into Algolia (which we -// can potentially use to craft more nuanced search experiences) are not used by DocSearch -const ignoredFiles = [ - 'pages/404.mdx', - 'pages/faq.mdx', - 'pages/support.mdx', - 'pages/oss.tsx', - 'pages/_app.tsx', - 'pages/_document.tsx', - 'pages/[...slug].tsx', - 'pages/handbook/contributing.mdx', - 'pages/handbook/introduction.mdx', - 'pages/handbook/supasquad.mdx', -] - -async function walk(dir) { - let files = await fs.promises.readdir(dir) - //@ts-ignore - files = await Promise.all( - files.map(async (file) => { - const filePath = path.join(dir, file) - const stats = await fs.promises.stat(filePath) - if (stats.isDirectory()) return walk(filePath) - else if (stats.isFile()) return filePath - }) - ) - - return files.reduce((all, folderContents) => all.concat(folderContents), []) -} - -;(async function () { - // initialize environment variables - dotenv.config() - - if (!process.env.NEXT_PUBLIC_ALGOLIA_APP_ID || !process.env.ALGOLIA_SEARCH_ADMIN_KEY) { - return console.log( - 'Missing Algolia app ID / admin Key: skipping saving of Algolia search index' - ) - } - - console.log('Preparing docs indexing for Algolia') - - try { - const indexName = process.env.NEXT_PUBLIC_ALGOLIA_INDEX_NAME - const client = algoliasearch( - process.env.NEXT_PUBLIC_ALGOLIA_APP_ID, - process.env.ALGOLIA_SEARCH_ADMIN_KEY - ) - const index = client.initIndex(indexName) - - const guidePages = (await walk('pages')).filter((slug) => !ignoredFiles.includes(slug)) - - // generate search objects for mdx guide pages - const guidePagesearchObjects = guidePages - .map((slug) => { - let id, title, description - const fileContents = fs.readFileSync(slug, 'utf8') - const { data, content } = matter(fileContents) - - if (isEmpty(data)) { - // Guide pages do not have front-matter meta, unlike reference pages, have to manually extract - const metaIndex = fileContents.indexOf('export const meta = {') - if (metaIndex !== -1) { - const metaString = - fileContents - .slice(metaIndex + 20, fileContents.indexOf('}', metaIndex + 1) + 1) - .replace(/\n/g, '') - .slice(0, -2) + '}' - const meta = eval(`(${metaString})`) - id = meta.id - title = meta.title - description = meta.description - } - } else { - id = data.id - title = data.title - description = data.description - } - - const url = (slug.includes('/generated') ? slug.replace('/generated', '') : slug) - .replace('docs', '') - .replace('pages', '') - .replace(/\.mdx$/, '') - const source = slug.includes('/reference') ? 'reference' : 'guide' - - const object = { - // For Algolia - objectID: crypto.randomUUID(), - id, - title, - description, - url, - source, - //pageContent: content, - pageContent: '', - category: undefined, - version: undefined, - - // Docsearch specific - type: 'lvl1', - hierarchy: { - lvl0: 'Guides', - lvl1: title, - lvl2: null, - lvl3: null, - lvl4: null, - lvl5: null, - lvl6: null, - }, - } - - return object - }) - // Some of the reference generated files come with an 'index' page that we can ignore - .filter((object) => !object.url.endsWith('/index')) - .filter((object) => !object.url.endsWith('/.gitkeep')) - - const combinedSearchObjects = guidePagesearchObjects.concat( - clientLibSearchObjects, - apiObjects, - cliObjects - ) - - //@ts-ignore - await index.clearObjects() - console.log(`Successfully cleared records from ${indexName}`) - - //@ts-ignore - const algoliaResponse = await index.saveObjects(combinedSearchObjects) - - //@ts-ignore - console.log(`Successfully saved ${algoliaResponse.objectIDs.length} records into ${indexName}.`) - } catch (error) { - console.log('Error:', error) - } -})() diff --git a/apps/docs/scripts/files/api.ts b/apps/docs/scripts/files/api.ts deleted file mode 100644 index ed98a9c4921..00000000000 --- a/apps/docs/scripts/files/api.ts +++ /dev/null @@ -1,43 +0,0 @@ -import crypto from 'crypto' -import { flattenSections } from '../../lib/helpers' - -import apiCommonSections from '~/../../spec/common-api-sections.json' -import specFile from '~/../../spec/transforms/api_v0_openapi_deparsed.json' -import { gen_v3 } from '../../lib/refGenerator/helpers' - -// @ts-ignore -const generatedSpec = gen_v3(specFile, 'wat', { apiUrl: 'apiv0' }) -const sections = flattenSections(apiCommonSections) - -export function generateAPISearchObjects() { - let searchObjects = [] - - //@ts-ignore - sections.map((section) => { - const object = searchObjects.push({ - objectID: crypto.randomUUID(), - id: section.id, - title: section.title, - // @ts-ignore - description: generatedSpec.operations.find((item) => item.operationId === section.id) - ?.summary, - url: `/reference/api/${section.slug}`, - source: 'reference', - pageContent: '', - category: section.product, - version: '', - type: 'lvl2', - hierarchy: { - lvl0: 'References', - lvl1: 'Management API', - lvl2: section.title, - lvl3: null, - lvl4: null, - lvl5: null, - lvl6: null, - }, - }) - return object - }) - return searchObjects -} diff --git a/apps/docs/scripts/files/cli.ts b/apps/docs/scripts/files/cli.ts deleted file mode 100644 index 84caae225c6..00000000000 --- a/apps/docs/scripts/files/cli.ts +++ /dev/null @@ -1,43 +0,0 @@ -import crypto from 'crypto' -import fs from 'fs' -import { flattenSections } from '../../lib/helpers' -import cliCommonSections from '~/../../spec/common-cli-sections.json' -import yaml from 'js-yaml' - -// @ts-ignore - -const spec = yaml.load(fs.readFileSync(`../../spec/cli_v1_commands.yaml`, 'utf8')) - -const commonSections = flattenSections(cliCommonSections) - -export function generateCLISearchObjects() { - let searchObjects = [] - - //@ts-ignore - spec.commands.map((section) => { - const object = searchObjects.push({ - objectID: crypto.randomUUID(), - id: section.id, - title: section.title, - // @ts-ignore - description: section.description?.substr(0, section.description.indexOf('\n')), - url: `/reference/cli/${commonSections.find((item) => item.id === section.id)?.slug}`, - source: 'reference', - pageContent: '', - category: section.product, - version: '', - type: 'lvl2', - hierarchy: { - lvl0: 'References', - lvl1: 'Supabase CLI', - lvl2: section.title, - lvl3: null, - lvl4: null, - lvl5: null, - lvl6: null, - }, - }) - return object - }) - return searchObjects -} diff --git a/apps/docs/scripts/files/client-libs.ts b/apps/docs/scripts/files/client-libs.ts deleted file mode 100644 index 714769ebcc7..00000000000 --- a/apps/docs/scripts/files/client-libs.ts +++ /dev/null @@ -1,57 +0,0 @@ -import fs from 'fs' -import crypto from 'crypto' -import yaml from 'js-yaml' -import { flattenSections } from '../../lib/helpers' -import { nameMap } from '../helpers' - -import commonLibSections from '~/../../spec/common-client-libs-sections.json' - -const clientLibFiles = [ - { fileName: 'supabase_js_v2', label: 'javascript', version: 'v2', versionSlug: false }, - { fileName: 'supabase_js_v1', label: 'javascript', version: 'v1', versionSlug: true }, - { fileName: 'supabase_dart_v1', label: 'dart', version: 'v1', versionSlug: false }, - { fileName: 'supabase_dart_v0', label: 'dart', version: 'v0', versionSlug: true }, - { fileName: 'supabase_dart_v0', label: 'dart', version: 'v0', versionSlug: true }, -] - -const flatCommonLibSections = flattenSections(commonLibSections) - -export function generateClientLibSearchObjects() { - // loop through each spec file, find the correspending entry in the common file and grab the title / description / slug - let clientLibSearchObjects = [] - - clientLibFiles.map((file) => { - const specs = yaml.load(fs.readFileSync(`../../spec/${file.fileName}.yml`, 'utf8')) - - //take each function id, find it in the commonLibSections file and return { id, title, slug, description, } - //@ts-ignore - specs.functions.map((fn) => { - const item = flatCommonLibSections.find((section) => section.id === fn.id) - if (item) { - const object = clientLibSearchObjects.push({ - objectID: crypto.randomUUID(), - id: item.id, - title: item.title, - description: item.title, - url: `/reference/${file.label}/${file.versionSlug ? file.version + '/' : ''}${item.slug}`, - source: 'reference', - pageContent: '', - category: item.product, - version: file.version, - type: 'lvl2', - hierarchy: { - lvl0: 'References', - lvl1: `${nameMap[file.label]} ${file.version}`, - lvl2: item.title, - lvl3: file.version, - lvl4: null, - lvl5: null, - lvl6: null, - }, - }) - return object - } - }) - }) - return clientLibSearchObjects -} diff --git a/apps/docs/scripts/helpers.ts b/apps/docs/scripts/helpers.ts deleted file mode 100644 index 97ae8ddb86e..00000000000 --- a/apps/docs/scripts/helpers.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const nameMap = { - api: 'Management API', - cli: 'Supabase CLI', - auth: 'Auth Server', - storage: 'Storage Server', - postgres: 'Postgres', - dart: 'Supabase Flutter Library', - javascript: 'Supabase JavaScript Library', -} diff --git a/apps/docs/scripts/generate-embeddings.ts b/apps/docs/scripts/search/generate-embeddings.ts similarity index 99% rename from apps/docs/scripts/generate-embeddings.ts rename to apps/docs/scripts/search/generate-embeddings.ts index 23158f7f5bc..d04765afc67 100644 --- a/apps/docs/scripts/generate-embeddings.ts +++ b/apps/docs/scripts/search/generate-embeddings.ts @@ -3,7 +3,7 @@ import dotenv from 'dotenv' import 'openai' import { Configuration, OpenAIApi } from 'openai' import { inspect } from 'util' -import { fetchSources } from './search/sources' +import { fetchSources } from './sources' dotenv.config()