From 149c6d1ecd1f4ea0ed0aeaeac42157edfef86e7d Mon Sep 17 00:00:00 2001 From: Greg Richardson Date: Wed, 31 May 2023 18:39:18 -0600 Subject: [PATCH] feat(docs): pull docs from external vecs repo --- .../NavigationMenu.constants.ts | 10 ++ apps/docs/lib/mdx/rehypeLinkTransform.ts | 26 ++++ apps/docs/package.json | 4 + apps/docs/pages/guides/ai/python/[slug].tsx | 136 ++++++++++++++++++ package-lock.json | 8 ++ 5 files changed, 184 insertions(+) create mode 100644 apps/docs/lib/mdx/rehypeLinkTransform.ts create mode 100644 apps/docs/pages/guides/ai/python/[slug].tsx diff --git a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts index 12c5df4b5d5..ff2f609789a 100644 --- a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts +++ b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts @@ -875,6 +875,16 @@ export const ai: NavMenuConstant = { { name: 'Face similarity search', url: '/guides/ai/quickstarts/face-similarity' }, ], }, + { + name: 'Python Client', + url: undefined, + items: [ + { name: 'API', url: '/guides/ai/python/api' }, + { name: 'Collections', url: '/guides/ai/python/collections' }, + { name: 'Indexes', url: '/guides/ai/python/indexes' }, + { name: 'Metadata', url: '/guides/ai/python/metadata' }, + ], + }, { name: 'Guides', url: undefined, diff --git a/apps/docs/lib/mdx/rehypeLinkTransform.ts b/apps/docs/lib/mdx/rehypeLinkTransform.ts new file mode 100644 index 00000000000..90a3fde05af --- /dev/null +++ b/apps/docs/lib/mdx/rehypeLinkTransform.ts @@ -0,0 +1,26 @@ +import { Element } from 'hast' +import { hasProperty } from 'hast-util-has-property' +import { Node } from 'unist' +import { visit } from 'unist-util-visit' + +export type UrlTransformFunction = (url: string, node: Element) => string + +function modify(node: Element, prop: string, fn?: UrlTransformFunction) { + if (hasProperty(node, prop)) { + const property = node.properties[prop] + if (typeof property !== 'string') { + return + } + + node.properties[prop] = fn?.(property, node) ?? property + } +} + +export function linkTransform(fn?: UrlTransformFunction) { + return function transformer(tree: Node) { + visit(tree, 'element', (node: Element) => { + modify(node, 'href', fn) + modify(node, 'src', fn) + }) + } +} diff --git a/apps/docs/package.json b/apps/docs/package.json index c59b1dcf3c4..d537bbc0fea 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -65,6 +65,7 @@ "framer-motion": "^6.5.1", "github-slugger": "^2.0.0", "gray-matter": "^4.0.3", + "hast-util-has-property": "^2.0.1", "isbot": "^3.6.5", "jsrsasign": "^10.5.26", "lodash": "^4.17.21", @@ -101,12 +102,15 @@ "ui": "*", "unist-builder": "^3.0.1", "unist-util-filter": "^4.0.1", + "unist-util-visit": "^4.1.2", "uuid": "^9.0.0", "valtio": "^1.7.6" }, "devDependencies": { + "@types/hast": "^2.3.4", "@types/node": "^17.0.12", "@types/react": "17.0.39", + "@types/unist": "^2.0.6", "config": "*", "dotenv": "^16.0.3", "ejs": "^3.1.8", diff --git a/apps/docs/pages/guides/ai/python/[slug].tsx b/apps/docs/pages/guides/ai/python/[slug].tsx new file mode 100644 index 00000000000..07804e27db3 --- /dev/null +++ b/apps/docs/pages/guides/ai/python/[slug].tsx @@ -0,0 +1,136 @@ +import { GetStaticPaths, GetStaticProps } from 'next' +import { MDXRemote, MDXRemoteSerializeResult } from 'next-mdx-remote' +import { serialize } from 'next-mdx-remote/serialize' +import { join, relative } from 'path' +import rehypeSlug from 'rehype-slug' +import remarkGfm from 'remark-gfm' +import components from '~/components' +import Layout from '~/layouts/DefaultGuideLayout' +import { UrlTransformFunction, linkTransform } from '~/lib/mdx/rehypeLinkTransform' + +// We fetch these docs at build time from an external repo +const org = 'supabase' +const repo = 'vecs' +const branch = 'main' +const docsDir = 'docs' +const externalSite = 'https://supabase.github.io/vecs' + +// Each external docs page is mapped to a local page +const pageMap = [ + { + slug: 'api', + meta: { + title: 'API', + }, + remoteFile: 'api.md', + }, + { + slug: 'collections', + meta: { + title: 'Collections', + }, + remoteFile: 'concepts_collections.md', + }, + { + slug: 'indexes', + meta: { + title: 'Indexes', + }, + remoteFile: 'concepts_indexes.md', + }, + { + slug: 'metadata', + meta: { + title: 'Metadata', + }, + remoteFile: 'concepts_metadata.md', + }, +] + +interface PythonClientDocsProps { + source: MDXRemoteSerializeResult + meta: { + title: string + description?: string + } +} + +export default function PythonClientDocs({ source, meta }: PythonClientDocsProps) { + return ( + + + + ) +} + +/** + * Fetch markdown from external repo and transform links + */ +export const getStaticProps: GetStaticProps = async ({ params }) => { + const page = pageMap.find(({ slug }) => slug === params.slug) + + if (!page) { + throw new Error(`No page mapping found for slug '${params.slug}'`) + } + + const { remoteFile, meta } = page + + const response = await fetch( + `https://raw.githubusercontent.com/${org}/${repo}/${branch}/${docsDir}/${remoteFile}` + ) + + const source = await response.text() + + const urlTransform: UrlTransformFunction = (url) => { + try { + const externalSiteUrl = new URL(externalSite) + + const placeholderHostname = 'placeholder' + const { hostname, pathname, hash } = new URL(url, `http://${placeholderHostname}`) + + // Don't modify a url with a FQDN or a url that's only a hash + if (hostname !== placeholderHostname || pathname === '/') { + return url + } + + const relativePage = ( + pathname.endsWith('.md') + ? pathname.replace(/\.md$/, '') + : relative(externalSiteUrl.pathname, pathname) + ).replace(/^\//, '') + + const page = pageMap.find(({ remoteFile }) => `${relativePage}.md` === remoteFile) + + // If we have a mapping for this page, use the mapped path + if (page) { + return page.slug + hash + } + + // If we don't have this page in our docs, link to original docs + return join(externalSite, relativePage) + } catch (err) { + console.error('Error transforming markdown URL', err) + return url + } + } + + const mdxSource = await serialize(source, { + mdxOptions: { + remarkPlugins: [remarkGfm], + rehypePlugins: [[linkTransform, urlTransform], rehypeSlug], + }, + }) + + return { props: { source: mdxSource, meta } } +} + +export const getStaticPaths: GetStaticPaths = async () => { + return { + paths: pageMap.map(({ slug }) => ({ + params: { + slug, + }, + })), + fallback: false, + } +} diff --git a/package-lock.json b/package-lock.json index 2436954f16f..19458c23d72 100644 --- a/package-lock.json +++ b/package-lock.json @@ -58,6 +58,7 @@ "framer-motion": "^6.5.1", "github-slugger": "^2.0.0", "gray-matter": "^4.0.3", + "hast-util-has-property": "^2.0.1", "isbot": "^3.6.5", "jsrsasign": "^10.5.26", "lodash": "^4.17.21", @@ -94,12 +95,15 @@ "ui": "*", "unist-builder": "^3.0.1", "unist-util-filter": "^4.0.1", + "unist-util-visit": "^4.1.2", "uuid": "^9.0.0", "valtio": "^1.7.6" }, "devDependencies": { + "@types/hast": "^2.3.4", "@types/node": "^17.0.12", "@types/react": "17.0.39", + "@types/unist": "^2.0.6", "config": "*", "dotenv": "^16.0.3", "ejs": "^3.1.8", @@ -62892,8 +62896,10 @@ "@supabase/auth-helpers-nextjs": "^0.5.6", "@supabase/auth-helpers-react": "^0.3.1", "@supabase/supabase-js": "^2.13.0", + "@types/hast": "^2.3.4", "@types/node": "^17.0.12", "@types/react": "17.0.39", + "@types/unist": "^2.0.6", "algoliasearch": "^4.14.2", "babel": "^6.23.0", "clsx": "^1.2.1", @@ -62906,6 +62912,7 @@ "github-slugger": "^2.0.0", "globby": "^12.0.2", "gray-matter": "^4.0.3", + "hast-util-has-property": "^2.0.1", "isbot": "^3.6.5", "jsrsasign": "^10.5.26", "lodash": "^4.17.21", @@ -62950,6 +62957,7 @@ "ui": "*", "unist-builder": "^3.0.1", "unist-util-filter": "^4.0.1", + "unist-util-visit": "^4.1.2", "uuid": "^9.0.0", "valtio": "^1.7.6" },