feat(docs): pull docs from external vecs repo

apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts

@@ -875,6 +875,16 @@ export const ai: NavMenuConstant = {
         { name: 'Face similarity search', url: '/guides/ai/quickstarts/face-similarity' },
       ],
     },
+    {
+      name: 'Python Client',
+      url: undefined,
+      items: [
+        { name: 'API', url: '/guides/ai/python/api' },
+        { name: 'Collections', url: '/guides/ai/python/collections' },
+        { name: 'Indexes', url: '/guides/ai/python/indexes' },
+        { name: 'Metadata', url: '/guides/ai/python/metadata' },
+      ],
+    },
     {
       name: 'Guides',
       url: undefined,

apps/docs/lib/mdx/rehypeLinkTransform.ts

@@ -0,0 +1,26 @@
+import { Element } from 'hast'
+import { hasProperty } from 'hast-util-has-property'
+import { Node } from 'unist'
+import { visit } from 'unist-util-visit'
+
+export type UrlTransformFunction = (url: string, node: Element) => string
+
+function modify(node: Element, prop: string, fn?: UrlTransformFunction) {
+  if (hasProperty(node, prop)) {
+    const property = node.properties[prop]
+    if (typeof property !== 'string') {
+      return
+    }
+
+    node.properties[prop] = fn?.(property, node) ?? property
+  }
+}
+
+export function linkTransform(fn?: UrlTransformFunction) {
+  return function transformer(tree: Node) {
+    visit(tree, 'element', (node: Element) => {
+      modify(node, 'href', fn)
+      modify(node, 'src', fn)
+    })
+  }
+}

apps/docs/package.json

@@ -65,6 +65,7 @@
     "framer-motion": "^6.5.1",
     "github-slugger": "^2.0.0",
     "gray-matter": "^4.0.3",
+    "hast-util-has-property": "^2.0.1",
     "isbot": "^3.6.5",
     "jsrsasign": "^10.5.26",
     "lodash": "^4.17.21",
@@ -101,12 +102,15 @@
     "ui": "*",
     "unist-builder": "^3.0.1",
     "unist-util-filter": "^4.0.1",
+    "unist-util-visit": "^4.1.2",
     "uuid": "^9.0.0",
     "valtio": "^1.7.6"
   },
   "devDependencies": {
+    "@types/hast": "^2.3.4",
     "@types/node": "^17.0.12",
     "@types/react": "17.0.39",
+    "@types/unist": "^2.0.6",
     "config": "*",
     "dotenv": "^16.0.3",
     "ejs": "^3.1.8",

apps/docs/pages/guides/ai/python/[slug].tsx

@@ -0,0 +1,136 @@
+import { GetStaticPaths, GetStaticProps } from 'next'
+import { MDXRemote, MDXRemoteSerializeResult } from 'next-mdx-remote'
+import { serialize } from 'next-mdx-remote/serialize'
+import { join, relative } from 'path'
+import rehypeSlug from 'rehype-slug'
+import remarkGfm from 'remark-gfm'
+import components from '~/components'
+import Layout from '~/layouts/DefaultGuideLayout'
+import { UrlTransformFunction, linkTransform } from '~/lib/mdx/rehypeLinkTransform'
+
+// We fetch these docs at build time from an external repo
+const org = 'supabase'
+const repo = 'vecs'
+const branch = 'main'
+const docsDir = 'docs'
+const externalSite = 'https://supabase.github.io/vecs'
+
+// Each external docs page is mapped to a local page
+const pageMap = [
+  {
+    slug: 'api',
+    meta: {
+      title: 'API',
+    },
+    remoteFile: 'api.md',
+  },
+  {
+    slug: 'collections',
+    meta: {
+      title: 'Collections',
+    },
+    remoteFile: 'concepts_collections.md',
+  },
+  {
+    slug: 'indexes',
+    meta: {
+      title: 'Indexes',
+    },
+    remoteFile: 'concepts_indexes.md',
+  },
+  {
+    slug: 'metadata',
+    meta: {
+      title: 'Metadata',
+    },
+    remoteFile: 'concepts_metadata.md',
+  },
+]
+
+interface PythonClientDocsProps {
+  source: MDXRemoteSerializeResult
+  meta: {
+    title: string
+    description?: string
+  }
+}
+
+export default function PythonClientDocs({ source, meta }: PythonClientDocsProps) {
+  return (
+    <Layout meta={meta}>
+      <MDXRemote {...source} components={components} />
+    </Layout>
+  )
+}
+
+/**
+ * Fetch markdown from external repo and transform links
+ */
+export const getStaticProps: GetStaticProps<PythonClientDocsProps> = async ({ params }) => {
+  const page = pageMap.find(({ slug }) => slug === params.slug)
+
+  if (!page) {
+    throw new Error(`No page mapping found for slug '${params.slug}'`)
+  }
+
+  const { remoteFile, meta } = page
+
+  const response = await fetch(
+    `https://raw.githubusercontent.com/${org}/${repo}/${branch}/${docsDir}/${remoteFile}`
+  )
+
+  const source = await response.text()
+
+  const urlTransform: UrlTransformFunction = (url) => {
+    try {
+      const externalSiteUrl = new URL(externalSite)
+
+      const placeholderHostname = 'placeholder'
+      const { hostname, pathname, hash } = new URL(url, `http://${placeholderHostname}`)
+
+      // Don't modify a url with a FQDN or a url that's only a hash
+      if (hostname !== placeholderHostname || pathname === '/') {
+        return url
+      }
+
+      const relativePage = (
+        pathname.endsWith('.md')
+          ? pathname.replace(/\.md$/, '')
+          : relative(externalSiteUrl.pathname, pathname)
+      ).replace(/^\//, '')
+
+      const page = pageMap.find(({ remoteFile }) => `${relativePage}.md` === remoteFile)
+
+      // If we have a mapping for this page, use the mapped path
+      if (page) {
+        return page.slug + hash
+      }
+
+      // If we don't have this page in our docs, link to original docs
+      return join(externalSite, relativePage)
+    } catch (err) {
+      console.error('Error transforming markdown URL', err)
+      return url
+    }
+  }
+
+  const mdxSource = await serialize(source, {
+    mdxOptions: {
+      remarkPlugins: [remarkGfm],
+      rehypePlugins: [[linkTransform, urlTransform], rehypeSlug],
+    },
+  })
+
+  return { props: { source: mdxSource, meta } }
+}
+
+export const getStaticPaths: GetStaticPaths = async () => {
+  return {
+    paths: pageMap.map(({ slug }) => ({
+      params: {
+        slug,
+      },
+    })),
+    fallback: false,
+  }
+}

package-lock.json

@@ -58,6 +58,7 @@
         "framer-motion": "^6.5.1",
         "github-slugger": "^2.0.0",
         "gray-matter": "^4.0.3",
+        "hast-util-has-property": "^2.0.1",
         "isbot": "^3.6.5",
         "jsrsasign": "^10.5.26",
         "lodash": "^4.17.21",
@@ -94,12 +95,15 @@
         "ui": "*",
         "unist-builder": "^3.0.1",
         "unist-util-filter": "^4.0.1",
+        "unist-util-visit": "^4.1.2",
         "uuid": "^9.0.0",
         "valtio": "^1.7.6"
       },
       "devDependencies": {
+        "@types/hast": "^2.3.4",
         "@types/node": "^17.0.12",
         "@types/react": "17.0.39",
+        "@types/unist": "^2.0.6",
         "config": "*",
         "dotenv": "^16.0.3",
         "ejs": "^3.1.8",
@@ -62892,8 +62896,10 @@
         "@supabase/auth-helpers-nextjs": "^0.5.6",
         "@supabase/auth-helpers-react": "^0.3.1",
         "@supabase/supabase-js": "^2.13.0",
+        "@types/hast": "^2.3.4",
         "@types/node": "^17.0.12",
         "@types/react": "17.0.39",
+        "@types/unist": "^2.0.6",
         "algoliasearch": "^4.14.2",
         "babel": "^6.23.0",
         "clsx": "^1.2.1",
@@ -62906,6 +62912,7 @@
         "github-slugger": "^2.0.0",
         "globby": "^12.0.2",
         "gray-matter": "^4.0.3",
+        "hast-util-has-property": "^2.0.1",
         "isbot": "^3.6.5",
         "jsrsasign": "^10.5.26",
         "lodash": "^4.17.21",
@@ -62950,6 +62957,7 @@
         "ui": "*",
         "unist-builder": "^3.0.1",
         "unist-util-filter": "^4.0.1",
+        "unist-util-visit": "^4.1.2",
         "uuid": "^9.0.0",
         "valtio": "^1.7.6"
       },