From c773072ae4e43bc8342bf73178e0fef0ae1d6c4d Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Tue, 30 May 2023 09:37:13 -0500 Subject: [PATCH] update structure/unstructured vector doc --- .../NavigationMenu.constants.ts | 4 +- .../ai/structured-unstructured-embeddings.mdx | 61 --------- .../guides/ai/structured-unstructured.mdx | 116 ++++++++++++++++++ 3 files changed, 118 insertions(+), 63 deletions(-) delete mode 100644 apps/docs/pages/guides/ai/structured-unstructured-embeddings.mdx create mode 100644 apps/docs/pages/guides/ai/structured-unstructured.mdx diff --git a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts index 665897064c0..58c5bb7d916 100644 --- a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts +++ b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts @@ -862,8 +862,8 @@ export const ai: NavMenuConstant = { { name: 'Overview', url: '/guides/ai' }, { name: 'Concepts', url: '/guides/ai/concepts' }, { - name: 'Structured & unstructured embeddings', - url: '/guides/ai/structured-unstructured-embeddings', + name: 'Structured & unstructured', + url: '/guides/ai/structured-unstructured', }, { name: 'Quickstarts', diff --git a/apps/docs/pages/guides/ai/structured-unstructured-embeddings.mdx b/apps/docs/pages/guides/ai/structured-unstructured-embeddings.mdx deleted file mode 100644 index e1a18bc4126..00000000000 --- a/apps/docs/pages/guides/ai/structured-unstructured-embeddings.mdx +++ /dev/null @@ -1,61 +0,0 @@ -import Layout from '~/layouts/DefaultGuideLayout' - -export const meta = { - id: 'structured-unstructured-embeddings', - title: 'Structured and unstructured embeddings', - description: - 'Supabase is flexible enough to provide structured and unstructured embeddings using pgvector.', - subtitle: - 'Supabase is flexible enough to provide structured and unstructured embeddings using pgvector.', - sidebar_label: 'Structured and unstructured embeddings', -} - -Most vector stores treat embeddings like NoSQL, unstructured data. Supabase is flexible enough to fit either a structured or an unstructured approach. - -Compare these code snippets: - -## Structured - -```sql -create table docs ( - id uuid primary key, - content text, - url string, - embedding vector(1536) -); - -insert into docs - (id, content, url, embedding) -values - ('79409372-7556-4ccc-ab8f-5786a6cfa4f7', 'Hello world', '/hello-world', [100, 200, 300]); -``` - -A structured approach is usually defined in SQL, and managed via database [migrations](/docs/guides/getting-started/local-development#database-migrations). - -## Unstructured - -```py -import vecs - -docs = vx.create_collection(name="docs", dimension=1536) - -docs.upsert(vectors=[ - ('79409372-7556-4ccc-ab8f-5786a6cfa4f7', [100, 200, 300], { url: '/hello-world' }) -]) - -``` - -An unstructured approach is usually defined in Python and has a looser table definition, storing metadata as a json document along with the embedding. - -## Choosing the right model - -Both approaches create a table where you can store your embeddings and some metadata. You should choose the best approach for your use-case. - -- Structured embeddings are typically co-located with some content that is already stored in your database. -- Unstructured embeddings are typically defined at runtime, better-suited for a large body of external content. - -Both approaches are fine, and the one you should choose depends on your use-case. - -export const Page = ({ children }) => - -export default Page diff --git a/apps/docs/pages/guides/ai/structured-unstructured.mdx b/apps/docs/pages/guides/ai/structured-unstructured.mdx new file mode 100644 index 00000000000..f2919c283c8 --- /dev/null +++ b/apps/docs/pages/guides/ai/structured-unstructured.mdx @@ -0,0 +1,116 @@ +import Layout from '~/layouts/DefaultGuideLayout' + +export const meta = { + id: 'structured-unstructured-embeddings', + title: 'Structured and Unstructured', + description: + 'Supabase is flexible enough to associate structured and unstructured metadata with embeddings.', + subtitle: + 'Supabase is flexible enough to associate structured and unstructured metadata with embeddings.', + sidebar_label: 'Structured and unstructured embeddings', +} + +Most vector stores treat metadata associated with embeddings like NoSQL, unstructured data. Supabase is flexible enough to store unstructured and structured metadata. + +## Structured + +```sql +create table docs ( + id uuid primary key, + embedding vector(3), + content text, + url string +); + +insert into docs + (id, content, url, embedding) +values + ('79409372-7556-4ccc-ab8f-5786a6cfa4f7', array[0.1, 0.2, 0.3], 'Hello world', '/hello-world'); +``` + +Notice that we've assoicated two pieces of metadata, `content` and `url`, with the embedding. Those fields can be filtered, constrained, indexed, and generally operated on using the full power of SQL. Structured metadata fits naturally with a traditional Supabase application, and can be managed via database [migrations](/docs/guides/getting-started/local-development#database-migrations). + +## Unstructured + +```sql +create table docs ( + id uuid primary key, + embedding vector(3), + meta jsonb +); + +insert into docs + (id, embedding, meta) +values + ( + '79409372-7556-4ccc-ab8f-5786a6cfa4f7', + array[0.1, 0.2, 0.3], + '{"content": "Hello world", "url": "/hello-world"}' + ); +``` + +An unstructured approach does not specify the metadata fields that are expected. It stores all metadata in a flexible `json`/`jsonb` column. The tradeoff is that the querying/filtering capabilities of a schemaless data type are less flexible than when each field has a dedicated column. It also pushes the burden of metadata data integrity onto application code, which is more error prone than enforcing constraints in the database. + +The unstructured approach is recommended: +- for ephemeral/interactive workloads e.g. data science or scientific research +- when metadata fields are user-defined or unknown +- during rapid prototyping + + +Client libraries like python's [vecs](https://github.com/supabase/vecs) use this structure. For example, running: + +```py +#!/usr/bin/env python3 +import vecs + +docs = vx.create_collection(name="docs", dimension=1536) + +docs.upsert(vectors=[ + ('79409372-7556-4ccc-ab8f-5786a6cfa4f7', [100, 200, 300], { url: '/hello-world' }) +]) + +``` + +automatically creates the unstructured SQL table during the call to `create_collection`. + +Note that when working with client libraries that emit SQL DDL, like `create table ...`, you should add that SQL to your migrations when moving to production to maintain a single source of truth for your database's schema. + +## Hybrid + +The structured metadata style is recommended when the fields being tracked are known in advance. If you have a combination of known and unknown metadata fields, you can accomodate the unknown fields by adding a `json`/`jsonb` column to the table. In that situation, known fields should continue to use dedicated columns for best query performance and throughput. + +```sql +create table docs ( + id uuid primary key, + embedding vector(3), + content text, + url string, + meta jsonb +); + +insert into docs + (id, embedding, meta) +values + ( + '79409372-7556-4ccc-ab8f-5786a6cfa4f7', + array[0.1, 0.2, 0.3], + 'Hello world', + '/hello-world', + '{"key": "value"}' + ); +``` + + + +## Choosing the right model + +Both approaches create a table where you can store your embeddings and some metadata. You should choose the best approach for your use-case. In summary: + +- Structured metadata is best when fields are known in advance or query patterns are predictable e.g. a production Supabase application +- Unstructured metadata is best when fields are unknown/user-defined or when working with data interactively e.g. exploratory research + +Both approaches are valid, and the one you should choose depends on your use-case. + +export const Page = ({ children }) => + +export default Page