diff --git a/.github/workflows/docs-search-v2-ingest.yml b/.github/workflows/docs-search-v2-ingest.yml new file mode 100644 index 00000000000..5f54f2babb5 --- /dev/null +++ b/.github/workflows/docs-search-v2-ingest.yml @@ -0,0 +1,53 @@ +name: Ingest Docs Content for Search V2 + +on: + push: + branches: + - master + paths: + - '.github/workflows/docs-search-v2-ingest.yml' + - 'apps/docs/content/**/*.mdx' + - 'apps/docs/scripts/search_v2/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-prod + cancel-in-progress: false + +permissions: + contents: read + +jobs: + ingest: + runs-on: blacksmith-4vcpu-ubuntu-2404 + + env: + SEARCH_V2_SUPABASE_PROJECT_ID: ${{ secrets.SEARCH_V2_SUPABASE_PROJECT_ID }} + SEARCH_V2_SUPABASE_SECRET_KEY: ${{ secrets.SEARCH_V2_SUPABASE_SECRET_KEY }} + + steps: + - name: Check out repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + sparse-checkout: | + apps/docs + packages + patches + + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + name: Install pnpm + with: + run_install: false + + - name: Setup node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: '.nvmrc' + + - name: Download dependencies + run: pnpm install --frozen-lockfile + + - name: Ingest content + working-directory: ./apps/docs + run: pnpm run search-v2:ingest diff --git a/apps/docs/package.json b/apps/docs/package.json index b71b1038541..862ba3687e3 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -38,9 +38,11 @@ "predev": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples", "preembeddings": "pnpm run codegen:references", "preinstall": "npx only-allow pnpm", + "presearch-v2:ingest": "pnpm run build:guides-markdown", "presync": "pnpm run codegen:graphql", "pretest": "pnpm run codegen:examples", "pretypecheck": "pnpm run codegen:graphql && next typegen", + "search-v2:ingest": "tsx scripts/search_v2/ingest.ts", "start": "next start", "sync": "tsx --conditions=react-server ./resources/rootSync.ts", "test": "pnpm supabase start && pnpm run test:local && pnpm supabase stop", @@ -162,11 +164,14 @@ "config": "workspace:*", "dotenv": "^16.0.3", "ejs": "^3.1.10", + "fast-glob": "^3.3.3", "globby": "^13.2.2", "graphiql": "^4.0.2", "npm-run-all": "^4.1.5", "openapi-types": "^12.1.3", "postcss": "catalog:", + "remark-frontmatter": "^5.0.0", + "remark-parse": "^11.0.0", "shiki": "^3.2.1", "shx": "^0.4.0", "simple-git": "^3.24.0", @@ -178,6 +183,7 @@ "tsx": "catalog:", "twoslash": "^0.3.1", "typescript": "catalog:", + "unified": "^11.0.5", "unist-util-visit-parents": "5.1.3", "vite": "catalog:", "vitest": "catalog:" diff --git a/apps/docs/public/markdown/manifest.json b/apps/docs/public/markdown/manifest.json index fe51488c706..5b32e748566 100644 --- a/apps/docs/public/markdown/manifest.json +++ b/apps/docs/public/markdown/manifest.json @@ -1 +1,850 @@ -[] +[ + "ai-tools", + "ai", + "api", + "auth", + "cli", + "cron", + "deployment", + "functions", + "getting-started", + "graphql", + "integrations", + "local-development", + "observability", + "platform", + "queues", + "realtime", + "resources", + "security", + "self-hosting", + "storage", + "ai/automatic-embeddings", + "ai/choosing-compute-addon", + "ai/concepts", + "ai/engineering-for-scale", + "ai/going-to-prod", + "ai/google-colab", + "ai/hugging-face", + "ai/hybrid-search", + "ai/keyword-search", + "ai/langchain", + "ai/python-clients", + "ai/rag-with-permissions", + "ai/semantic-search", + "ai/structured-unstructured", + "ai/vecs-python-client", + "ai/vector-columns", + "ai/vector-indexes", + "ai-tools/ai-prompts", + "ai-tools/ai-skills", + "ai-tools/byo-mcp", + "ai-tools/mcp", + "ai-tools/plugins", + "api/automatic-retries-in-supabase-js", + "api/creating-routes", + "api/custom-claims-and-role-based-access-control-rbac", + "api/handling-errors-in-supabase-js", + "api/quickstart", + "api/securing-your-api", + "api/sql-to-api", + "api/sql-to-rest", + "api/using-custom-schemas", + "auth/architecture", + "auth/audit-logs", + "auth/auth-anonymous", + "auth/auth-captcha", + "auth/auth-email-passwordless", + "auth/auth-email-templates", + "auth/auth-hooks", + "auth/auth-identity-linking", + "auth/auth-mfa", + "auth/auth-smtp", + "auth/auth-web3", + "auth/choosing-a-server-package", + "auth/custom-oauth-providers", + "auth/enterprise-sso", + "auth/general-configuration", + "auth/identities", + "auth/jwt-fields", + "auth/jwts", + "auth/managing-user-data", + "auth/native-mobile-deep-linking", + "auth/oauth-server", + "auth/passkeys", + "auth/password-security", + "auth/passwords", + "auth/phone-login", + "auth/rate-limits", + "auth/redirect-urls", + "auth/server-side", + "auth/sessions", + "auth/signing-keys", + "auth/signout", + "auth/social-login", + "auth/users", + "cron/install", + "cron/quickstart", + "database/arrays", + "database/beekeeper-studio", + "database/connecting-to-postgres", + "database/connection-management", + "database/custom-postgres-config", + "database/dbeaver", + "database/debugging-performance", + "database/drizzle", + "database/extensions", + "database/full-text-search", + "database/functions", + "database/import-data", + "database/joins-and-nesting", + "database/json", + "database/metabase", + "database/migrating-to-pg-partman", + "database/multigres", + "database/orioledb", + "database/overview", + "database/partitions", + "database/pgadmin", + "database/postgres-js", + "database/prisma", + "database/psql", + "database/query-optimization", + "database/replication", + "database/secure-data", + "database/supavisor", + "database/tables", + "database/testing", + "database/vault", + "database/views", + "database/webhooks", + "deployment/branching", + "deployment/database-migrations", + "deployment/going-into-prod", + "deployment/managing-environments", + "deployment/maturity-model", + "deployment/shared-responsibility-model", + "deployment/terraform", + "functions/ai-models", + "functions/architecture", + "functions/auth-headers", + "functions/auth-legacy-jwt", + "functions/auth", + "functions/background-tasks", + "functions/compression", + "functions/connect-to-postgres", + "functions/cors", + "functions/dart-edge", + "functions/debugging-tools", + "functions/dependencies", + "functions/deploy", + "functions/development-environment", + "functions/development-tips", + "functions/ephemeral-storage", + "functions/error-codes", + "functions/error-handling", + "functions/function-configuration", + "functions/http-methods", + "functions/kysely-postgres", + "functions/limits", + "functions/logging", + "functions/pricing", + "functions/quickstart-dashboard", + "functions/quickstart", + "functions/recursive-functions", + "functions/regional-invocation", + "functions/routing", + "functions/schedule-functions", + "functions/secrets", + "functions/status-codes", + "functions/storage-caching", + "functions/unit-test", + "functions/wasm", + "functions/websockets", + "getting-started/api-keys", + "getting-started/architecture", + "getting-started/features", + "getting-started/migrating-to-new-api-keys", + "graphql/api", + "graphql/computed-fields", + "graphql/configuration", + "graphql/functions", + "graphql/security", + "graphql/views", + "graphql/with-apollo", + "graphql/with-relay", + "integrations/build-a-supabase-oauth-integration", + "integrations/partner-catalog", + "integrations/partner-integration-guide", + "integrations/stripe-projects", + "integrations/supabase-for-platforms", + "integrations/vercel-marketplace", + "local-development/cli-workflows", + "local-development/customizing-email-templates", + "local-development/database-migrations", + "local-development/declarative-database-schemas", + "local-development/managing-config", + "local-development/restoring-downloaded-backup", + "local-development/seeding-your-database", + "observability/access-data", + "observability/advanced-log-filtering", + "observability/advisors", + "observability/automate-with-agents", + "observability/client-side-tracing", + "observability/detecting", + "observability/inspect", + "observability/log-drains", + "observability/log-field-reference", + "observability/logs", + "observability/metrics", + "observability/reports", + "observability/sentry-monitoring", + "platform/access-control", + "platform/aws-marketplace", + "platform/backups", + "platform/billing-faq", + "platform/billing-on-supabase", + "platform/clone-project", + "platform/compute-and-disk", + "platform/cost-control", + "platform/credits", + "platform/custom-domains", + "platform/database-size", + "platform/delete-project", + "platform/free-project-pausing", + "platform/get-set-up-for-billing", + "platform/hipaa-projects", + "platform/ipv4-address", + "platform/manage-your-subscription", + "platform/manage-your-usage", + "platform/migrating-to-supabase", + "platform/migrating-within-supabase", + "platform/multi-factor-authentication", + "platform/network-restrictions", + "platform/performance", + "platform/permissions", + "platform/personal-access-tokens", + "platform/postgres-connection-logging", + "platform/privatelink", + "platform/project-transfer", + "platform/read-replicas", + "platform/regions", + "platform/ssl-enforcement", + "platform/sso", + "platform/temporary-access", + "platform/upgrading", + "platform/your-monthly-invoice", + "queues/api", + "queues/consuming-messages-with-edge-functions", + "queues/expose-self-hosted-queues", + "queues/pgmq", + "queues/quickstart", + "realtime/architecture", + "realtime/authorization", + "realtime/benchmarks", + "realtime/broadcast", + "realtime/concepts", + "realtime/error_codes", + "realtime/getting_started", + "realtime/limits", + "realtime/postgres-changes", + "realtime/presence", + "realtime/pricing", + "realtime/protocol", + "realtime/realtime-listening-flutter", + "realtime/realtime-user-presence", + "realtime/realtime-with-nextjs", + "realtime/reports", + "realtime/settings", + "realtime/subscribing-to-database-changes", + "resources/glossary", + "security/gdpr-compliance", + "security/hipaa-compliance", + "security/npm-security", + "security/platform-audit-logs", + "security/platform-security", + "security/product-security", + "security/security-testing", + "security/soc-2-compliance", + "self-hosting/accessing-postgres", + "self-hosting/copy-from-platform-s3", + "self-hosting/custom-email-templates", + "self-hosting/custom-postgres-extensions", + "self-hosting/docker", + "self-hosting/enable-mcp", + "self-hosting/postgres-upgrade-17", + "self-hosting/remove-superuser-access", + "self-hosting/restore-from-platform", + "self-hosting/self-hosted-auth-hooks", + "self-hosting/self-hosted-auth-keys", + "self-hosting/self-hosted-envoy", + "self-hosting/self-hosted-functions", + "self-hosting/self-hosted-oauth", + "self-hosting/self-hosted-passkeys", + "self-hosting/self-hosted-phone-mfa", + "self-hosting/self-hosted-proxy-https", + "self-hosting/self-hosted-s3", + "self-hosting/self-hosted-saml-sso", + "self-hosting/updating", + "storage/pricing", + "storage/quickstart", + "ai/python/api", + "ai/python/collections", + "ai/python/indexes", + "ai/python/metadata", + "ai/examples/building-chatgpt-plugins", + "ai/examples/headless-vector-search", + "ai/examples/huggingface-image-captioning", + "ai/examples/image-search-openai-clip", + "ai/examples/mixpeek-video-search", + "ai/examples/nextjs-vector-search", + "ai/examples/openai", + "ai/examples/semantic-image-search-amazon-titan", + "ai/integrations/amazon-bedrock", + "ai/integrations/llamaindex", + "ai/integrations/roboflow", + "ai/quickstarts/face-similarity", + "ai/quickstarts/generate-text-embeddings", + "ai/quickstarts/hello-world", + "ai/quickstarts/text-deduplication", + "ai/vector-indexes/hnsw-indexes", + "ai/vector-indexes/ivf-indexes", + "api/rest/auto-generated-docs", + "api/rest/client-libs", + "api/rest/generating-python-types", + "api/rest/generating-types", + "api/rest/postgrest-error-codes", + "auth/auth-mfa/phone", + "auth/auth-mfa/totp", + "auth/debugging/error-codes", + "auth/auth-hooks/before-user-created-hook", + "auth/auth-hooks/custom-access-token-hook", + "auth/auth-hooks/mfa-verification-hook", + "auth/auth-hooks/password-verification-hook", + "auth/auth-hooks/send-email-hook", + "auth/auth-hooks/send-sms-hook", + "auth/enterprise-sso/auth-sso-saml", + "auth/oauth-server/getting-started", + "auth/oauth-server/mcp-authentication", + "auth/oauth-server/oauth-flows", + "auth/oauth-server/token-security", + "auth/sessions/implicit-flow", + "auth/sessions/pkce-flow", + "auth/social-login/auth-apple", + "auth/social-login/auth-azure", + "auth/social-login/auth-bitbucket", + "auth/social-login/auth-discord", + "auth/social-login/auth-facebook", + "auth/social-login/auth-figma", + "auth/social-login/auth-github", + "auth/social-login/auth-gitlab", + "auth/social-login/auth-google", + "auth/social-login/auth-kakao", + "auth/social-login/auth-keycloak", + "auth/social-login/auth-linkedin", + "auth/social-login/auth-notion", + "auth/social-login/auth-slack", + "auth/social-login/auth-spotify", + "auth/social-login/auth-twitch", + "auth/social-login/auth-twitter", + "auth/social-login/auth-workos", + "auth/social-login/auth-zoom", + "auth/quickstarts/astrojs", + "auth/quickstarts/nextjs", + "auth/quickstarts/react-native", + "auth/quickstarts/react", + "auth/quickstarts/with-expo-react-native-social-auth", + "auth/server-side/advanced-guide", + "auth/server-side/creating-a-client", + "auth/server-side/migrating-to-ssr-from-auth-helpers", + "auth/third-party/auth0", + "auth/third-party/aws-cognito", + "auth/third-party/clerk", + "auth/third-party/firebase-auth", + "auth/third-party/overview", + "auth/third-party/workos", + "database/connecting-to-postgres/pooling-and-limits", + "database/connecting-to-postgres/serverless-drivers", + "database/extensions/http", + "database/extensions/hypopg", + "database/extensions/index_advisor", + "database/extensions/pg_cron", + "database/extensions/pg_graphql", + "database/extensions/pg_hashids", + "database/extensions/pg_jsonschema", + "database/extensions/pg_net", + "database/extensions/pg_partman", + "database/extensions/pg_plan_filter", + "database/extensions/pg_repack", + "database/extensions/pg_stat_statements", + "database/extensions/pgaudit", + "database/extensions/pgjwt", + "database/extensions/pgmq", + "database/extensions/pgroonga", + "database/extensions/pgrouting", + "database/extensions/pgsodium", + "database/extensions/pgtap", + "database/extensions/pgvector", + "database/extensions/plpgsql_check", + "database/extensions/plv8", + "database/extensions/postgis", + "database/extensions/postgres_fdw", + "database/extensions/rum", + "database/extensions/timescaledb", + "database/extensions/uuid-ossp", + "database/multigres/compatibility", + "database/postgres/cascade-deletes", + "database/postgres/column-level-security", + "database/postgres/configuration", + "database/postgres/data-deletion", + "database/postgres/dropping-all-tables-in-schema", + "database/postgres/enums", + "database/postgres/event-triggers", + "database/postgres/first-row-in-group", + "database/postgres/indexes", + "database/postgres/postgres-log-config", + "database/postgres/roles-superuser", + "database/postgres/roles", + "database/postgres/row-level-security-performance", + "database/postgres/row-level-security", + "database/postgres/setup-replication-external", + "database/postgres/timeouts", + "database/postgres/triggers", + "database/postgres/which-version-of-postgres", + "database/prisma/prisma-troubleshooting", + "database/replication/bigquery", + "database/replication/clickhouse", + "database/replication/ducklake", + "database/replication/manual-replication-faq", + "database/replication/manual-replication-monitoring", + "database/replication/manual-replication-setup", + "database/replication/pipelines-faq", + "database/replication/pipelines-monitoring", + "database/replication/pipelines", + "database/replication/snowflake", + "deployment/branching/configuration", + "deployment/branching/dashboard", + "deployment/branching/github-integration", + "deployment/branching/integrations", + "deployment/branching/troubleshooting", + "deployment/branching/working-with-branches", + "deployment/ci/backups", + "deployment/ci/generating-types", + "deployment/ci/testing", + "deployment/terraform/reference", + "deployment/terraform/tutorial", + "functions/examples/amazon-bedrock-image-generator", + "functions/examples/auth-send-email-hook-react-email-resend", + "functions/examples/cloudflare-turnstile", + "functions/examples/discord-bot", + "functions/examples/elevenlabs-generate-speech-stream", + "functions/examples/elevenlabs-transcribe-speech", + "functions/examples/github-actions", + "functions/examples/image-manipulation", + "functions/examples/mcp-server-mcp-lite", + "functions/examples/og-image", + "functions/examples/push-notifications", + "functions/examples/rate-limiting", + "functions/examples/resumable-websockets", + "functions/examples/screenshots", + "functions/examples/semantic-search", + "functions/examples/send-emails", + "functions/examples/sentry-monitoring", + "functions/examples/slack-bot-mention", + "functions/examples/stripe-webhooks", + "functions/examples/telegram-bot", + "functions/examples/upstash-redis", + "getting-started/quickstarts/astrojs", + "getting-started/quickstarts/expo-react-native", + "getting-started/quickstarts/flask", + "getting-started/quickstarts/flutter", + "getting-started/quickstarts/hono", + "getting-started/quickstarts/ios-swiftui", + "getting-started/quickstarts/kotlin", + "getting-started/quickstarts/laravel", + "getting-started/quickstarts/nextjs", + "getting-started/quickstarts/nuxtjs", + "getting-started/quickstarts/reactjs", + "getting-started/quickstarts/redwoodjs", + "getting-started/quickstarts/refine", + "getting-started/quickstarts/reflex", + "getting-started/quickstarts/ruby-on-rails", + "getting-started/quickstarts/solidjs", + "getting-started/quickstarts/spring-boot", + "getting-started/quickstarts/sveltekit", + "getting-started/quickstarts/tanstack", + "getting-started/quickstarts/vue", + "integrations/build-a-supabase-oauth-integration/oauth-scopes", + "getting-started/tutorials/with-angular", + "getting-started/tutorials/with-expo-react-native", + "getting-started/tutorials/with-flutter", + "getting-started/tutorials/with-ionic-angular", + "getting-started/tutorials/with-ionic-react", + "getting-started/tutorials/with-ionic-vue", + "getting-started/tutorials/with-kotlin", + "getting-started/tutorials/with-nextjs", + "getting-started/tutorials/with-nuxt-3", + "getting-started/tutorials/with-react", + "getting-started/tutorials/with-redwoodjs", + "getting-started/tutorials/with-refine", + "getting-started/tutorials/with-solidjs", + "getting-started/tutorials/with-svelte", + "getting-started/tutorials/with-sveltekit", + "getting-started/tutorials/with-swift", + "getting-started/tutorials/with-vue-3", + "local-development/cli/getting-started", + "local-development/cli/testing-and-linting", + "observability/metrics/grafana-cloud", + "observability/metrics/grafana-self-hosted", + "observability/metrics/vendor-agnostic", + "local-development/testing/overview", + "local-development/testing/pgtap-extended", + "observability/automate-with-agents/all", + "observability/automate-with-agents/health", + "observability/automate-with-agents/performance", + "observability/automate-with-agents/security", + "observability/automate-with-agents/usage", + "platform/aws-marketplace/account-setup", + "platform/aws-marketplace/faq", + "platform/aws-marketplace/getting-started", + "platform/aws-marketplace/invoices", + "platform/aws-marketplace/manage-your-subscription", + "platform/manage-your-usage/advanced-mfa-phone", + "platform/manage-your-usage/branching", + "platform/manage-your-usage/compute", + "platform/manage-your-usage/custom-domains", + "platform/manage-your-usage/disk-iops", + "platform/manage-your-usage/disk-size", + "platform/manage-your-usage/disk-throughput", + "platform/manage-your-usage/edge-function-invocations", + "platform/manage-your-usage/egress", + "platform/manage-your-usage/ipv4", + "platform/manage-your-usage/log-drains", + "platform/manage-your-usage/logs-ingest", + "platform/manage-your-usage/logs-query", + "platform/manage-your-usage/logs", + "platform/manage-your-usage/monthly-active-users-sso", + "platform/manage-your-usage/monthly-active-users-third-party", + "platform/manage-your-usage/monthly-active-users", + "platform/manage-your-usage/pipelines", + "platform/manage-your-usage/point-in-time-recovery", + "platform/manage-your-usage/read-replicas", + "platform/manage-your-usage/realtime-messages", + "platform/manage-your-usage/realtime-peak-connections", + "platform/manage-your-usage/storage-image-transformations", + "platform/manage-your-usage/storage-size", + "platform/mfa/org-mfa-enforcement", + "platform/migrating-to-supabase/amazon-rds", + "platform/migrating-to-supabase/auth0", + "platform/migrating-to-supabase/firebase-auth", + "platform/migrating-to-supabase/firebase-storage", + "platform/migrating-to-supabase/firestore-data", + "platform/migrating-to-supabase/heroku", + "platform/migrating-to-supabase/mssql", + "platform/migrating-to-supabase/mysql", + "platform/migrating-to-supabase/neon", + "platform/migrating-to-supabase/postgres", + "platform/migrating-to-supabase/render", + "platform/migrating-to-supabase/vercel-postgres", + "platform/read-replicas/getting-started", + "platform/migrating-within-supabase/backup-restore", + "platform/migrating-within-supabase/dashboard-restore", + "platform/sso/azure", + "platform/sso/choosing-login-flow", + "platform/sso/enterprise-mcp-authentication", + "platform/sso/gsuite", + "platform/sso/login-flows", + "platform/sso/multiple-providers", + "platform/sso/okta", + "platform/sso/testing-best-practices", + "storage/analytics/connecting-to-analytics-bucket", + "storage/analytics/creating-analytics-buckets", + "storage/analytics/introduction", + "storage/analytics/limits", + "storage/analytics/pricing", + "storage/analytics/query-with-postgres", + "storage/buckets/creating-buckets", + "storage/buckets/fundamentals", + "storage/cdn/fundamentals", + "storage/cdn/metrics", + "storage/cdn/purge-cdn-cache", + "storage/cdn/smart-cdn", + "storage/management/copy-move-objects", + "storage/management/delete-objects", + "storage/management/download-objects", + "storage/debugging/error-codes", + "storage/debugging/logs", + "storage/production/scaling", + "storage/s3/authentication", + "storage/s3/compatibility", + "storage/serving/bandwidth", + "storage/serving/downloads", + "storage/serving/image-transformations", + "storage/schema/custom-roles", + "storage/schema/design", + "storage/schema/helper-functions", + "storage/security/access-control", + "storage/security/ownership", + "storage/uploads/file-limits", + "storage/uploads/resumable-uploads", + "storage/uploads/s3-uploads", + "storage/uploads/standard-uploads", + "storage/vector/creating-vector-buckets", + "storage/vector/introduction", + "storage/vector/limits", + "storage/vector/local-development", + "storage/vector/querying-vectors", + "storage/vector/storing-vectors", + "storage/vector/working-with-indexes", + "database/extensions/wrappers/airtable", + "database/extensions/wrappers/auth0", + "database/extensions/wrappers/bigquery", + "database/extensions/wrappers/cal", + "database/extensions/wrappers/calendly", + "database/extensions/wrappers/clerk", + "database/extensions/wrappers/clickhouse", + "database/extensions/wrappers/cloudflare-d1", + "database/extensions/wrappers/cognito", + "database/extensions/wrappers/duckdb", + "database/extensions/wrappers/dynamodb", + "database/extensions/wrappers/firebase", + "database/extensions/wrappers/gravatar", + "database/extensions/wrappers/hubspot", + "database/extensions/wrappers/iceberg", + "database/extensions/wrappers/infura", + "database/extensions/wrappers/logflare", + "database/extensions/wrappers/mongodb", + "database/extensions/wrappers/mssql", + "database/extensions/wrappers/mysql", + "database/extensions/wrappers/notion", + "database/extensions/wrappers/openapi", + "database/extensions/wrappers/orb", + "database/extensions/wrappers/overview", + "database/extensions/wrappers/paddle", + "database/extensions/wrappers/redis", + "database/extensions/wrappers/s3", + "database/extensions/wrappers/s3_vectors", + "database/extensions/wrappers/shopify", + "database/extensions/wrappers/slack", + "database/extensions/wrappers/snowflake", + "database/extensions/wrappers/stripe", + "storage/analytics/examples/apache-spark", + "storage/analytics/examples/duckdb", + "storage/analytics/examples/pyiceberg", + "troubleshooting/42501--permission-denied-for-table-httprequestqueue-KnozmQ", + "troubleshooting/FWD-wrappers-error-component-verification-failed-94a76b", + "troubleshooting/all-about-supabase-egress-a_Sg_e", + "troubleshooting/an-invalid-response-was-received-from-the-upstream-server-error-when-querying-auth-RI4Vl-", + "troubleshooting/app-store-rejection-tls-error-in-ipv6-only-environments-4e9c62", + "troubleshooting/are-all-features-available-in-self-hosted-supabase-THPcqw", + "troubleshooting/auth-error-401-invalid-claim-missing-sub--AFwMR", + "troubleshooting/auth-error-500-database-error-querying-schema-eb6b44", + "troubleshooting/auth-error-503-authretryablefetcherror-51b88c", + "troubleshooting/auth-hooks-invalid-payload-when-anonymous-users-attempt-phone-changes-022c47", + "troubleshooting/autovacuum-stalled-due-to-inactive-replication-slot-d55aa2", + "troubleshooting/avoiding-timeouts-in-long-running-queries-6nmbdN", + "troubleshooting/branch-in-migrations-failed-status", + "troubleshooting/canceling-statement-due-to-statement-timeout-581wFv", + "troubleshooting/cant-access-supabase-project-lovable-cloud", + "troubleshooting/certain-operations-are-too-complex-to-perform-directly-using-the-client-libraries-8JaphH", + "troubleshooting/change-email-associated-with-supabase-account-T5eHNT", + "troubleshooting/change-project-region-eWJo5Z", + "troubleshooting/check-usage-for-monthly-active-users-mau-MwZaBs", + "troubleshooting/cloudflare-origin-error-1016-on-custom-domain-a57af4", + "troubleshooting/converted-github-account-to-organisation---lost-supabase-account-access-5wsE_1", + "troubleshooting/customizing-emails-by-language-KZ_38Q", + "troubleshooting/dashboard-errors-when-managing-users-N1ls4A", + "troubleshooting/database-api-42501-errors", + "troubleshooting/database-error-remaining-connection-slots-are-reserved-for-non-replication-superuser-connections-3V3nIb", + "troubleshooting/database-error-saving-new-user-RU_EwB", + "troubleshooting/deprecated-rls-features-Pm77Zs", + "troubleshooting/disabling-prepared-statements-qL8lEL", + "troubleshooting/discovering-and-interpreting-api-errors-in-the-logs-7xREI9", + "troubleshooting/disk-size-not-shrinking-after-deleting-data-135390", + "troubleshooting/do-i-need-to-expose-security-definer-functions-in-row-level-security-policies-iI0uOw", + "troubleshooting/download-logical-backups", + "troubleshooting/edge-function-401-error-response", + "troubleshooting/edge-function-404-error-response", + "troubleshooting/edge-function-500-error-response", + "troubleshooting/edge-function-503-response", + "troubleshooting/edge-function-504-error-response", + "troubleshooting/edge-function-546-error-response", + "troubleshooting/edge-function-bundle-size-issues", + "troubleshooting/edge-function-cpu-limits", + "troubleshooting/edge-function-dependency-analysis", + "troubleshooting/edge-function-error-rate-limit-exceeded-for-trace-1094d3", + "troubleshooting/edge-function-fails-deploy", + "troubleshooting/edge-function-monitoring-resource-usage", + "troubleshooting/edge-function-shutdown-reasons-explained", + "troubleshooting/edge-function-takes-too-long-to-respond", + "troubleshooting/edge-function-wall-clock-time-limit-reached-Nk38bW", + "troubleshooting/edge-functions-worker-timeouts-and-websocket-drops", + "troubleshooting/email-password-login-disabled-supabase-vercel-marketplace-a7dd36", + "troubleshooting/email-template-not-updating", + "troubleshooting/enabling-ipv4-addon", + "troubleshooting/error-connection-refused-when-trying-to-connect-to-supabase-database-hwG0Dr", + "troubleshooting/error-index-row-size-exceeds-btree-version-4-maximum-for-index-LMmoeU", + "troubleshooting/error-invalid-byte-sequence-for-encoding-utf8-0x00-when-accessing-triggers-or-webhooks-e78cf8", + "troubleshooting/error-invalid-totp-code-entered-CukLCj", + "troubleshooting/error-no-pghbaconf-entry-for-host-xxxxxxxxxxx-user-postgres-database-postgres-ssl-off-GOt5Ja", + "troubleshooting/error-prepared-statement-xxx-already-exists-3laqeM", + "troubleshooting/error-relation-supabase_migrationsschema_migrations-does-not-exist-a787d6", + "troubleshooting/error-target-organization-is-not-managed-by-vercel-marketplace-during-project-transfer-5262c3", + "troubleshooting/exhaust-disk-io", + "troubleshooting/exhaust-ram", + "troubleshooting/exhaust-swap", + "troubleshooting/failed-to-fetch-in-dashboard-and-other-areas----browser-extension-dyDTRU", + "troubleshooting/failed-to-restore-from-backup-all-subscriptions-and-replication-slots-must-be-dropped-before-a-backup-can-be-restored-L-rCvt", + "troubleshooting/failed-to-retrieve-tables", + "troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "troubleshooting/fatal-password-authentication-failed", + "troubleshooting/fetch-requests-to-api-endpoints-arent-showing-the-session-UbUwRs", + "troubleshooting/fixing-520-errors-in-the-database-rest-api-Ur5-B2", + "troubleshooting/forbidden-resource-error-from-the-cli-L6rm6l", + "troubleshooting/formatting-support-requests-for-faster-resolution-616717", + "troubleshooting/get-detailed-storage-metrics-with-the-aws-cli-587a7d", + "troubleshooting/google-auth-fails-for-some-users-XcFXEu", + "troubleshooting/grafana-not-displaying-data-sXJrMj", + "troubleshooting/high-cpu-and-infinite-transaction-retries-when-using-custom-error-codes-in-rpc-functions-77326b", + "troubleshooting/high-cpu-and-slow-queries-with-error-must-be-a-superuser-to-terminate-superuser-process", + "troubleshooting/high-cpu-usage", + "troubleshooting/high-latency-with-supabase-client-z0pZzR", + "troubleshooting/how-can-i-revoke-execution-of-a-postgresql-function-2GYb0A", + "troubleshooting/how-do-i-check-gotrueapi-version-of-a-supabase-project-lQAnOR", + "troubleshooting/how-do-i-make-the-cookies-httponly-vwweFx", + "troubleshooting/how-do-i-reset-my-supabase-database-password-oTs5sB", + "troubleshooting/how-do-i-update-connection-pool-settings-in-my-dashboard-wAxTJ_", + "troubleshooting/how-do-you-troubleshoot-nextjs---supabase-auth-issues-riMCZV", + "troubleshooting/how-long-does-it-take-to-restore-a-database-from-a-point-in-time-backup-pitr-qO8gOG", + "troubleshooting/how-postgres-chooses-which-index-to-use-_JHrf4", + "troubleshooting/how-to-bypass-cooldown-period", + "troubleshooting/how-to-change-max-database-connections-_BQ8P5", + "troubleshooting/how-to-check-if-my-queries-are-being-blocked-by-other-queries-NSKtR1", + "troubleshooting/how-to-delete-a-role-in-postgres-8-AvxY", + "troubleshooting/how-to-delete-vercel-linked-projects-9d08aa", + "troubleshooting/how-to-interpret-and-explore-the-postgres-logs-OuCIOj", + "troubleshooting/how-to-migrate-from-supabase-auth-helpers-to-ssr-package-5NRunM", + "troubleshooting/how-to-view-database-metrics-uqf2z_", + "troubleshooting/http-api-issues", + "troubleshooting/http-status-codes", + "troubleshooting/i-am-not-receiving-password-reset-emails-for-supabase-dashboard--cO5yf", + "troubleshooting/identify-lovable-cloud-or-supabase-backend", + "troubleshooting/importing-stripe-or-other-modules-from-esmsh-on-deno-edge-functions-throws-an-error-TmbB5p", + "troubleshooting/increase-vector-lookup-speeds-by-applying-an-hsnw-index-ohLHUM", + "troubleshooting/inserting-into-sequenceserial-table-causes-duplicate-key-violates-unique-constraint-error-pi6DnC", + "troubleshooting/inspecting-edge-function-environment-variables-wg5qOQ", + "troubleshooting/insufficient-privilege-when-accessing-pgstatstatements-e5M_EQ", + "troubleshooting/interpreting-supabase-grafana-cpu-charts-9JSlkC", + "troubleshooting/interpreting-supabase-grafana-io-charts-MUynDR", + "troubleshooting/issues-serving-edge-functions-locally", + "troubleshooting/issues-with-rotating-pgsodium-and-vault-root-encryption-keys-efeb47", + "troubleshooting/jwt-expired-error-in-supabase-dashboard-F06k3x", + "troubleshooting/keeping-free-projects-after-pro-upgrade-Kf9Xm2", + "troubleshooting/kong-stops-responding-under-heavy-load-locally", + "troubleshooting/lost-accessforgot-the-mfa-device-nAPT-7", + "troubleshooting/managing-or-disconnecting-github-oauth-connections-e3dc3b", + "troubleshooting/manually-created-databases-are-not-visible-in-the-supabase-dashboard-4415aa", + "troubleshooting/memory-and-swap-usage-explained-aPNgm0", + "troubleshooting/migrating-auth-users-between-projects", + "troubleshooting/monitor-supavisor-postgres-connections", + "troubleshooting/new-branch-doesnt-copy-database", + "troubleshooting/new-variable-is-null-in-a-trigger-function--l9AOZ", + "troubleshooting/nextjs-1314-stale-data-when-changing-rls-or-table-data-85b8oQ", + "troubleshooting/no-toast-messages-on-the-dashboard-BrvP8h", + "troubleshooting/not-receiving-auth-emails-from-the-supabase-project-OFSNzw", + "troubleshooting/oauth-sign-in-isnt-redirecting-on-the-server-side-ShGMtr", + "troubleshooting/otp-verification-failures-token-has-expired-or-otp_expired-errors-5ee4d0", + "troubleshooting/partitioning-an-existing-table-with-same-name-VEnbzK", + "troubleshooting/pausing-pro-projects-vNL-2a", + "troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", + "troubleshooting/permission-denied-when-deleting-the-cli_login_postgres-role-808bae", + "troubleshooting/pg_cron-launcher-crashes-with-duplicate-key-value-violates-unique-constraint-cc6472", + "troubleshooting/pgcron-debugging-guide-n1KTaz", + "troubleshooting/pgrst106-the-schema-must-be-one-of-the-following-error-when-querying-an-exposed-schema", + "troubleshooting/pkce-flow-errors-cannot-parse-response-or-zgotmplz-in-magic-link-emails-433665", + "troubleshooting/postgres-error-could-not-access-status-of-transaction-or-could-not-open-file-pg_xact-f88389", + "troubleshooting/postgrest-error-400-column-example_tableexample_column-does-not-exist-when-using-or-operators-46ff23", + "troubleshooting/postgrest-error-pgrst002-could-not-query-the-database-for-the-schema-cache-c396e9", + "troubleshooting/postgrest-not-recognizing-new-columns-or-functions-bd75f5", + "troubleshooting/postgrest-not-recognizing-objects-in-schema", + "troubleshooting/powerbi-service-error-the-remote-certificate-is-invalid-according-to-the-validation-procedure-640a98", + "troubleshooting/prisma-error-management-Cm5P_o", + "troubleshooting/project-status-reports-unhealthy-services", + "troubleshooting/rclone-error-s3-protocol-error-received-listing-v1-with-istruncated-set-no-nextmarker-and-no-contents-e64d34", + "troubleshooting/realtime-client-presence-rate-limit-reached", + "troubleshooting/realtime-concurrent-peak-connections-quota-jdDqcp", + "troubleshooting/realtime-connections-timed_out-status", + "troubleshooting/realtime-debugging-with-logger", + "troubleshooting/realtime-egress-faq", + "troubleshooting/realtime-handling-silent-disconnections-in-backgrounded-applications-592794", + "troubleshooting/realtime-heartbeat-messages", + "troubleshooting/realtime-postgres-changes-troubleshooting", + "troubleshooting/realtime-project-suspended-for-exceeding-quotas", + "troubleshooting/realtime-too-many-channels-error", + "troubleshooting/realtime-warn-sending-broadcast-message", + "troubleshooting/refresh-postgrest-schema", + "troubleshooting/relation-objects-does-not-exist-error-during-storage-uploads-8f21f0", + "troubleshooting/resolving-42p01-relation-does-not-exist-error-W4_9-V", + "troubleshooting/resolving-500-status-authentication-errors-7bU5U8", + "troubleshooting/resolving-cannot-execute-update-in-a-read-only-transaction-on-transaction-pooler-connections-ef582c", + "troubleshooting/resolving-database-hostname-and-managing-your-ip-address-pVlwE0", + "troubleshooting/restore-project-after-90-days-pause", + "troubleshooting/rls-performance-and-best-practices-Z5Jjwv", + "troubleshooting/rls-simplified-BJTcS8", + "troubleshooting/rotating-anon-service-and-jwt-secrets-1Jq6yd", + "troubleshooting/running-explain-analyze-on-functions", + "troubleshooting/scan-error-on-column-confirmation_token-converting-null-to-string-is-unsupported-during-auth-login-a0c686", + "troubleshooting/schema-pg_pgrst_no_exposed_schemas-does-not-exist", + "troubleshooting/security-of-anonymous-sign-ins-iOrGCL", + "troubleshooting/seeing-no-pghbaconf-entry-for-host-errors-in-postgres-and-they-come-from-an-ip-address-that-i-dont-recognize-4gds9f", + "troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V", + "troubleshooting/slow-execution-of-alter-table-on-large-table-when-changing-column-type-qmZRpZ", + "troubleshooting/soft-deletes-with-supabase-js", + "troubleshooting/sso-error-you-do-not-have-permissions-to-join-this-organization-or-prompts-to-create-new-organization", + "troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "troubleshooting/storage-error-403-forbidden-new-row-violates-row-level-security-policy-on-upload-a94384", + "troubleshooting/storage-error-database-schema-is-incompatible-when-uploading-files-988baa", + "troubleshooting/storage-unexpectedly-high-usage-or-exceed_storage_size_quota-errors-ae21a5", + "troubleshooting/supabase--your-network-ipv4-and-ipv6-compatibility-cHe3BP", + "troubleshooting/supabase-cli-failed-sasl-auth-or-invalid-scram-server-final-message", + "troubleshooting/supabase-cli-transport-error-antivirus-firewall-tls-interception", + "troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", + "troubleshooting/supabase-grafana-memory-charts", + "troubleshooting/supabase-project-provisioned-via-bolt-not-visible-in-dashboard-7188fc", + "troubleshooting/supabase-storage-inefficient-folder-operations-and-hierarchical-rls-challenges-b05a4d", + "troubleshooting/supavisor-and-connection-terminology-explained-9pr_ZO", + "troubleshooting/supavisor-connection-error-fatal-eauthquery-unsupported-or-invalid-secret-format-d23dd4", + "troubleshooting/supavisor-error-circuit-breaker-open-after-password-rotation-0fdb72", + "troubleshooting/supavisor-error-password-authentication-failed-after-password-rotation", + "troubleshooting/supavisor-faq-YyP5tI", + "troubleshooting/tenant-or-user-not-found", + "troubleshooting/too-many-connections-for-database-postgres", + "troubleshooting/tracking-postgres-role-activity-to-specific-dashboard-users-8d3715", + "troubleshooting/transfer-edge-function-from-one-project-to-another", + "troubleshooting/transferring-from-cloud-to-self-host-in-supabase-2oWNvW", + "troubleshooting/troubleshooting-connect_timeout-or-hanging-queries-in-vercel-serverless-functions-775f92", + "troubleshooting/unable-to-call-edge-function", + "troubleshooting/unable-to-deploy-edge-function", + "troubleshooting/understanding-postgresql-explain-output-Un9dqX", + "troubleshooting/understanding-postgresql-logging-levels-and-how-they-impact-your-project-KXiJRm", + "troubleshooting/understanding-the-usage-summary-on-the-dashboard-D7Gnle", + "troubleshooting/unexpected-behavior-with-authupdateuser-phone-phone-linked-to-incorrect-user-id-45368f", + "troubleshooting/unused-external-import-warning-vite-rollup", + "troubleshooting/upload-file-size-restrictions-Y4wQLT", + "troubleshooting/using-google-smtp-with-supabase-custom-smtp-ZZzU4Y", + "troubleshooting/using-sqlalchemy-with-supabase-FUqebT", + "troubleshooting/vercel-integration-environment-variables-not-syncing-for-persistent-git-branches-b9191e", + "troubleshooting/webhook-debugging-guide-M8sk47", + "troubleshooting/why-am-i-being-redirected-to-the-wrong-url-when-using-auth-redirectto-option-_vqIeO", + "troubleshooting/why-are-there-gaps-in-my-postgres-id-sequence-Frifus", + "troubleshooting/why-cant-i-uploadlistetc-my-public-bucket-Z6CmGt", + "troubleshooting/why-do-i-see-auth--api-requests-in-the-dashboard-my-app-has-no-users-CyadiO", + "troubleshooting/why-is-my-camelcase-name-not-working-in-postgres-functions-or-rls-policies-EJMzVd", + "troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", + "troubleshooting/why-is-my-service-role-key-client-getting-rls-errors-or-not-returning-data-7_1K9z", + "troubleshooting/why-is-my-supabase-api-call-not-returning-PGzXw0", + "troubleshooting/why-supabase-edge-functions-cannot-provide-static-egress-ips-for-whitelisting-3d78b0", + "troubleshooting/will-backups-be-accessible-from-the-dashboard-immediately-after-upgrading-to-a-paid-plan-hXY4rs", + "troubleshooting/working-around-the-edge-function-secrets-limit", + "troubleshooting" +] diff --git a/apps/docs/scripts/search_v2/ingest.ts b/apps/docs/scripts/search_v2/ingest.ts new file mode 100644 index 00000000000..89767484b7e --- /dev/null +++ b/apps/docs/scripts/search_v2/ingest.ts @@ -0,0 +1,112 @@ +import '../utils/dotenv.js' + +import { readFile } from 'node:fs/promises' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import fg from 'fast-glob' + +import { parsePage } from './markdown.js' +import { filePathToSlug } from './routes.js' +import { createSupabaseClient, TABLE_NAME } from './supabase.js' + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +/** Folder that holds the markdown files. Searched recursively. */ +export const CONTENT_ROOT = path.resolve(process.cwd(), 'public/markdown') + +/** Glob patterns (relative to CONTENT_ROOT) for the files to ingest. */ +const FILE_PATTERNS = ['**/*.md', '**/*.mdx'] + +/** Glob patterns (relative to CONTENT_ROOT) for files/folders to skip. */ +const IGNORED_PATTERNS = ['reference/**', 'guides/troubleshooting/**', '_partials/**'] + +/** How many rows to send to Supabase per insert request. */ +const INSERT_BATCH_SIZE = 200 + +// --------------------------------------------------------------------------- + +interface SectionRow { + slug: string + file_path: string + page_title: string + heading: string + heading_level: number + heading_path: string[] + content: string + excerpt: string +} + +/** Build the DB rows for a single markdown file. Pure: no I/O. */ +export function buildRows(markdown: string, filePath: string, contentRoot: string): SectionRow[] { + const slug = filePathToSlug(filePath, contentRoot) + const relativePath = path.relative(contentRoot, filePath).split(path.sep).join('/') + const page = parsePage(markdown) + const pageTitle = page.title || path.basename(slug) || 'Untitled' + + return page.sections.map((section) => ({ + slug, + file_path: relativePath, + page_title: pageTitle, + heading: section.heading, + heading_level: section.level, + heading_path: section.headingPath, + content: section.content, + excerpt: page.excerpt, + })) +} + +function chunk(items: T[], size: number): T[][] { + const out: T[][] = [] + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)) + return out +} + +async function main() { + const supabase = createSupabaseClient() + + const files = await fg(FILE_PATTERNS, { + cwd: CONTENT_ROOT, + absolute: true, + onlyFiles: true, + ignore: IGNORED_PATTERNS, + }) + if (files.length === 0) { + console.warn(`No markdown files found under ${CONTENT_ROOT}`) + return + } + console.log(`Found ${files.length} markdown file(s) under ${CONTENT_ROOT}`) + + let totalRows = 0 + + for (const file of files.sort()) { + const markdown = await readFile(file, 'utf8') + const rows = buildRows(markdown, file, CONTENT_ROOT) + const slug = rows[0]?.slug ?? filePathToSlug(file, CONTENT_ROOT) + + // Replace everything previously ingested for this page. + const { error: deleteError } = await supabase.from(TABLE_NAME).delete().eq('slug', slug) + if (deleteError) throw new Error(`Failed to clear "${slug}": ${deleteError.message}`) + + for (const batch of chunk(rows, INSERT_BATCH_SIZE)) { + const { error: insertError } = await supabase.from(TABLE_NAME).insert(batch) + if (insertError) throw new Error(`Failed to insert "${slug}": ${insertError.message}`) + } + + totalRows += rows.length + console.log(` ✓ ${slug || '(root)'} (${rows.length} section${rows.length === 1 ? '' : 's'})`) + } + + console.log(`Done. Ingested ${totalRows} section(s) from ${files.length} file(s).`) +} + +// Only run when executed directly (so buildRows can be imported by tests). +const isDirectRun = + !!process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) +if (isDirectRun) { + main().catch((err) => { + console.error(err instanceof Error ? err.message : err) + process.exit(1) + }) +} diff --git a/apps/docs/scripts/search_v2/markdown.test.ts b/apps/docs/scripts/search_v2/markdown.test.ts new file mode 100644 index 00000000000..5695f2e8c84 --- /dev/null +++ b/apps/docs/scripts/search_v2/markdown.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from 'vitest' + +import { + extractExcerpt, + extractSections, + extractTitle, + nodeToText, + parseMarkdownAst, + parsePage, +} from './markdown.js' + +const USERS_MD = `# Users + +A **user** in Supabase Auth is someone with a user ID, stored in the Auth schema. You can restrict access via [RLS policies](https://example.com). + +## Permanent and anonymous users + +Supabase distinguishes between permanent and anonymous users. + +- **Permanent users** are tied to PII. +- **Anonymous users** aren't tied to any identities. + +## Inviting users + +You can invite someone by email. + +### Using the Dashboard + +1. Go to **Authentication > Users**. +2. Click **Add user**. + +### Using the Auth Admin API + +Call \`inviteUserByEmail()\` from a server. + +\`\`\`js +const x = 1 +\`\`\` +` + +describe('extractTitle', () => { + it('returns the first H1', () => { + expect(extractTitle(parseMarkdownAst(USERS_MD))).toBe('Users') + }) + + it('returns an empty string when there is no H1', () => { + expect(extractTitle(parseMarkdownAst('## Only an h2\n\nText.'))).toBe('') + }) + + it('strips inline formatting from the heading', () => { + expect(extractTitle(parseMarkdownAst('# The `code` **title**'))).toBe('The code title') + }) +}) + +describe('extractExcerpt', () => { + it('returns the first paragraph as plain text', () => { + expect(extractExcerpt(parseMarkdownAst(USERS_MD))).toBe( + 'A user in Supabase Auth is someone with a user ID, stored in the Auth schema. You can restrict access via RLS policies.' + ) + }) + + it('skips YAML frontmatter', () => { + const md = `---\ntitle: Hello\n---\n\n# Title\n\nFirst paragraph.` + expect(extractExcerpt(parseMarkdownAst(md))).toBe('First paragraph.') + }) + + it('returns an empty string when there is no paragraph', () => { + expect(extractExcerpt(parseMarkdownAst('# Just a heading'))).toBe('') + }) +}) + +describe('extractSections', () => { + const sections = extractSections(parseMarkdownAst(USERS_MD)) + + it('creates one section per heading', () => { + expect(sections.map((s) => s.heading)).toEqual([ + 'Users', + 'Permanent and anonymous users', + 'Inviting users', + 'Using the Dashboard', + 'Using the Auth Admin API', + ]) + }) + + it('records heading levels', () => { + expect(sections.map((s) => s.level)).toEqual([1, 2, 2, 3, 3]) + }) + + it('builds the heading path from the H1 down to the section', () => { + expect(sections[1].headingPath).toEqual(['Users', 'Permanent and anonymous users']) + expect(sections[3].headingPath).toEqual(['Users', 'Inviting users', 'Using the Dashboard']) + }) + + it('resets the path when a sibling heading appears', () => { + expect(sections[4].headingPath).toEqual(['Users', 'Inviting users', 'Using the Auth Admin API']) + }) + + it('collects body text (paragraphs and lists) under each heading', () => { + expect(sections[1].content).toContain( + 'Supabase distinguishes between permanent and anonymous users.' + ) + expect(sections[1].content).toContain('Permanent users are tied to PII.') + expect(sections[1].content).toContain("Anonymous users aren't tied to any identities.") + }) + + it('does not leak content across sections', () => { + expect(sections[0].content).not.toContain('Supabase distinguishes') + expect(sections[1].content).not.toContain('invite') + }) + + it('includes code block text in the section body', () => { + expect(sections[4].content).toContain('const x = 1') + }) + + it('keeps text before the first heading as a headless section', () => { + const md = `Intro paragraph.\n\n# Title\n\nBody.` + const result = extractSections(parseMarkdownAst(md)) + expect(result[0]).toEqual({ + heading: '', + level: 0, + headingPath: [], + content: 'Intro paragraph.', + }) + expect(result[1].headingPath).toEqual(['Title']) + }) +}) + +describe('nodeToText', () => { + it('separates table cells and rows so words do not run together', () => { + const md = `| Attr | Type |\n| --- | --- |\n| id | string |\n| aud | string |` + const table = parseMarkdownAst(md).children[0] + expect(nodeToText(table)).toBe('Attr | Type\nid | string\naud | string') + }) + + it('puts list items on separate lines', () => { + const list = parseMarkdownAst('- one\n- two').children[0] + expect(nodeToText(list)).toBe('one\ntwo') + }) +}) + +describe('parsePage', () => { + it('combines title, excerpt and sections', () => { + const page = parsePage(USERS_MD) + expect(page.title).toBe('Users') + expect(page.excerpt.startsWith('A user in Supabase Auth')).toBe(true) + expect(page.sections).toHaveLength(5) + }) +}) diff --git a/apps/docs/scripts/search_v2/markdown.ts b/apps/docs/scripts/search_v2/markdown.ts new file mode 100644 index 00000000000..7fb4c26d334 --- /dev/null +++ b/apps/docs/scripts/search_v2/markdown.ts @@ -0,0 +1,139 @@ +import { toString } from 'mdast-util-to-string' +import remarkFrontmatter from 'remark-frontmatter' +import remarkGfm from 'remark-gfm' +import remarkParse from 'remark-parse' +import { unified } from 'unified' + +const processor = unified().use(remarkParse).use(remarkFrontmatter, ['yaml']).use(remarkGfm) + +/** + * Types are derived from the processor's own output rather than imported from + * `mdast` directly: this app pins `@types/mdast` v3 for its wider MDX pipeline, + * while `remark-parse` v11 resolves its own (structurally incompatible) v4 + * types for its nested `@types/mdast` dependency. + */ +export type Root = ReturnType +/** Any child of the root node (paragraph, heading, list, table, ...). */ +type Content = Root['children'][number] +type Heading = Extract +/** Union of every node type that can appear in the tree (root or content). */ +type Nodes = Root | Content + +export interface Section { + /** Text of this section's heading ('' for text that appears before any heading). */ + heading: string + /** 1..6, or 0 for text before any heading. */ + level: number + /** Headings from the H1 down to this section, e.g. ['Users', 'Permanent and anonymous users']. */ + headingPath: string[] + /** Plain text of everything under the heading until the next heading. */ + content: string +} + +export interface ParsedPage { + title: string + excerpt: string + sections: Section[] +} + +/** Parse markdown into an mdast tree (GFM tables/lists + YAML frontmatter supported). */ +export function parseMarkdownAst(markdown: string): Root { + return processor.parse(markdown) +} + +/** + * Convert a node to plain text. Like mdast-util-to-string but keeps words apart + * for block containers (list items on their own line, table cells separated by " | "). + */ +export function nodeToText(node: Nodes): string { + switch (node.type) { + case 'tableRow': + return node.children.map((cell) => toString(cell).trim()).join(' | ') + case 'table': + case 'list': + case 'listItem': + case 'blockquote': + return (node.children as Nodes[]).map(nodeToText).filter(Boolean).join('\n') + default: + return toString(node).trim() + } +} + +/** The first H1 of the document, or '' if there is none. */ +export function extractTitle(tree: Root): string { + const h1 = tree.children.find( + (node): node is Heading => node.type === 'heading' && node.depth === 1 + ) + return h1 ? nodeToText(h1) : '' +} + +/** The first paragraph of the document (used as the page excerpt), or ''. */ +export function extractExcerpt(tree: Root): string { + const paragraph = tree.children.find((node) => node.type === 'paragraph') + return paragraph ? nodeToText(paragraph) : '' +} + +/** Nodes that carry no searchable text. */ +function isIgnored(node: Content): boolean { + return ( + node.type === 'yaml' || + node.type === 'thematicBreak' || + node.type === 'html' || + node.type === 'definition' + ) +} + +/** + * Split the document into sections: one per heading, each with the text below it + * (until the next heading) and the path of ancestor headings leading to it. + */ +export function extractSections(tree: Root): Section[] { + const sections: Section[] = [] + const stack: Array<{ text: string; level: number }> = [] + let current: Section | null = null + let body: string[] = [] + + const flush = () => { + if (current) { + current.content = body.join('\n\n').trim() + if (current.heading || current.content) sections.push(current) + } + body = [] + } + + for (const node of tree.children) { + if (isIgnored(node)) continue + + if (node.type === 'heading') { + flush() + while (stack.length && stack[stack.length - 1].level >= node.depth) stack.pop() + const text = nodeToText(node) + stack.push({ text, level: node.depth }) + current = { + heading: text, + level: node.depth, + headingPath: stack.map((s) => s.text), + content: '', + } + continue + } + + // Body text before the first heading gets a headless section. + if (!current) current = { heading: '', level: 0, headingPath: [], content: '' } + const text = nodeToText(node) + if (text) body.push(text) + } + + flush() + return sections +} + +/** Everything the ingester needs from one markdown file. */ +export function parsePage(markdown: string): ParsedPage { + const tree = parseMarkdownAst(markdown) + return { + title: extractTitle(tree), + excerpt: extractExcerpt(tree), + sections: extractSections(tree), + } +} diff --git a/apps/docs/scripts/search_v2/routes.test.ts b/apps/docs/scripts/search_v2/routes.test.ts new file mode 100644 index 00000000000..9fc322d8c2b --- /dev/null +++ b/apps/docs/scripts/search_v2/routes.test.ts @@ -0,0 +1,23 @@ +import path from 'node:path' +import { describe, expect, it } from 'vitest' + +import { filePathToSlug } from './routes.js' + +const root = path.resolve('/content') + +describe('filePathToSlug', () => { + it('strips the content root and the .md extension', () => { + expect(filePathToSlug(path.join(root, 'guides', 'auth', 'users.md'), root)).toBe( + 'guides/auth/users' + ) + }) + + it('handles .mdx files', () => { + expect(filePathToSlug(path.join(root, 'intro.mdx'), root)).toBe('intro') + }) + + it('maps index files to their folder', () => { + expect(filePathToSlug(path.join(root, 'guides', 'index.md'), root)).toBe('guides') + expect(filePathToSlug(path.join(root, 'index.md'), root)).toBe('') + }) +}) diff --git a/apps/docs/scripts/search_v2/routes.ts b/apps/docs/scripts/search_v2/routes.ts new file mode 100644 index 00000000000..accc0e7f089 --- /dev/null +++ b/apps/docs/scripts/search_v2/routes.ts @@ -0,0 +1,19 @@ +import path from 'node:path' + +const MARKDOWN_EXTENSIONS = new Set(['.md', '.mdx', '.markdown']) + +/** + * Turn an absolute file path into a slug relative to the content root, without extension. + * /root/guides/auth/users.md -> 'guides/auth/users' + * /root/guides/index.md -> 'guides' + * /root/index.md -> '' + */ +export function filePathToSlug(filePath: string, contentRoot: string): string { + const relative = path.relative(contentRoot, filePath) + const parsed = path.parse(relative) + const name = MARKDOWN_EXTENSIONS.has(parsed.ext.toLowerCase()) ? parsed.name : parsed.base + + const segments = parsed.dir ? parsed.dir.split(path.sep).filter(Boolean) : [] + if (name !== 'index') segments.push(name) + return segments.join('/') +} diff --git a/apps/docs/scripts/search_v2/sql/setup.sql b/apps/docs/scripts/search_v2/sql/setup.sql new file mode 100644 index 00000000000..203f1212183 --- /dev/null +++ b/apps/docs/scripts/search_v2/sql/setup.sql @@ -0,0 +1,92 @@ + +-- Run this in the Supabase SQL editor before `pnpm search-v2:ingest`. +-- Safe to re-run: it uses IF NOT EXISTS / CREATE OR REPLACE. + +-- One row per markdown *section* (a heading + the text below it, up to the next heading). +create table if not exists public.docs_sections ( + id bigint generated always as identity primary key, + slug text not null, -- 'guides/auth/users' (route without base path) + file_path text not null, -- path relative to the content root + page_title text not null default '', -- the page's H1 + heading text not null default '', -- this section's heading + heading_level smallint not null default 0, -- 1..6, 0 = text before any heading + heading_path text[] not null default '{}',-- ['Users', 'Permanent and anonymous users'] + content text not null default '', -- plain text of the section body + excerpt text not null default '', -- first paragraph of the page + updated_at timestamptz not null default now(), + + -- Weighted search document, kept up to date by Postgres automatically. + -- A = H1 (page title) + -- B = H2 + -- C = H3 + -- D = H4+ headings and all body text + -- ts_rank_cd() weights these A > B > C > D by default ({0.1, 0.2, 0.4, 1.0} for D, C, B, A). + fts tsvector generated always as ( + setweight(to_tsvector('english', coalesce(page_title, '')), 'A') + || setweight( + to_tsvector('english', coalesce(heading, '')), + (case heading_level when 1 then 'A' when 2 then 'B' when 3 then 'C' else 'D' end)::"char" + ) + || setweight(to_tsvector('english', coalesce(content, '')), 'D') + ) stored +); + +create index if not exists docs_sections_fts_idx on public.docs_sections using gin (fts); +create index if not exists docs_sections_slug_idx on public.docs_sections (slug); + +-- Read-only access for anon/authenticated (the ingest script writes with the secret key, which bypasses RLS). +alter table public.docs_sections enable row level security; + +drop policy if exists "docs_sections are publicly readable" on public.docs_sections; +create policy "docs_sections are publicly readable" + on public.docs_sections for select + to anon, authenticated + using (true); + +-- Search RPC: called from the server as supabase.rpc('search_docs', { query_text, match_limit }). +-- Score is ts_rank_cd normalised to 0..1 (normalization flag 32 => rank / (rank + 1)) and scaled to an +-- integer 0..1000 so the API can return a plain integer. +drop function if exists public.search_docs(text, int); + +create or replace function public.search_docs(query_text text, match_limit int default 10) +returns table ( + slug text, + page_title text, + heading text, + heading_level smallint, + heading_path text[], + excerpt text, + score int +) +language sql +stable +as $$ + with q as ( + select websearch_to_tsquery('english', query_text) as tsq + ), + ranked as ( + select + d.slug, + d.page_title, + d.heading, + d.heading_level, + d.heading_path, + d.excerpt, + round(ts_rank_cd(d.fts, q.tsq, 32) * 1000)::int as score + from public.docs_sections d, q + where d.fts @@ q.tsq + ), + -- A page can have several matching sections; keep only its best-scoring one so the + -- same route doesn't take up multiple slots in the results. + best_per_page as ( + select distinct on (r.slug) r.* + from ranked r + order by r.slug, r.score desc, r.heading_level asc + ) + select b.slug, b.page_title, b.heading, b.heading_level, b.heading_path, b.excerpt, b.score + from best_per_page b + order by b.score desc, b.heading_level asc, b.slug asc + limit greatest(1, least(coalesce(match_limit, 10), 100)); +$$; + +grant execute on function public.search_docs(text, int) to anon, authenticated, service_role; diff --git a/apps/docs/scripts/search_v2/supabase.ts b/apps/docs/scripts/search_v2/supabase.ts new file mode 100644 index 00000000000..d139bcd69fb --- /dev/null +++ b/apps/docs/scripts/search_v2/supabase.ts @@ -0,0 +1,22 @@ +import { createClient, type SupabaseClient } from '@supabase/supabase-js' + +export const TABLE_NAME = 'docs_sections' +export const SEARCH_FUNCTION = 'search_docs' + +function requireEnv(name: string): string { + const value = process.env[name] + if (!value) { + throw new Error(`Missing ${name}. Set it in .env.local.`) + } + return value +} + +/** Server-side client built from SEARCH_V2_SUPABASE_PROJECT_ID + SEARCH_V2_SUPABASE_SECRET_KEY in .env.local. */ +export function createSupabaseClient(): SupabaseClient { + const projectId = requireEnv('SEARCH_V2_SUPABASE_PROJECT_ID') + const secretKey = requireEnv('SEARCH_V2_SUPABASE_SECRET_KEY') + + return createClient(`https://${projectId}.supabase.co`, secretKey, { + auth: { autoRefreshToken: false, persistSession: false, detectSessionInUrl: false }, + }) +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d549984d8f7..b3dcceb5402 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -666,6 +666,9 @@ importers: ejs: specifier: ^3.1.10 version: 3.1.10 + fast-glob: + specifier: ^3.3.3 + version: 3.3.3 globby: specifier: ^13.2.2 version: 13.2.2 @@ -681,6 +684,12 @@ importers: postcss: specifier: ^8.5.18 version: 8.5.28 + remark-frontmatter: + specifier: ^5.0.0 + version: 5.0.0(supports-color@8.1.1) + remark-parse: + specifier: ^11.0.0 + version: 11.0.0(supports-color@8.1.1) shiki: specifier: ^3.2.1 version: 3.13.0 @@ -714,6 +723,9 @@ importers: typescript: specifier: 'catalog:' version: 6.0.2 + unified: + specifier: ^11.0.5 + version: 11.0.5 unist-util-visit-parents: specifier: 5.1.3 version: 5.1.3 @@ -12378,10 +12390,6 @@ packages: resolution: {integrity: sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==} engines: {node: '>=8.6.0'} - fast-glob@3.3.2: - resolution: {integrity: sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==} - engines: {node: '>=8.6.0'} - fast-glob@3.3.3: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} @@ -12427,6 +12435,9 @@ packages: fault@1.0.4: resolution: {integrity: sha512-CJ0HCB5tL5fYTEA7ToAq5+kTwd++Borf1/bifxd9iT70QcXr4MRrO3Llf8Ifs70q+SJcGHFtnIE/Nw6giCtECA==} + fault@2.0.1: + resolution: {integrity: sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ==} + fb-watchman@2.0.2: resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} @@ -14441,6 +14452,9 @@ packages: mdast-util-from-markdown@2.0.2: resolution: {integrity: sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA==} + mdast-util-frontmatter@2.0.1: + resolution: {integrity: sha512-LRqI9+wdgC25P0URIJY9vwocIzCcksduHQ9OF2joxQoyTNVduwLAFUzjoopuRJbJAReaKrNQKAZKL3uCMugWJA==} + mdast-util-gfm-autolink-literal@1.0.3: resolution: {integrity: sha512-My8KJ57FYEy2W2LyNom4n3E7hKTuQk/0SES0u16tjA9Z3oFkF4RrC/hPAPgjlSpezsOvI8ObcXcElo92wn5IGA==} @@ -14605,6 +14619,9 @@ packages: micromark-core-commonmark@2.0.1: resolution: {integrity: sha512-CUQyKr1e///ZODyD1U3xit6zXwy1a8q2a1S1HKtIlmgvurrEpaw/Y9y6KSIbF8P59cn/NjzHyO+Q2fAyYLQrAA==} + micromark-extension-frontmatter@2.0.0: + resolution: {integrity: sha512-C4AkuM3dA58cgZha7zVnuVxBhDsbttIMiytjgsM2XbHAB2faRVaHRle40558FBN+DJcrLNCoqG5mlrpdU4cRtg==} + micromark-extension-gfm-autolink-literal@1.0.5: resolution: {integrity: sha512-z3wJSLrDf8kRDOh2qBtoTRD53vJ+CWIyo7uyZuxf/JAbNJjiHsOpG1y5wxk8drtv3ETAHutCu6N3thkOOgueWg==} @@ -16720,6 +16737,9 @@ packages: resolution: {integrity: sha512-QwhAzNk27Ol64uV4z/3n55MKrNz9bhr8wg+mO5aGqIYDS+jUarS1d8Y0ZIeEBVhfGkXj6gGYM+727sOgAPvV/A==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + remark-frontmatter@5.0.0: + resolution: {integrity: sha512-XTFYvNASMe5iPN0719nPrdItC9aU0ssC4v14mH1BCi1u0n1gAocqcujWUrByftZTbLhRtiKRyjYTSIOcr69UVQ==} + remark-gfm@4.0.1: resolution: {integrity: sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==} @@ -26157,7 +26177,7 @@ snapshots: exsolve: 1.1.1 lightningcss: 1.33.0 pathe: 2.0.3 - picomatch: 4.0.5 + picomatch: 4.0.7 seroval: 1.6.2 source-map: 0.7.6 srvx: 0.11.22 @@ -30215,14 +30235,6 @@ snapshots: merge2: 1.4.1 micromatch: 4.0.8 - fast-glob@3.3.2: - dependencies: - '@nodelib/fs.stat': 2.0.5 - '@nodelib/fs.walk': 1.2.8 - glob-parent: 5.1.2 - merge2: 1.4.1 - micromatch: 4.0.8 - fast-glob@3.3.3: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -30275,6 +30287,10 @@ snapshots: dependencies: format: 0.2.2 + fault@2.0.1: + dependencies: + format: 0.2.2 + fb-watchman@2.0.2: dependencies: bser: 2.1.1 @@ -30715,7 +30731,7 @@ snapshots: globby@13.2.2: dependencies: dir-glob: 3.0.1 - fast-glob: 3.3.2 + fast-glob: 3.3.3 ignore: 5.2.4 merge2: 1.4.1 slash: 4.0.0 @@ -32449,6 +32465,17 @@ snapshots: transitivePeerDependencies: - supports-color + mdast-util-frontmatter@2.0.1(supports-color@8.1.1): + dependencies: + '@types/mdast': 4.0.4 + devlop: 1.1.0 + escape-string-regexp: 5.0.0 + mdast-util-from-markdown: 2.0.2(supports-color@8.1.1) + mdast-util-to-markdown: 2.1.0 + micromark-extension-frontmatter: 2.0.0 + transitivePeerDependencies: + - supports-color + mdast-util-gfm-autolink-literal@1.0.3: dependencies: '@types/mdast': 3.0.15 @@ -32831,6 +32858,13 @@ snapshots: micromark-util-symbol: 2.0.1 micromark-util-types: 2.0.0 + micromark-extension-frontmatter@2.0.0: + dependencies: + fault: 2.0.1 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.0 + micromark-extension-gfm-autolink-literal@1.0.5: dependencies: micromark-util-character: 1.2.0 @@ -35730,6 +35764,15 @@ snapshots: mdast-util-find-and-replace: 2.2.2 node-emoji: 1.11.0 + remark-frontmatter@5.0.0(supports-color@8.1.1): + dependencies: + '@types/mdast': 4.0.4 + mdast-util-frontmatter: 2.0.1(supports-color@8.1.1) + micromark-extension-frontmatter: 2.0.0 + unified: 11.0.5 + transitivePeerDependencies: + - supports-color + remark-gfm@4.0.1(supports-color@8.1.1): dependencies: '@types/mdast': 4.0.4 @@ -37968,7 +38011,7 @@ snapshots: dependencies: cac: 7.0.0 es-module-lexer: 2.3.2 - obug: 2.1.1 + obug: 2.1.4 pathe: 2.0.3 vite: 8.2.1(@types/node@22.13.14)(esbuild@0.28.1)(jiti@2.7.0)(sass@1.77.4)(terser@5.48.0)(tsx@4.22.4)(yaml@2.9.0) transitivePeerDependencies: @@ -38003,7 +38046,7 @@ snapshots: dependencies: ansis: 4.3.1 error-stack-parser-es: 1.0.5 - obug: 2.1.1 + obug: 2.1.4 ohash: 2.0.11 open: 11.0.0 perfect-debounce: 2.1.0