From 2d0bcd4714af7623ec4cabd07589fb0b4084cfef Mon Sep 17 00:00:00 2001 From: Pamela Chia Date: Thu, 25 Jun 2026 20:56:23 +0800 Subject: [PATCH] feat(telemetry): classify funnel creation errors (#47293) ## Summary The KPI-3 friction dashboard needs to know *why* users hit errors on the signup, project-creation, and org-creation funnels, not just that they did. The existing `dashboard_error_created` event already fires for these paths (10% sampled, with `$pathname`), but carries no reason: ~98.5% of events have no `errorType` and no property carries an error message. This adds PII-safe classification computed client-side from a controlled vocabulary, so raw error text never leaves the browser. Validation errors (previously invisible, since they are inline form errors that never raise a toast) are now captured on invalid submit. ## Changes - Extend `dashboard_error_created` with `origin`, `errorCategory`, `errorReason`, `errorCode`, and a `form` source value - Add a pure, unit-tested classifier (`funnel-errors.ts`) and a 10%-sampled tracking hook (`use-track-funnel-error.ts`); the classifier maps errors to stable slugs and emits only slugs + HTTP status, never raw message text - Classify signup errors (API failures + validation) in `SignUpForm` - Classify project-creation errors (API failures, OrioleDB guard, validation) in the new-project wizard - Classify org-creation errors (API failures, payment/card declines, confirm-subscription, validation) in `NewOrgForm` ## Testing 13 unit tests cover every classifier branch (validation / api / network / payment, status-code handling, message-pattern matching, and fallbacks). To verify on the Vercel preview (events are 10% sampled; set the sample rate to 1 locally to observe each fire): - Signup with a weak but non-empty password: `origin=signup, source=form, errorCategory=validation, errorReason=password_invalid` - Signup with an already-registered email: `origin=signup, source=toast, errorCategory=api, errorReason=email_already_registered` - New project with an empty name: `origin=project_creation, source=form, errorReason=project_name_invalid` - New org with an empty name: `origin=org_creation, source=form, errorReason=org_name_missing` - New org with a declined test card: `origin=org_creation, errorCategory=payment` PII: raw `error.message` is never sent; only controlled slugs and HTTP status. Dashboard consumers must filter `origin IS NOT NULL` so these do not collide with the generic toast events the global tracker still emits. ## Linear - fixes FE-3691 ## Summary by CodeRabbit * **New Features** * Added improved, categorized telemetry for signup, project creation, and organization creation errors, including payment, subscription-change, and validation failures. * Extended dashboard error events with optional structured diagnostics (origin, category, reason, and optional error code) and support for form-origin reporting. * **Bug Fixes** * Improved project-creation handling to record a validation telemetry event when an Oriole image is unavailable. * Ensured payment-related and subscription-change failures are captured consistently alongside existing user toasts. * **Tests** * Added unit tests covering API/network/validation/Stripe error classification and reason mapping. --- .../Organization/NewOrg/NewOrgForm.tsx | 30 +++- .../interfaces/SignIn/SignUpForm.tsx | 13 +- .../lib/telemetry/funnel-errors.test.ts | 114 ++++++++++++++ apps/studio/lib/telemetry/funnel-errors.ts | 142 ++++++++++++++++++ .../lib/telemetry/use-track-funnel-error.ts | 28 ++++ apps/studio/pages/new/[slug].tsx | 22 ++- packages/common/telemetry-constants.ts | 18 ++- 7 files changed, 363 insertions(+), 4 deletions(-) create mode 100644 apps/studio/lib/telemetry/funnel-errors.test.ts create mode 100644 apps/studio/lib/telemetry/funnel-errors.ts create mode 100644 apps/studio/lib/telemetry/use-track-funnel-error.ts diff --git a/apps/studio/components/interfaces/Organization/NewOrg/NewOrgForm.tsx b/apps/studio/components/interfaces/Organization/NewOrg/NewOrgForm.tsx index a7c147df6d1..c21083f1d1f 100644 --- a/apps/studio/components/interfaces/Organization/NewOrg/NewOrgForm.tsx +++ b/apps/studio/components/interfaces/Organization/NewOrg/NewOrgForm.tsx @@ -58,6 +58,12 @@ import { useLastVisitedOrganization } from '@/hooks/misc/useLastVisitedOrganizat import { PRICING_TIER_LABELS_ORG, STRIPE_PUBLIC_KEY } from '@/lib/constants' import { validateReturnTo } from '@/lib/gotrue' import { useProfile } from '@/lib/profile' +import { + classifyApiError, + classifyStripeError, + classifyValidationError, +} from '@/lib/telemetry/funnel-errors' +import { useTrackFunnelError } from '@/lib/telemetry/use-track-funnel-error' interface NewOrgFormProps { onPaymentMethodReset: () => void @@ -226,6 +232,8 @@ export const NewOrgForm = ({ [freeOrgs, projectsByOrg] ) + const trackFunnelError = useTrackFunnelError() + const { mutate: createOrganization } = useOrganizationCreateMutation({ onSuccess: async (org) => { if ('pending_payment_intent_secret' in org && org.pending_payment_intent_secret) { @@ -236,6 +244,7 @@ export const NewOrgForm = ({ }, onError: (data) => { toast.error(data.message, { duration: 10_000 }) + trackFunnelError('org_creation', classifyApiError('org_creation', data), 'toast') setNewOrgLoading(false) }, }) @@ -246,6 +255,10 @@ export const NewOrgForm = ({ onOrganizationCreated({ slug: data.slug }) } }, + onError: (error) => { + toast.error(error.message, { dismissible: true, duration: 10_000 }) + trackFunnelError('org_creation', classifyApiError('org_creation', error), 'toast') + }, }) const paymentIntentConfirmed = async (paymentIntentConfirmation: PaymentIntentResult) => { @@ -260,6 +273,11 @@ export const NewOrgForm = ({ size: form.getValues('size'), }) } else { + trackFunnelError( + 'org_creation', + classifyStripeError(paymentIntentConfirmation.error), + 'toast' + ) // If the payment intent is not successful, we reset the payment method and show an error toast.error(`Could not confirm payment. Please try again or use a different card.`, { duration: 10_000, @@ -354,7 +372,12 @@ export const NewOrgForm = ({ return (
- + + trackFunnelError('org_creation', classifyValidationError('org_creation', errors), 'form') + )} + id={FORM_ID} + > @@ -547,6 +570,11 @@ export const NewOrgForm = ({ onLoadingChange={(loading) => setPaymentConfirmationLoading(loading)} onError={(err) => { toast.error(err.message, { duration: 10_000 }) + trackFunnelError( + 'org_creation', + { errorCategory: 'payment', errorReason: 'payment_error' }, + 'toast' + ) setNewOrgLoading(false) resetPaymentMethod() }} diff --git a/apps/studio/components/interfaces/SignIn/SignUpForm.tsx b/apps/studio/components/interfaces/SignIn/SignUpForm.tsx index f1bcacd97aa..e3c6c3f2490 100644 --- a/apps/studio/components/interfaces/SignIn/SignUpForm.tsx +++ b/apps/studio/components/interfaces/SignIn/SignUpForm.tsx @@ -25,6 +25,8 @@ import PasswordConditionsHelper from './PasswordConditionsHelper' import { useSignUpMutation } from '@/data/misc/signup-mutation' import { BASE_PATH } from '@/lib/constants' import { buildPathWithParams } from '@/lib/gotrue' +import { classifyApiError, classifyValidationError } from '@/lib/telemetry/funnel-errors' +import { useTrackFunnelError } from '@/lib/telemetry/use-track-funnel-error' const schema = z.object({ email: z.string().min(1, 'Email is required').email('Must be a valid email'), @@ -67,6 +69,8 @@ export const SignUpForm = () => { token: parseAsString.withDefault(''), }) + const trackFunnelError = useTrackFunnelError() + const { mutate: signup, isPending: isSigningUp } = useSignUpMutation({ onSuccess: () => { toast.success(`Signed up successfully!`) @@ -76,6 +80,7 @@ export const SignUpForm = () => { setCaptchaToken(null) captchaRef.current?.resetCaptcha() toast.error(`Failed to sign up: ${error.message}`) + trackFunnelError('signup', classifyApiError('signup', error), 'toast') }, }) @@ -144,7 +149,13 @@ export const SignUpForm = () => { )} > - + + trackFunnelError('signup', classifyValidationError('signup', errors), 'form') + )} + > { + it('classifies connection timeout as network', () => { + expect(classifyApiError('signup', { errorType: 'connection-timeout' })).toEqual({ + errorCategory: 'network', + errorReason: 'connection_timeout', + }) + }) + + it('classifies a missing status code as network_error', () => { + expect(classifyApiError('project_creation', { message: 'Failed to fetch' })).toEqual({ + errorCategory: 'network', + errorReason: 'network_error', + }) + }) + + it('classifies 429 as rate_limited regardless of message', () => { + expect(classifyApiError('signup', { code: 429, message: 'whatever' })).toEqual({ + errorCategory: 'api', + errorReason: 'rate_limited', + errorCode: 429, + }) + }) + + it('classifies 5xx as server_error', () => { + expect(classifyApiError('org_creation', { code: 500, message: 'boom' })).toEqual({ + errorCategory: 'api', + errorReason: 'server_error', + errorCode: 500, + }) + }) + + it('matches a known 4xx signup message to a reason slug', () => { + expect(classifyApiError('signup', { code: 400, message: 'User already registered' })).toEqual({ + errorCategory: 'api', + errorReason: 'email_already_registered', + errorCode: 400, + }) + }) + + it('matches a known 4xx project message to a reason slug', () => { + expect( + classifyApiError('project_creation', { + code: 403, + message: 'Your organization can only have 2 projects', + }) + ).toEqual({ errorCategory: 'api', errorReason: 'project_limit_reached', errorCode: 403 }) + }) + + it('falls back to other for an unmapped 4xx message', () => { + expect(classifyApiError('signup', { code: 400, message: 'totally novel error' })).toEqual({ + errorCategory: 'api', + errorReason: 'other', + errorCode: 400, + }) + }) +}) + +describe('classifyValidationError', () => { + it('maps a signup password error to password_invalid', () => { + expect( + classifyValidationError('signup', { password: { type: 'too_small' } } as FieldErrors) + ).toEqual({ + errorCategory: 'validation', + errorReason: 'password_invalid', + }) + }) + + it('respects field priority (email before password)', () => { + expect( + classifyValidationError('signup', { + email: { type: 'invalid' }, + password: { type: 'too_small' }, + } as FieldErrors) + ).toEqual({ errorCategory: 'validation', errorReason: 'email_invalid' }) + }) + + it('maps an org name error to org_name_missing', () => { + expect( + classifyValidationError('org_creation', { name: { type: 'too_small' } } as FieldErrors) + ).toEqual({ + errorCategory: 'validation', + errorReason: 'org_name_missing', + }) + }) + + it('falls back to other for an unmapped field', () => { + expect( + classifyValidationError('project_creation', { somethingNew: { type: 'x' } } as FieldErrors) + ).toEqual({ errorCategory: 'validation', errorReason: 'other' }) + }) +}) + +describe('classifyStripeError', () => { + it('maps a decline_code to a card reason slug', () => { + expect( + classifyStripeError({ code: 'card_declined', decline_code: 'insufficient_funds' }) + ).toEqual({ + errorCategory: 'payment', + errorReason: 'card_insufficient_funds', + }) + }) + + it('falls back to payment_failed for an unknown code', () => { + expect(classifyStripeError({ code: 'mystery' })).toEqual({ + errorCategory: 'payment', + errorReason: 'payment_failed', + }) + }) +}) diff --git a/apps/studio/lib/telemetry/funnel-errors.ts b/apps/studio/lib/telemetry/funnel-errors.ts new file mode 100644 index 00000000000..579d0cc6bea --- /dev/null +++ b/apps/studio/lib/telemetry/funnel-errors.ts @@ -0,0 +1,142 @@ +import type { FieldErrors } from 'react-hook-form' + +export type FunnelOrigin = 'signup' | 'project_creation' | 'org_creation' +export type ErrorCategory = 'validation' | 'api' | 'network' | 'payment' | 'unknown' + +export interface FunnelErrorClassification { + errorCategory: ErrorCategory + errorReason: FunnelErrorReason + errorCode?: number +} + +const RATE_LIMIT_STATUS = 429 + +const API_REASON_PATTERNS = { + signup: [ + [/already registered|already been registered|already exists/i, 'email_already_registered'], + [/rate limit|too many requests|after \d+ second/i, 'rate_limited'], + [/captcha/i, 'captcha_failed'], + [/password/i, 'password_rejected'], + [/valid email|invalid email|email address/i, 'email_invalid'], + ], + project_creation: [ + [/already exists/i, 'project_name_taken'], + [/free plan|free tier/i, 'free_tier_limit'], + [/limit|maximum number|can only have/i, 'project_limit_reached'], + [/payment|invoice|overdue|past due|billing/i, 'billing_issue'], + [/region/i, 'region_unavailable'], + [/db_pass|password/i, 'db_password_rejected'], + ], + org_creation: [ + [/already exists|name.*taken/i, 'org_name_taken'], + [/payment|card|invoice|billing/i, 'billing_issue'], + [/limit/i, 'org_limit_reached'], + ], +} as const satisfies Record> + +const VALIDATION_FIELD_REASONS = { + signup: { + email: 'email_invalid', + password: 'password_invalid', + }, + project_creation: { + organization: 'organization_missing', + projectName: 'project_name_invalid', + dbPass: 'db_password_weak', + dbPassStrength: 'db_password_weak', + dbRegion: 'region_missing', + cloudProvider: 'cloud_provider_invalid', + postgresVersion: 'postgres_version_missing', + highAvailability: 'incompatible_options', + useOrioleDb: 'incompatible_options', + }, + org_creation: { + name: 'org_name_missing', + kind: 'org_kind_invalid', + size: 'org_size_invalid', + }, +} as const satisfies Record>> + +const STRIPE_DECLINE_REASONS = { + insufficient_funds: 'card_insufficient_funds', + card_declined: 'card_declined', + expired_card: 'card_expired', + incorrect_cvc: 'card_incorrect_cvc', + incorrect_number: 'card_incorrect_number', + processing_error: 'card_processing_error', +} as const satisfies Record + +const GENERIC_REASONS = [ + 'rate_limited', + 'server_error', + 'connection_timeout', + 'network_error', + 'payment_failed', + 'payment_error', + 'oriole_unavailable', + 'other', +] as const + +type ValuesOf = T extends Readonly> ? V : never + +export type FunnelErrorReason = + | (typeof API_REASON_PATTERNS)[FunnelOrigin][number][1] + | ValuesOf<(typeof VALIDATION_FIELD_REASONS)[FunnelOrigin]> + | ValuesOf + | (typeof GENERIC_REASONS)[number] + +export function classifyApiError(origin: FunnelOrigin, error: unknown): FunnelErrorClassification { + const err = error as { code?: unknown; errorType?: unknown; message?: unknown } + const code = typeof err?.code === 'number' ? err.code : undefined + const message = typeof err?.message === 'string' ? err.message : '' + + if (err?.errorType === 'connection-timeout') { + return { errorCategory: 'network', errorReason: 'connection_timeout' } + } + if (code === undefined) { + return { errorCategory: 'network', errorReason: 'network_error' } + } + if (code === RATE_LIMIT_STATUS) { + return { errorCategory: 'api', errorReason: 'rate_limited', errorCode: code } + } + if (code >= 500) { + return { errorCategory: 'api', errorReason: 'server_error', errorCode: code } + } + for (const [pattern, reason] of API_REASON_PATTERNS[origin]) { + if (pattern.test(message)) { + return { errorCategory: 'api', errorReason: reason, errorCode: code } + } + } + return { errorCategory: 'api', errorReason: 'other', errorCode: code } +} + +export function classifyValidationError( + origin: FunnelOrigin, + errors: FieldErrors +): FunnelErrorClassification { + const fieldErrors = errors as Record + const reasons = VALIDATION_FIELD_REASONS[origin] as Readonly> + for (const field of Object.keys(reasons)) { + if (fieldErrors[field]) { + return { errorCategory: 'validation', errorReason: reasons[field] } + } + } + return { errorCategory: 'validation', errorReason: 'other' } +} + +export function classifyStripeError(error: unknown): FunnelErrorClassification { + const err = error as { code?: unknown; decline_code?: unknown } + const key = + typeof err?.decline_code === 'string' + ? err.decline_code + : typeof err?.code === 'string' + ? err.code + : undefined + const reason = key + ? (STRIPE_DECLINE_REASONS as Readonly>)[key] + : undefined + if (reason) { + return { errorCategory: 'payment', errorReason: reason } + } + return { errorCategory: 'payment', errorReason: 'payment_failed' } +} diff --git a/apps/studio/lib/telemetry/use-track-funnel-error.ts b/apps/studio/lib/telemetry/use-track-funnel-error.ts new file mode 100644 index 00000000000..3aa00091a55 --- /dev/null +++ b/apps/studio/lib/telemetry/use-track-funnel-error.ts @@ -0,0 +1,28 @@ +import { useCallback } from 'react' + +import type { FunnelErrorClassification, FunnelOrigin } from '@/lib/telemetry/funnel-errors' +import { useTrack } from '@/lib/telemetry/track' + +// Matches the existing dashboard_error_created capture rate; keeps PostHog volume bounded. +const SAMPLE_RATE = 0.1 + +export function useTrackFunnelError() { + const track = useTrack() + return useCallback( + ( + origin: FunnelOrigin, + classification: FunnelErrorClassification, + source: 'toast' | 'form' = 'toast' + ) => { + if (Math.random() >= SAMPLE_RATE) return + track('dashboard_error_created', { + source, + origin, + errorCategory: classification.errorCategory, + errorReason: classification.errorReason, + ...(classification.errorCode !== undefined && { errorCode: classification.errorCode }), + }) + }, + [track] + ) +} diff --git a/apps/studio/pages/new/[slug].tsx b/apps/studio/pages/new/[slug].tsx index a01856a6536..a1738298df2 100644 --- a/apps/studio/pages/new/[slug].tsx +++ b/apps/studio/pages/new/[slug].tsx @@ -71,13 +71,16 @@ import { usePHFlag } from '@/hooks/ui/useFlag' import { DOCS_URL, PROJECT_STATUS, PROVIDERS, useDefaultProvider } from '@/lib/constants' import { buildStudioPageTitle } from '@/lib/page-title' import { useProfile } from '@/lib/profile' +import { classifyApiError, classifyValidationError } from '@/lib/telemetry/funnel-errors' import { useTrack } from '@/lib/telemetry/track' +import { useTrackFunnelError } from '@/lib/telemetry/use-track-funnel-error' import type { NextPageWithLayout } from '@/types' const sizesWithNoCostConfirmationRequired: DesiredInstanceSize[] = ['micro', 'small'] const Wizard: NextPageWithLayout = () => { const track = useTrack() + const trackFunnelError = useTrackFunnelError() const router = useRouter() const { slug, projectName } = useParams() const { appTitle } = useCustomContent(['app:title']) @@ -317,6 +320,10 @@ const Wizard: NextPageWithLayout = () => { ) router.push(`/project/${res.ref}`) }, + onError: (error) => { + toast.error(`Failed to create new project: ${error.message}`) + trackFunnelError('project_creation', classifyApiError('project_creation', error), 'toast') + }, }) const onSubmitWithComputeCostsConfirmation = async (values: z.infer) => { @@ -356,6 +363,11 @@ const Wizard: NextPageWithLayout = () => { } = values if (useOrioleDb && !availableOrioleVersion) { + trackFunnelError( + 'project_creation', + { errorCategory: 'validation', errorReason: 'oriole_unavailable' }, + 'toast' + ) return toast.error('No available OrioleDB image found, only Postgres is available') } @@ -501,7 +513,15 @@ const Wizard: NextPageWithLayout = () => { - + + trackFunnelError( + 'project_creation', + classifyValidationError('project_creation', errors), + 'form' + ) + )} + >