diff --git a/.github/workflows/www-tests.yml b/.github/workflows/www-tests.yml index 321df0844af..c54fb5bdce8 100644 --- a/.github/workflows/www-tests.yml +++ b/.github/workflows/www-tests.yml @@ -4,7 +4,11 @@ on: pull_request: branches: ['master'] paths: + - '.github/workflows/www-tests.yml' - 'apps/www/**/*.ts*' + - 'apps/www/package.json' + - 'apps/www/turbo.jsonc' + - 'scripts/upload-static-assets.sh' - 'packages/common/sentry.ts' - 'packages/common/sentry.test.ts' - 'apps/www/next.config.mjs' @@ -45,6 +49,7 @@ jobs: apps/www apps/docs/content/guides packages + scripts supabase patches diff --git a/apps/www/.gitignore b/apps/www/.gitignore index 1d2c2d81a47..51c0c7bb2f2 100644 --- a/apps/www/.gitignore +++ b/apps/www/.gitignore @@ -25,7 +25,7 @@ yarn-error.log* !**/*/generated/**/.gitkeep .generated -# Sitemap (regenerated on every Vercel deploy by `pnpm postbuild` -> internals/generate-sitemap.mjs) +# Sitemaps (generated by build:next and restored from the Turbo build cache) public/sitemap.xml public/sitemap_www.xml diff --git a/apps/www/README.md b/apps/www/README.md index f7f66b9cc03..3f04f5ebb7c 100644 --- a/apps/www/README.md +++ b/apps/www/README.md @@ -6,6 +6,25 @@ Refer to the [Development Guide](../../DEVELOPERS.md) to learn how to run this s To get started copy the example env file using `cp .env.local.example .env.local`. +## Production builds + +Run `pnpm build` from `apps/www`, or `pnpm build --filter=www` from the repository +root. Vercel uses the app's `build` script by default. + +`prebuild` refreshes remote content before Turbo hashes it. Turbo caches +`build:next` (Next.js, sitemaps, and the customer RSS feed), then `postbuild` +uploads assets even on cache hits. The outer `build` task stays uncached so +neither preparation nor upload can be skipped. Uploads run after caching because +the upload script removes `.next/static`. + +Production cache reuse requires the same commit and inputs: CDN URLs include +`VERCEL_GIT_COMMIT_SHA`. Set `FORCE_ASSET_CDN=-1` to disable local uploads. +Running `build:next` directly skips content preparation and uploads. + +Run `pnpm exec vitest run turbo-build.test.ts` to check cache invalidation and +asset restoration using real pnpm, Turbo, and the upload script with fixture +generators, a stub compiler, and a fake AWS CLI. + ## Best practices ### Images diff --git a/apps/www/package.json b/apps/www/package.json index 0184c510cb0..fcd364ed9e3 100644 --- a/apps/www/package.json +++ b/apps/www/package.json @@ -6,8 +6,9 @@ "scripts": { "preinstall": "npx only-allow pnpm", "dev": "pnpm run content:build && next --port 3000", - "prebuild": "pnpm --filter=docs run build:federated-content && pnpm --filter=docs run build:guides-markdown", - "build": "pnpm run content:build && next build", + "prebuild": "pnpm --filter=docs run build:federated-content && pnpm --filter=docs run build:guides-markdown && pnpm run content:build", + "build": "turbo run build:next --filter=www", + "build:next": "next build && node ./internals/generate-sitemap.mjs", "export": "next export", "start": "next start", "lint": "eslint .", @@ -20,7 +21,7 @@ "test": "vitest --run", "test:watch": "vitest watch", "test:partner-form-sync": "vitest run --config vitest.sync.config.ts", - "postbuild": "node ./internals/generate-sitemap.mjs && ./../../scripts/upload-static-assets.sh" + "postbuild": "./../../scripts/upload-static-assets.sh" }, "dependencies": { "@bsmnt/scrollytelling": "^0.3.3", diff --git a/apps/www/turbo-build.test.ts b/apps/www/turbo-build.test.ts new file mode 100644 index 00000000000..0c210e7520c --- /dev/null +++ b/apps/www/turbo-build.test.ts @@ -0,0 +1,287 @@ +import { spawnSync } from 'node:child_process' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import ts from 'typescript' +import { afterAll, describe, expect, it } from 'vitest' + +const wwwRoot = process.cwd() +const repoRoot = path.resolve(wwwRoot, '../..') +const packageJson = JSON.parse(fs.readFileSync(path.join(wwwRoot, 'package.json'), 'utf8')) +const turboConfig = ts.parseConfigFileTextToJson( + 'turbo.jsonc', + fs.readFileSync(path.join(wwwRoot, 'turbo.jsonc'), 'utf8') +).config +const fixture = fs.mkdtempSync(path.join(os.tmpdir(), 'www-turbo-')) +const app = path.join(fixture, 'apps/www') + +function write(file: string, content: string) { + const target = path.join(fixture, file) + fs.mkdirSync(path.dirname(target), { recursive: true }) + fs.writeFileSync(target, content) +} + +function json(file: string, value: unknown) { + write(file, JSON.stringify(value)) +} + +function executable(file: string, content: string) { + write(file, `#!/usr/bin/env node\n${content}`) + fs.chmodSync(path.join(fixture, file), 0o755) +} + +function events() { + return fs.readFileSync(path.join(fixture, '.checks/events'), 'utf8').trim().split('\n') +} + +const helpers = ` +const fs = require('node:fs'); +const path = require('node:path'); +const root = ${JSON.stringify(fixture)}; +const write = (file, value) => { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, value); +}; +const event = (value) => fs.appendFileSync(path.join(root, '.checks/events'), value + '\\n'); +` + +// Run the real pnpm lifecycle, Turbo binary/config, and uploader. Only remote +// generators, Next compilation, and the aws executable are fixture stand-ins. +json('package.json', { name: 'fixture', private: true, packageManager: 'pnpm@11.13.1' }) +write('pnpm-workspace.yaml', 'packages:\n - apps/*\n - packages/*\n') +write( + '.gitignore', + 'node_modules\n.turbo\n.next\n.generated\n.checks\npublic\ncontent\n*.generated.ts\n.env*\n' +) +write( + 'pnpm-lock.yaml', + "lockfileVersion: '9.0'\nimporters:\n .: {}\n apps/www: {}\n apps/docs: {}\n packages/common: {}\n" +) +json('turbo.json', { + tasks: { build: { dependsOn: ['^build'] } }, + remoteCache: { enabled: false }, +}) +json('apps/www/package.json', { + name: 'www', + private: true, + scripts: packageJson.scripts, + dependencies: { common: 'workspace:*' }, +}) +json('apps/www/turbo.json', turboConfig) +json('apps/docs/package.json', { + name: 'docs', + private: true, + scripts: { + 'build:federated-content': 'node fetch.cjs', + 'build:guides-markdown': 'node markdown.cjs', + }, +}) +json('packages/common/package.json', { name: 'common', private: true }) +write('packages/common/index.js', 'export const version = 1\n') +write('apps/studio/components/ui/TwoOptionToggle.tsx', 'export const version = 1\n') +write('.checks/events', '') +write( + 'apps/docs/fetch.cjs', + helpers + + ` +event('docs:fetch'); +if (process.env.FIXTURE_FAIL_PREPARE) process.exit(1); +write('content/guides/example.mdx', process.env.FIXTURE_DOCS || 'docs-v1'); +` +) +write( + 'apps/docs/markdown.cjs', + helpers + + ` +event('docs:markdown'); +write('public/markdown/guides/example.md', fs.readFileSync('content/guides/example.mdx')); +` +) +const generateContent = + helpers + + ` +event('www:content'); +const content = process.env.FIXTURE_CONTENT || 'content-v1'; +for (const file of ['.generated/staticContent/_index.json', 'public/rss.xml', 'public/changelog-rss.xml', 'public/changelog.md', 'public/changelog/entry.md']) write(file, content); +` +write( + 'apps/www/scripts/generateStaticContent.mjs', + `import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);\n${generateContent}` +) +write( + 'apps/www/scripts/generateMdContent.mjs', + `import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);\n${helpers} +write('app/api-v2/md/content.generated.ts', fs.readFileSync('.generated/staticContent/_index.json')); +` +) +write( + 'apps/www/scripts/fetchAgentSkills.mjs', + `import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);\n${helpers} +write('public/.well-known/agent-skills/index.json', process.env.FIXTURE_SKILLS || 'skills-v1'); +` +) +write( + 'apps/www/internals/generate-sitemap.mjs', + `import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);\n${helpers} +event('sitemap'); +write('public/sitemap.xml', 'sitemap'); +write('public/sitemap_www.xml', 'www-sitemap'); +` +) +executable( + 'node_modules/.bin/next', + helpers + + ` +event('compile'); +if (process.env.NEXT_PUBLIC_FIXTURE_FAIL_BUILD) process.exit(1); +const content = fs.readFileSync('.generated/staticContent/_index.json', 'utf8'); +const docs = fs.readFileSync('../docs/public/markdown/guides/example.md', 'utf8'); +write('.next/server/page.html', content + docs); +write('.next/static/chunk.js', content + docs); +write('.next/cache/temporary', 'not a deploy artifact'); +write('public/customers-rss.xml', content); +` +) +executable( + 'node_modules/.bin/aws', + helpers + + ` +event('upload:' + process.argv[4]); +if (!fs.existsSync('.next/static/chunk.js')) throw new Error('Upload is missing restored static assets'); +for (const file of ['public/rss.xml', 'public/customers-rss.xml', 'public/sitemap.xml', 'public/sitemap_www.xml', 'public/.well-known/agent-skills/index.json', '../docs/public/markdown/guides/example.md']) { + if (!fs.existsSync(file)) throw new Error('Missing deployment artifact: ' + file); +} +` +) +write( + 'scripts/upload-static-assets.sh', + fs.readFileSync(path.join(repoRoot, 'scripts/upload-static-assets.sh'), 'utf8') +) +fs.chmodSync(path.join(fixture, 'scripts/upload-static-assets.sh'), 0o755) +fs.symlinkSync( + path.join(repoRoot, 'node_modules/.bin/turbo'), + path.join(fixture, 'node_modules/.bin/turbo') +) +fs.symlinkSync(path.join(repoRoot, 'node_modules/turbo'), path.join(fixture, 'node_modules/turbo')) + +function build(env: Partial = {}, rootCommand = false) { + const command = rootCommand ? path.join(fixture, 'node_modules/.bin/turbo') : 'pnpm' + const result = spawnSync( + command, + rootCommand ? ['run', 'build', '--filter=www'] : ['run', 'build'], + { + cwd: rootCommand ? fixture : app, + encoding: 'utf8', + timeout: 60_000, + env: { + ...process.env, + CI: 'true', + FORCE_ASSET_CDN: '1', + VERCEL_ENV: 'preview', + VERCEL_GIT_COMMIT_SHA: '1111111111111111111111111111111111111111', + SITE_NAME: 'www', + ASSET_CDN_S3_ENDPOINT: 'https://example.invalid', + PATH: `${path.join(fixture, 'node_modules/.bin')}${path.delimiter}${process.env.PATH}`, + ...env, + }, + } + ) + return { ...result, output: result.stdout + result.stderr } +} + +afterAll(() => fs.rmSync(fixture, { recursive: true, force: true })) + +describe('www deployment build lifecycle', () => { + it('builds fresh artifacts before uploading them', () => { + const result = build() + expect(result.status, result.output).toBe(0) + expect(events()).toEqual([ + 'docs:fetch', + 'docs:markdown', + 'www:content', + 'compile', + 'sitemap', + 'upload:.next/static', + 'upload:public', + ]) + expect(fs.existsSync(path.join(app, '.next/static/chunk.js'))).toBe(false) + }, 90_000) + + it('restores compiled assets and sitemaps and still uploads on a cache hit', () => { + fs.rmSync(path.join(app, '.next'), { recursive: true, force: true }) + fs.rmSync(path.join(app, 'public'), { recursive: true, force: true }) + const result = build() + expect(result.status, result.output).toBe(0) + expect(result.output).toContain('cache hit') + expect(events().filter((event) => event === 'compile')).toHaveLength(1) + expect(events().filter((event) => event === 'docs:fetch')).toHaveLength(2) + expect(events().filter((event) => event.startsWith('upload:'))).toHaveLength(4) + expect(fs.existsSync(path.join(app, '.next/cache/temporary'))).toBe(false) + }, 90_000) + + it.each([ + { FIXTURE_CONTENT: 'content-v2' }, + { FIXTURE_DOCS: 'docs-v2' }, + { FIXTURE_SKILLS: 'skills-v2' }, + { NEXT_PUBLIC_NEW_SETTING: 'new-value' }, + { VERCEL_GIT_COMMIT_SHA: '2222222222222222222222222222222222222222' }, + ])( + 'invalidates compilation when an input changes: %j', + (env) => { + const count = events().filter((event) => event === 'compile').length + const result = build(env) + expect(result.status, result.output).toBe(0) + expect(events().filter((event) => event === 'compile')).toHaveLength(count + 1) + }, + 90_000 + ) + + it('invalidates compilation when a shared package changes', () => { + write('packages/common/index.js', 'export const version = 2\n') + const count = events().filter((event) => event === 'compile').length + const result = build() + expect(result.status, result.output).toBe(0) + expect(events().filter((event) => event === 'compile')).toHaveLength(count + 1) + }, 90_000) + + it('invalidates compilation when a local environment file changes', () => { + write('apps/www/.env.local', 'NEXT_PUBLIC_LOCAL_SETTING=changed\n') + const count = events().filter((event) => event === 'compile').length + const result = build() + expect(result.status, result.output).toBe(0) + expect(events().filter((event) => event === 'compile')).toHaveLength(count + 1) + }, 90_000) + + it('invalidates compilation when the imported Studio component changes', () => { + write('apps/studio/components/ui/TwoOptionToggle.tsx', 'export const version = 2\n') + const count = events().filter((event) => event === 'compile').length + const result = build() + expect(result.status, result.output).toBe(0) + expect(events().filter((event) => event === 'compile')).toHaveLength(count + 1) + }, 90_000) + + it('also refreshes content and uploads through the root Turbo build', () => { + const count = events().filter((event) => event === 'compile').length + const result = build({}, true) + expect(result.status, result.output).toBe(0) + expect(events().filter((event) => event === 'compile')).toHaveLength(count) + expect(events().slice(-5)).toEqual([ + 'docs:fetch', + 'docs:markdown', + 'www:content', + 'upload:.next/static', + 'upload:public', + ]) + }, 90_000) + + it.each([{ FIXTURE_FAIL_PREPARE: '1' }, { NEXT_PUBLIC_FIXTURE_FAIL_BUILD: '1' }])( + 'does not upload after an earlier build stage fails: %j', + (env) => { + const uploads = events().filter((event) => event.startsWith('upload:')).length + const result = build(env) + expect(result.status, result.output).not.toBe(0) + expect(events().filter((event) => event.startsWith('upload:'))).toHaveLength(uploads) + }, + 90_000 + ) +}) diff --git a/apps/www/turbo.jsonc b/apps/www/turbo.jsonc index f0a1a1b59dc..16fb164e4b4 100644 --- a/apps/www/turbo.jsonc +++ b/apps/www/turbo.jsonc @@ -9,63 +9,47 @@ "env": ["GITHUB_TOKEN"], }, "build": { + // pnpm's prebuild refreshes remote content before the nested build:next run hashes it. + // postbuild uploads assets on every deployment, including build:next cache hits. + "cache": false, + "dependsOn": [], + "outputs": [], + "passThroughEnv": ["*"], + }, + "build:next": { "dependsOn": ["^build"], + "inputs": [ + "$TURBO_DEFAULT$", + "!public/customers-rss.xml", + ".env*", + ".generated/**", + "app/api-v2/md/content.generated.ts", + "public/rss.xml", + "public/changelog-rss.xml", + "public/changelog-rss/**", + "public/changelog.md", + "public/changelog/**", + "public/.well-known/agent-skills/index.json", + "$TURBO_ROOT$/apps/docs/content/guides/**", + "$TURBO_ROOT$/apps/docs/public/markdown/guides/**", + "$TURBO_ROOT$/apps/studio/components/ui/TwoOptionToggle.tsx", + ], "env": [ + "NEXT_PUBLIC_*", "ANALYZE", "CI", "CF_ACCESS_CLIENT_ID", "CF_ACCESS_CLIENT_SECRET", - "GITHUB_TOKEN", - "NEXT_PUBLIC_MISC_USE_URL", - "NEXT_PUBLIC_MISC_USE_ANON_KEY", - "NEXT_PUBLIC_STUDIO_URL", - "NEXT_PUBLIC_DOCS_URL", - "NEXT_PUBLIC_REFERENCE_DOCS_URL", - "NEXT_PUBLIC_LIBRARY_URL", - "NEXT_PUBLIC_KB_URL", - // Temporary fallback during the production environment-variable migration. - "NEXT_PUBLIC_UI_LIBRARY_URL", - "NEXT_PUBLIC_SUPABASE_URL", - "NEXT_PUBLIC_SUPABASE_ANON_KEY", - "NEXT_PUBLIC_VERCEL_ENV", - "NEXT_PUBLIC_VERCEL_BRANCH_URL", - "NEXT_PUBLIC_API_URL", - "NEXT_PUBLIC_HCAPTCHA_SITE_KEY", "HCAPTCHA_SECRET_KEY", "NODE_ENV", - "NEXT_PUBLIC_SENTRY_DSN", "SENTRY_AUTH_TOKEN", - "NEXT_PUBLIC_SURVEY_SUPABASE_URL", - "NEXT_PUBLIC_SURVEY_SUPABASE_ANON_KEY", "VERCEL", "VERCEL_ENV", "VERCEL_GIT_COMMIT_SHA", - "NEXT_PUBLIC_INTEGRATIONS_MARKETPLACE_DB", - "NEXT_PUBLIC_MARKETPLACE_API_URL", - "NEXT_PUBLIC_MARKETPLACE_PUBLISHABLE_KEY", - // These envs are used in the packages - "NEXT_PUBLIC_STORAGE_KEY", - "NEXT_PUBLIC_AUTH_DEBUG_KEY", - "NEXT_PUBLIC_AUTH_PERSISTED_KEY", - "NEXT_PUBLIC_AUTH_NAVIGATOR_LOCK_KEY", - "NEXT_PUBLIC_GOOGLE_TAG_MANAGER_ID", - "NEXT_PUBLIC_IS_PLATFORM", - "NEXT_PUBLIC_AUTH_DETECT_SESSION_IN_URL", - "NEXT_PUBLIC_GOTRUE_URL", - "NEXT_PUBLIC_BASE_PATH", - "NEXT_PUBLIC_EXAMPLES_SUPABASE_ANON_KEY", - "NEXT_PUBLIC_EXAMPLES_SUPABASE_URL", - "NEXT_PUBLIC_USERCENTRICS_RULESET_ID", - // These envs are technically passthrough env vars because they're only used on the server side of Nextjs "LIVE_SUPABASE_COM_SERVICE_ROLE_KEY", - "GITHUB_CHANGELOG_APP_ID", - "GITHUB_CHANGELOG_APP_INSTALLATION_ID", - "GITHUB_CHANGELOG_APP_REST_KEY", - "GITHUB_CHANGELOG_APP_PRIVATE_KEY", "CHANGELOG_SYNC_APP_ID", "CHANGELOG_SYNC_APP_INSTALLATION_ID", "CHANGELOG_SYNC_APP_PRIVATE_KEY", - "NEXT_PUBLIC_EMAIL_ABUSE_URL", "EMAIL_ABUSE_SERVICE_KEY", "HUBSPOT_PORTAL_ID", "HUBSPOT_ENTERPRISE_FORM_GUID", @@ -74,29 +58,31 @@ "NOTION_FORMS_API_KEY", "OPENAI_API_KEY", "EMAIL_REPORT_SLACK_WEBHOOK", - "npm_lifecycle_event", - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", "FORCE_ASSET_CDN", - "ASSET_CDN_S3_ENDPOINT", "SITE_NAME", - "AGENT_SKILLS_GITHUB_TOKEN", "LUMA_API_KEY", "LUMA_HACKATHONS_API_KEY", "NEXT_RUNTIME", - // These env vars are used by some community and marketing integrations "SENTRY_DSN_COMMUNITY", "NOTION_SUPASQUAD_API_KEY", "NOTION_SUPASQUAD_APPLICATIONS_DB_ID", - "NEXT_PUBLIC_SUPASQUAD_APPLICATION_FORM_URL", "CUSTOMERIO_SITE_ID", "CUSTOMERIO_API_KEY", "CUSTOMERIO_APP_API_KEY", "HUBSPOT_MEETINGS_TOKEN", "NOTION_EVENTS_API_KEY", "NOTION_EVENTS_DB_ID_ACTUAL", + "ENABLED_FEATURES_OVERRIDE_DISABLE_ALL", + ], + "outputs": [ + ".next/**", + "!.next/cache/**", + "!.next/dev/**", + ".contentlayer/**", + "public/sitemap.xml", + "public/sitemap_www.xml", + "public/customers-rss.xml", ], - "outputs": [".next/**", "!.next/cache/**", "!.next/dev/**/*", ".contentlayer/**"], }, }, } diff --git a/apps/www/vercel.json b/apps/www/vercel.json deleted file mode 100644 index 8df7e53755b..00000000000 --- a/apps/www/vercel.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "buildCommand": "pnpm build" -}