diff --git a/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdx b/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdx index a012aaa51aa..c4c2275fcb9 100644 --- a/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdx +++ b/apps/docs/content/guides/platform/migrating-within-supabase/backup-restore.mdx @@ -236,47 +236,325 @@ const OLD_PROJECT_SERVICE_KEY = 'old-project-service-key-xxx' const NEW_PROJECT_URL = 'https://yyy.supabase.co' const NEW_PROJECT_SERVICE_KEY = 'new-project-service-key-yyy' -;(async () => { - const oldSupabaseRestClient = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY, { - db: { - schema: 'storage', - }, - }) - const oldSupabaseClient = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY) - const newSupabaseClient = createClient(NEW_PROJECT_URL, NEW_PROJECT_SERVICE_KEY) +const oldSupabase = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY) +const newSupabase = createClient(NEW_PROJECT_URL, NEW_PROJECT_SERVICE_KEY) - // make sure you update max_rows in postgrest settings if you have a lot of objects - // or paginate here - const { data: oldObjects, error } = await oldSupabaseRestClient.from('objects').select() - if (error) { - console.log('error getting objects from old bucket') +function createLoadingAnimation(message) { + const readline = require('readline') + const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] + let i = 0 + let timer + let stopped = false + + const animate = () => { + if (stopped) return + process.stdout.write(`\r${frames[i]} ${message}`) + i = (i + 1) % frames.length + timer = setTimeout(animate, 80) + } + + animate() + + return { + stop: (finalMessage = '') => { + stopped = true + clearTimeout(timer) + readline.clearLine(process.stdout, 0) + readline.cursorTo(process.stdout, 0) + process.stdout.write(`✓ ${finalMessage || message}\n`) + }, + } +} + +/** + * Lists all files in a bucket, handling nested folders recursively. + */ +async function listAllFiles(bucket, path = '') { + const loader = createLoadingAnimation(`Listing files in '${bucket}${path ? '/' + path : ''}'...`) + + try { + const { data, error } = await oldSupabase.storage.from(bucket).list(path, { limit: 1000 }) + if (error) { + loader.stop(`Error listing files in '${bucket}${path ? '/' + path : ''}'`) + throw new Error(`❌ Error listing files in bucket '${bucket}': ${error.message}`) + } + + if (!data || data.length === 0) { + loader.stop(`No files found in '${bucket}${path ? '/' + path : ''}'`) + return [] + } + + let files = [] + for (const item of data) { + if (!item.metadata) { + loader.stop(`Found folder '${item.name}' in '${bucket}${path ? '/' + path : ''}'`) + const subFiles = await listAllFiles(bucket, `${path}${item.name}/`) + files = files.concat(subFiles) + } else { + files.push({ fullPath: `${path}${item.name}`, metadata: item.metadata }) + } + } + + loader.stop(`Found ${files.length} files in '${bucket}${path ? '/' + path : ''}'`) + return files + } catch (error) { + loader.stop() throw error } +} - for (const objectData of oldObjects) { - console.log(`moving ${objectData.id}`) - try { - const { data, error: downloadObjectError } = await oldSupabaseClient.storage - .from(objectData.bucket_id) - .download(objectData.name) - if (downloadObjectError) { - throw downloadObjectError - } +/** + * Creates a bucket in the new Supabase project if it doesn't exist. + */ +async function ensureBucketExists(bucketName, options = {}) { + const { data: existingBucket, error: getBucketError } = + await newSupabase.storage.getBucket(bucketName) - const { _, error: uploadObjectError } = await newSupabaseClient.storage - .from(objectData.bucket_id) - .upload(objectData.name, data, { - upsert: true, - contentType: objectData.metadata.mimetype, - cacheControl: objectData.metadata.cacheControl, - }) - if (uploadObjectError) { - throw uploadObjectError - } - } catch (err) { - console.log('error moving ', objectData) - console.log(err) + if (getBucketError && !getBucketError.message.includes('not found')) { + throw new Error(`❌ Error checking if bucket '${bucketName}' exists: ${getBucketError.message}`) + } + + if (!existingBucket) { + console.log(`🪣 Creating bucket '${bucketName}' in new project...`) + const { error } = await newSupabase.storage.createBucket(bucketName, options) + if (error) throw new Error(`❌ Failed to create bucket '${bucketName}': ${error.message}`) + console.log(`✅ Created bucket '${bucketName}'`) + } else { + console.log(`ℹ️ Bucket '${bucketName}' already exists in new project`) + } +} + +/** + * Migrates a single file from the old project to the new one. + */ +async function migrateFile(sourceBucketName, targetBucketName, file) { + const loader = createLoadingAnimation( + `Migrating ${file.fullPath} in bucket '${sourceBucketName}' to '${targetBucketName}'...` + ) + + try { + const { data, error: downloadError } = await oldSupabase.storage + .from(sourceBucketName) + .download(file.fullPath) + if (downloadError) { + loader.stop(`Failed to migrate ${file.fullPath}: Download error`) + throw new Error(`Download failed: ${downloadError.message}`) + } + + // Preserve all available metadata from the original file + const uploadOptions = { + upsert: true, + contentType: file.metadata?.mimetype, + cacheControl: file.metadata?.cacheControl, + } + + const { error: uploadError } = await newSupabase.storage + .from(targetBucketName) + .upload(file.fullPath, data, uploadOptions) + if (uploadError) { + loader.stop(`Failed to migrate ${file.fullPath}: Upload error`) + throw new Error(`Upload failed: ${uploadError.message}`) + } + + loader.stop( + `Migrated ${file.fullPath} in bucket '${sourceBucketName}' to '${targetBucketName}'` + ) + return { success: true, path: file.fullPath } + } catch (err) { + console.error( + `❌ Error migrating ${file.fullPath} in bucket '${targetBucketName}':`, + err.message + ) + return { success: false, path: file.fullPath, error: err.message } + } +} + +function chunkArray(array, size) { + const chunks = [] + for (let i = 0; i < array.length; i += size) { + chunks.push(array.slice(i, i + size)) + } + return chunks +} + +/** + * Migrates all buckets and files from the old Supabase project to the new one. + * Processes files in parallel within batches for efficiency. + */ +async function migrateBuckets() { + console.log('🔄 Starting Supabase Storage migration...') + console.log(`📦 Source project: ${OLD_PROJECT_URL}`) + console.log(`📦 Target project: ${NEW_PROJECT_URL}`) + + const readline = require('readline').createInterface({ + input: process.stdin, + output: process.stdout, + }) + + console.log( + '\n⚠️ WARNING: This migration may overwrite files in the target project if they have the same paths.' + ) + console.log('⚠️ It is recommended to back up your target project before proceeding.') + + const answer = await new Promise((resolve) => { + readline.question('Do you want to proceed with the migration? (yes/no): ', resolve) + }) + + readline.close() + + if (answer.toLowerCase() !== 'yes') { + console.log('Migration canceled by user.') + return { canceled: true } + } + + console.log('\n📦 Fetching all buckets from old project...') + + const { data: oldBuckets, error: bucketListError } = await oldSupabase.storage.listBuckets() + + if (bucketListError) throw new Error(`❌ Error fetching buckets: ${bucketListError.message}`) + console.log(`✅ Found ${oldBuckets.length} buckets to migrate.`) + + const { data: existingBuckets, error: existingBucketsError } = + await newSupabase.storage.listBuckets() + if (existingBucketsError) + throw new Error(`❌ Error fetching existing buckets: ${existingBucketsError.message}`) + + const existingBucketNames = existingBuckets.map((b) => b.name) + const conflictingBuckets = oldBuckets.filter((b) => existingBucketNames.includes(b.name)) + + let conflictStrategy = 2 + + if (conflictingBuckets.length > 0) { + console.log('\n⚠️ The following buckets already exist in the target project:') + conflictingBuckets.forEach((b) => console.log(` - ${b.name}`)) + + const conflictAnswer = await new Promise((resolve) => { + const rl = require('readline').createInterface({ + input: process.stdin, + output: process.stdout, + }) + rl.question( + '\nHow do you want to handle existing buckets?\n' + + '1. Skip existing buckets\n' + + '2. Merge files (may overwrite existing files)\n' + + '3. Rename buckets in target (add suffix "_migrated")\n' + + '4. Cancel migration\n' + + 'Enter your choice (1-4): ', + (answer) => { + rl.close() + resolve(answer) + } + ) + }) + + if (conflictAnswer === '4') { + console.log('Migration canceled by user.') + return { canceled: true } + } + + conflictStrategy = parseInt(conflictAnswer) + if (isNaN(conflictStrategy) || conflictStrategy < 1 || conflictStrategy > 3) { + console.log('Invalid choice. Migration canceled.') + return { canceled: true } } } -})() + + const migrationStats = { + totalBuckets: oldBuckets.length, + processedBuckets: 0, + skippedBuckets: 0, + totalFiles: 0, + successfulFiles: 0, + failedFiles: 0, + failedFilesList: [], + } + + for (const bucket of oldBuckets) { + const bucketName = bucket.name + console.log(`\n📁 Processing bucket: ${bucketName}`) + + let targetBucketName = bucketName + + if (existingBucketNames.includes(bucketName)) { + if (conflictStrategy === 1) { + console.log(`⏩ Skipping bucket '${bucketName}' as it already exists in target project`) + migrationStats.skippedBuckets++ + continue + } else if (conflictStrategy === 3) { + targetBucketName = `${bucketName}_migrated` + console.log(`🔄 Renaming bucket to '${targetBucketName}' in target project`) + } else { + console.log(`🔄 Merging files into existing bucket '${bucketName}' in target project`) + } + } + + // Preserve bucket configuration when creating in the new project + if (targetBucketName !== bucketName || !existingBucketNames.includes(bucketName)) { + await ensureBucketExists(targetBucketName, { + public: bucket.public, + fileSizeLimit: bucket.file_size_limit, + allowedMimeTypes: bucket.allowed_mime_types, + }) + } + + const files = await listAllFiles(bucketName) + console.log(`✅ Found ${files.length} files in bucket '${bucketName}'.`) + migrationStats.totalFiles += files.length + + const batches = chunkArray(files, 10) + + for (let i = 0; i < batches.length; i++) { + console.log(`\n🚀 Processing batch ${i + 1}/${batches.length} (${batches[i].length} files)`) + + const results = await Promise.all( + batches[i].map((file) => migrateFile(bucketName, targetBucketName, file)) + ) + + const batchSuccesses = results.filter((r) => r.success).length + const batchFailures = results.filter((r) => !r.success) + + migrationStats.successfulFiles += batchSuccesses + migrationStats.failedFiles += batchFailures.length + migrationStats.failedFilesList.push(...batchFailures.map((f) => f.path)) + + console.log( + `✅ Completed batch ${i + 1}/${batches.length}: ${batchSuccesses} succeeded, ${batchFailures.length} failed` + ) + } + + migrationStats.processedBuckets++ + console.log(`✅ Completed bucket '${bucketName}' migration`) + } + + console.log('\n📊 Migration Summary:') + console.log( + `Buckets: ${migrationStats.processedBuckets}/${migrationStats.totalBuckets} processed, ${migrationStats.skippedBuckets} skipped` + ) + console.log( + `Files: ${migrationStats.successfulFiles} succeeded, ${migrationStats.failedFiles} failed (${migrationStats.totalFiles} total)` + ) + + if (migrationStats.failedFiles > 0) { + console.log('\n⚠️ Failed files:') + migrationStats.failedFilesList.forEach((path) => console.log(` - ${path}`)) + return migrationStats + } + + return migrationStats +} + +migrateBuckets() + .then((stats) => { + if (stats.failedFiles > 0) { + console.log(`\n⚠️ Migration completed with ${stats.failedFiles} failed files.`) + process.exit(1) + } else { + console.log('\n🎉 Migration completed successfully!') + process.exit(0) + } + }) + .catch((err) => { + console.error('❌ Fatal error during migration:', err.message) + process.exit(1) + }) ```