mirror of
https://github.com/supabase/supabase.git
synced 2026-10-06 01:45:10 +03:00
chore: improve storage migration script (#33927)
## I have read the [CONTRIBUTING.md](https://github.com/supabase/supabase/blob/master/CONTRIBUTING.md) file. YES ## What kind of change does this PR introduce? docs update ## What is the current behavior? Issues described in this [Slack thread](https://supabase.slack.com/archives/C02BJ2239GA/p1740408115511769) ## What is the new behavior? - Full Storage Migration: Transfers all buckets and their contents. - Bucket Configuration & Metadata Retention: Maintains settings like public access, file size limits, MIME types, and file metadata. - Conflict Resolution Options: Skip, merge (overwrite), or rename buckets in the target project. - Safety Measures: Requires user confirmation before destructive actions. - Efficient Processing: Batches files (10 per batch) and supports nested directories. - Progress & Reporting: Real-time updates, detailed migration summaries, and error tracking for failed files. --------- Co-authored-by: Illia Basalaiev <illiab@IMB3.local>
This commit is contained in:
Illia BasalaievandIllia Basalaiev authored and GitHub committed
1 parent
8c6dbf6433
commit
9b1377c2ba
1 file changed
+314
-36
@@ -236,47 +236,325 @@ const OLD_PROJECT_SERVICE_KEY = 'old-project-service-key-xxx'
|
||||
const NEW_PROJECT_URL = 'https://yyy.supabase.co'
|
||||
const NEW_PROJECT_SERVICE_KEY = 'new-project-service-key-yyy'
|
||||
|
||||
;(async () => {
|
||||
const oldSupabaseRestClient = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY, {
|
||||
db: {
|
||||
schema: 'storage',
|
||||
},
|
||||
})
|
||||
const oldSupabaseClient = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY)
|
||||
const newSupabaseClient = createClient(NEW_PROJECT_URL, NEW_PROJECT_SERVICE_KEY)
|
||||
const oldSupabase = createClient(OLD_PROJECT_URL, OLD_PROJECT_SERVICE_KEY)
|
||||
const newSupabase = createClient(NEW_PROJECT_URL, NEW_PROJECT_SERVICE_KEY)
|
||||
|
||||
// make sure you update max_rows in postgrest settings if you have a lot of objects
|
||||
// or paginate here
|
||||
const { data: oldObjects, error } = await oldSupabaseRestClient.from('objects').select()
|
||||
if (error) {
|
||||
console.log('error getting objects from old bucket')
|
||||
function createLoadingAnimation(message) {
|
||||
const readline = require('readline')
|
||||
const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
|
||||
let i = 0
|
||||
let timer
|
||||
let stopped = false
|
||||
|
||||
const animate = () => {
|
||||
if (stopped) return
|
||||
process.stdout.write(`\r${frames[i]} ${message}`)
|
||||
i = (i + 1) % frames.length
|
||||
timer = setTimeout(animate, 80)
|
||||
}
|
||||
|
||||
animate()
|
||||
|
||||
return {
|
||||
stop: (finalMessage = '') => {
|
||||
stopped = true
|
||||
clearTimeout(timer)
|
||||
readline.clearLine(process.stdout, 0)
|
||||
readline.cursorTo(process.stdout, 0)
|
||||
process.stdout.write(`✓ ${finalMessage || message}\n`)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists all files in a bucket, handling nested folders recursively.
|
||||
*/
|
||||
async function listAllFiles(bucket, path = '') {
|
||||
const loader = createLoadingAnimation(`Listing files in '${bucket}${path ? '/' + path : ''}'...`)
|
||||
|
||||
try {
|
||||
const { data, error } = await oldSupabase.storage.from(bucket).list(path, { limit: 1000 })
|
||||
if (error) {
|
||||
loader.stop(`Error listing files in '${bucket}${path ? '/' + path : ''}'`)
|
||||
throw new Error(`❌ Error listing files in bucket '${bucket}': ${error.message}`)
|
||||
}
|
||||
|
||||
if (!data || data.length === 0) {
|
||||
loader.stop(`No files found in '${bucket}${path ? '/' + path : ''}'`)
|
||||
return []
|
||||
}
|
||||
|
||||
let files = []
|
||||
for (const item of data) {
|
||||
if (!item.metadata) {
|
||||
loader.stop(`Found folder '${item.name}' in '${bucket}${path ? '/' + path : ''}'`)
|
||||
const subFiles = await listAllFiles(bucket, `${path}${item.name}/`)
|
||||
files = files.concat(subFiles)
|
||||
} else {
|
||||
files.push({ fullPath: `${path}${item.name}`, metadata: item.metadata })
|
||||
}
|
||||
}
|
||||
|
||||
loader.stop(`Found ${files.length} files in '${bucket}${path ? '/' + path : ''}'`)
|
||||
return files
|
||||
} catch (error) {
|
||||
loader.stop()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
for (const objectData of oldObjects) {
|
||||
console.log(`moving ${objectData.id}`)
|
||||
try {
|
||||
const { data, error: downloadObjectError } = await oldSupabaseClient.storage
|
||||
.from(objectData.bucket_id)
|
||||
.download(objectData.name)
|
||||
if (downloadObjectError) {
|
||||
throw downloadObjectError
|
||||
}
|
||||
/**
|
||||
* Creates a bucket in the new Supabase project if it doesn't exist.
|
||||
*/
|
||||
async function ensureBucketExists(bucketName, options = {}) {
|
||||
const { data: existingBucket, error: getBucketError } =
|
||||
await newSupabase.storage.getBucket(bucketName)
|
||||
|
||||
const { _, error: uploadObjectError } = await newSupabaseClient.storage
|
||||
.from(objectData.bucket_id)
|
||||
.upload(objectData.name, data, {
|
||||
upsert: true,
|
||||
contentType: objectData.metadata.mimetype,
|
||||
cacheControl: objectData.metadata.cacheControl,
|
||||
})
|
||||
if (uploadObjectError) {
|
||||
throw uploadObjectError
|
||||
}
|
||||
} catch (err) {
|
||||
console.log('error moving ', objectData)
|
||||
console.log(err)
|
||||
if (getBucketError && !getBucketError.message.includes('not found')) {
|
||||
throw new Error(`❌ Error checking if bucket '${bucketName}' exists: ${getBucketError.message}`)
|
||||
}
|
||||
|
||||
if (!existingBucket) {
|
||||
console.log(`🪣 Creating bucket '${bucketName}' in new project...`)
|
||||
const { error } = await newSupabase.storage.createBucket(bucketName, options)
|
||||
if (error) throw new Error(`❌ Failed to create bucket '${bucketName}': ${error.message}`)
|
||||
console.log(`✅ Created bucket '${bucketName}'`)
|
||||
} else {
|
||||
console.log(`ℹ️ Bucket '${bucketName}' already exists in new project`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrates a single file from the old project to the new one.
|
||||
*/
|
||||
async function migrateFile(sourceBucketName, targetBucketName, file) {
|
||||
const loader = createLoadingAnimation(
|
||||
`Migrating ${file.fullPath} in bucket '${sourceBucketName}' to '${targetBucketName}'...`
|
||||
)
|
||||
|
||||
try {
|
||||
const { data, error: downloadError } = await oldSupabase.storage
|
||||
.from(sourceBucketName)
|
||||
.download(file.fullPath)
|
||||
if (downloadError) {
|
||||
loader.stop(`Failed to migrate ${file.fullPath}: Download error`)
|
||||
throw new Error(`Download failed: ${downloadError.message}`)
|
||||
}
|
||||
|
||||
// Preserve all available metadata from the original file
|
||||
const uploadOptions = {
|
||||
upsert: true,
|
||||
contentType: file.metadata?.mimetype,
|
||||
cacheControl: file.metadata?.cacheControl,
|
||||
}
|
||||
|
||||
const { error: uploadError } = await newSupabase.storage
|
||||
.from(targetBucketName)
|
||||
.upload(file.fullPath, data, uploadOptions)
|
||||
if (uploadError) {
|
||||
loader.stop(`Failed to migrate ${file.fullPath}: Upload error`)
|
||||
throw new Error(`Upload failed: ${uploadError.message}`)
|
||||
}
|
||||
|
||||
loader.stop(
|
||||
`Migrated ${file.fullPath} in bucket '${sourceBucketName}' to '${targetBucketName}'`
|
||||
)
|
||||
return { success: true, path: file.fullPath }
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`❌ Error migrating ${file.fullPath} in bucket '${targetBucketName}':`,
|
||||
err.message
|
||||
)
|
||||
return { success: false, path: file.fullPath, error: err.message }
|
||||
}
|
||||
}
|
||||
|
||||
function chunkArray(array, size) {
|
||||
const chunks = []
|
||||
for (let i = 0; i < array.length; i += size) {
|
||||
chunks.push(array.slice(i, i + size))
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrates all buckets and files from the old Supabase project to the new one.
|
||||
* Processes files in parallel within batches for efficiency.
|
||||
*/
|
||||
async function migrateBuckets() {
|
||||
console.log('🔄 Starting Supabase Storage migration...')
|
||||
console.log(`📦 Source project: ${OLD_PROJECT_URL}`)
|
||||
console.log(`📦 Target project: ${NEW_PROJECT_URL}`)
|
||||
|
||||
const readline = require('readline').createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
})
|
||||
|
||||
console.log(
|
||||
'\n⚠️ WARNING: This migration may overwrite files in the target project if they have the same paths.'
|
||||
)
|
||||
console.log('⚠️ It is recommended to back up your target project before proceeding.')
|
||||
|
||||
const answer = await new Promise((resolve) => {
|
||||
readline.question('Do you want to proceed with the migration? (yes/no): ', resolve)
|
||||
})
|
||||
|
||||
readline.close()
|
||||
|
||||
if (answer.toLowerCase() !== 'yes') {
|
||||
console.log('Migration canceled by user.')
|
||||
return { canceled: true }
|
||||
}
|
||||
|
||||
console.log('\n📦 Fetching all buckets from old project...')
|
||||
|
||||
const { data: oldBuckets, error: bucketListError } = await oldSupabase.storage.listBuckets()
|
||||
|
||||
if (bucketListError) throw new Error(`❌ Error fetching buckets: ${bucketListError.message}`)
|
||||
console.log(`✅ Found ${oldBuckets.length} buckets to migrate.`)
|
||||
|
||||
const { data: existingBuckets, error: existingBucketsError } =
|
||||
await newSupabase.storage.listBuckets()
|
||||
if (existingBucketsError)
|
||||
throw new Error(`❌ Error fetching existing buckets: ${existingBucketsError.message}`)
|
||||
|
||||
const existingBucketNames = existingBuckets.map((b) => b.name)
|
||||
const conflictingBuckets = oldBuckets.filter((b) => existingBucketNames.includes(b.name))
|
||||
|
||||
let conflictStrategy = 2
|
||||
|
||||
if (conflictingBuckets.length > 0) {
|
||||
console.log('\n⚠️ The following buckets already exist in the target project:')
|
||||
conflictingBuckets.forEach((b) => console.log(` - ${b.name}`))
|
||||
|
||||
const conflictAnswer = await new Promise((resolve) => {
|
||||
const rl = require('readline').createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
})
|
||||
rl.question(
|
||||
'\nHow do you want to handle existing buckets?\n' +
|
||||
'1. Skip existing buckets\n' +
|
||||
'2. Merge files (may overwrite existing files)\n' +
|
||||
'3. Rename buckets in target (add suffix "_migrated")\n' +
|
||||
'4. Cancel migration\n' +
|
||||
'Enter your choice (1-4): ',
|
||||
(answer) => {
|
||||
rl.close()
|
||||
resolve(answer)
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
if (conflictAnswer === '4') {
|
||||
console.log('Migration canceled by user.')
|
||||
return { canceled: true }
|
||||
}
|
||||
|
||||
conflictStrategy = parseInt(conflictAnswer)
|
||||
if (isNaN(conflictStrategy) || conflictStrategy < 1 || conflictStrategy > 3) {
|
||||
console.log('Invalid choice. Migration canceled.')
|
||||
return { canceled: true }
|
||||
}
|
||||
}
|
||||
})()
|
||||
|
||||
const migrationStats = {
|
||||
totalBuckets: oldBuckets.length,
|
||||
processedBuckets: 0,
|
||||
skippedBuckets: 0,
|
||||
totalFiles: 0,
|
||||
successfulFiles: 0,
|
||||
failedFiles: 0,
|
||||
failedFilesList: [],
|
||||
}
|
||||
|
||||
for (const bucket of oldBuckets) {
|
||||
const bucketName = bucket.name
|
||||
console.log(`\n📁 Processing bucket: ${bucketName}`)
|
||||
|
||||
let targetBucketName = bucketName
|
||||
|
||||
if (existingBucketNames.includes(bucketName)) {
|
||||
if (conflictStrategy === 1) {
|
||||
console.log(`⏩ Skipping bucket '${bucketName}' as it already exists in target project`)
|
||||
migrationStats.skippedBuckets++
|
||||
continue
|
||||
} else if (conflictStrategy === 3) {
|
||||
targetBucketName = `${bucketName}_migrated`
|
||||
console.log(`🔄 Renaming bucket to '${targetBucketName}' in target project`)
|
||||
} else {
|
||||
console.log(`🔄 Merging files into existing bucket '${bucketName}' in target project`)
|
||||
}
|
||||
}
|
||||
|
||||
// Preserve bucket configuration when creating in the new project
|
||||
if (targetBucketName !== bucketName || !existingBucketNames.includes(bucketName)) {
|
||||
await ensureBucketExists(targetBucketName, {
|
||||
public: bucket.public,
|
||||
fileSizeLimit: bucket.file_size_limit,
|
||||
allowedMimeTypes: bucket.allowed_mime_types,
|
||||
})
|
||||
}
|
||||
|
||||
const files = await listAllFiles(bucketName)
|
||||
console.log(`✅ Found ${files.length} files in bucket '${bucketName}'.`)
|
||||
migrationStats.totalFiles += files.length
|
||||
|
||||
const batches = chunkArray(files, 10)
|
||||
|
||||
for (let i = 0; i < batches.length; i++) {
|
||||
console.log(`\n🚀 Processing batch ${i + 1}/${batches.length} (${batches[i].length} files)`)
|
||||
|
||||
const results = await Promise.all(
|
||||
batches[i].map((file) => migrateFile(bucketName, targetBucketName, file))
|
||||
)
|
||||
|
||||
const batchSuccesses = results.filter((r) => r.success).length
|
||||
const batchFailures = results.filter((r) => !r.success)
|
||||
|
||||
migrationStats.successfulFiles += batchSuccesses
|
||||
migrationStats.failedFiles += batchFailures.length
|
||||
migrationStats.failedFilesList.push(...batchFailures.map((f) => f.path))
|
||||
|
||||
console.log(
|
||||
`✅ Completed batch ${i + 1}/${batches.length}: ${batchSuccesses} succeeded, ${batchFailures.length} failed`
|
||||
)
|
||||
}
|
||||
|
||||
migrationStats.processedBuckets++
|
||||
console.log(`✅ Completed bucket '${bucketName}' migration`)
|
||||
}
|
||||
|
||||
console.log('\n📊 Migration Summary:')
|
||||
console.log(
|
||||
`Buckets: ${migrationStats.processedBuckets}/${migrationStats.totalBuckets} processed, ${migrationStats.skippedBuckets} skipped`
|
||||
)
|
||||
console.log(
|
||||
`Files: ${migrationStats.successfulFiles} succeeded, ${migrationStats.failedFiles} failed (${migrationStats.totalFiles} total)`
|
||||
)
|
||||
|
||||
if (migrationStats.failedFiles > 0) {
|
||||
console.log('\n⚠️ Failed files:')
|
||||
migrationStats.failedFilesList.forEach((path) => console.log(` - ${path}`))
|
||||
return migrationStats
|
||||
}
|
||||
|
||||
return migrationStats
|
||||
}
|
||||
|
||||
migrateBuckets()
|
||||
.then((stats) => {
|
||||
if (stats.failedFiles > 0) {
|
||||
console.log(`\n⚠️ Migration completed with ${stats.failedFiles} failed files.`)
|
||||
process.exit(1)
|
||||
} else {
|
||||
console.log('\n🎉 Migration completed successfully!')
|
||||
process.exit(0)
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error('❌ Fatal error during migration:', err.message)
|
||||
process.exit(1)
|
||||
})
|
||||
```
|
||||
Reference in new issue
Block a user