From 6cf174b92534e4f3eb2288da8720c26e8314fa06 Mon Sep 17 00:00:00 2001 From: "Andrey A." <56412611+aantti@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:28:20 +0200 Subject: [PATCH] feat(self-hosted): change the default db version to postgres 17 (#46981) --- docker/docker-compose.pg15.yml | 18 +++++ docker/docker-compose.pg17.yml | 6 +- docker/docker-compose.yml | 3 +- docker/tests/test-pg17-upgrade.sh | 61 ++++++++++++++- docker/tests/test-self-hosted.sh | 31 +++++++- docker/utils/upgrade-pg17.sh | 122 +++++++++++++++++++++++------- 6 files changed, 204 insertions(+), 37 deletions(-) create mode 100644 docker/docker-compose.pg15.yml diff --git a/docker/docker-compose.pg15.yml b/docker/docker-compose.pg15.yml new file mode 100644 index 00000000000..f87fe8c4828 --- /dev/null +++ b/docker/docker-compose.pg15.yml @@ -0,0 +1,18 @@ +# Postgres 15 override for self-hosted Supabase. +# +# Postgres 17 is now the default in docker-compose.yml. Use this override to pin +# Postgres 15, for two cases: +# 1. An existing Postgres 15 deployment that has not been upgraded yet. +# 2. As the rollback target for utils/upgrade-pg17.sh (PG 15 and PG 17 use +# different postgres UIDs, so rollback must start with the PG 15 image). +# +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d +# +# To upgrade an existing Postgres 15 database to Postgres 17 in place, see: +# - https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17 +# - utils/upgrade-pg17.sh + +services: + db: + image: supabase/postgres:15.8.1.085 diff --git a/docker/docker-compose.pg17.yml b/docker/docker-compose.pg17.yml index d35f0b91a18..a8e37117e1b 100644 --- a/docker/docker-compose.pg17.yml +++ b/docker/docker-compose.pg17.yml @@ -1,5 +1,9 @@ # Postgres 17 override for self-hosted Supabase. # +# NOTE: Postgres 17 is now the default in docker-compose.yml, so this override +# is redundant for new installs (kept for explicitness / backwards compatibility). +# Keep the image tag here in sync with the db image in docker-compose.yml. +# # Usage (new install or after upgrade): # docker compose -f docker-compose.yml -f docker-compose.pg17.yml up -d # @@ -12,4 +16,4 @@ services: db: - image: supabase/postgres:17.6.1.084 + image: supabase/postgres:17.6.1.136 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 33e961257e1..82eb4102bb4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -474,7 +474,8 @@ services: # Comment out everything below this point if you are using an external Postgres database db: container_name: supabase-db - image: supabase/postgres:15.8.1.085 + # To upgrade an existing Postgres 15 database in place, see utils/upgrade-pg17.sh. + image: supabase/postgres:17.6.1.136 restart: unless-stopped volumes: - ./volumes/db/realtime.sql:/docker-entrypoint-initdb.d/migrations/99-realtime.sql:Z diff --git a/docker/tests/test-pg17-upgrade.sh b/docker/tests/test-pg17-upgrade.sh index 0015b868ff5..6ac7ff9f091 100644 --- a/docker/tests/test-pg17-upgrade.sh +++ b/docker/tests/test-pg17-upgrade.sh @@ -10,8 +10,9 @@ # sudo bash tests/test-pg17-upgrade.sh # # Prerequisites: -# - Running self-hosted Supabase with a clean, tests-only Postgres 15: -# docker compose up -d +# - Running self-hosted Supabase with a clean, tests-only Postgres 15. +# Postgres 17 is now the default, so start PG 15 explicitly via the override: +# docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d # - .env file with POSTGRES_PASSWORD, ANON_KEY # @@ -48,7 +49,7 @@ echo "" current_version=$(run_sql -A -t -c "SHOW server_version;" | head -1) case "$current_version" in 15.*) echo "Starting version: PostgreSQL $current_version" ;; - 17.*) echo "Error: Already on Postgres 17. Start with a PG15 stack."; exit 1 ;; + 17.*) echo "Error: Already on Postgres 17. Start with a PG 15 stack."; exit 1 ;; *) echo "Error: Unexpected version: $current_version"; exit 1 ;; esac @@ -88,6 +89,21 @@ $$; -- Grant access so PostgREST can read it GRANT SELECT ON public._upgrade_test TO anon, authenticated; + +-- pg_cron: created here as supabase_admin (the common manually-enabled case), +-- so the extension is owned by supabase_admin, NOT postgres. complete.sh's +-- drop+recreate only fires for postgres-owned pg_cron, so this exercises the +-- version reconcile in the upgrade script: Supabase PG 15 registers pg_cron +-- as '1.6', while the target image packages it as '1.6.4' with no update +-- path between them. +CREATE EXTENSION IF NOT EXISTS pg_cron; +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM cron.job WHERE jobname = 'upgrade_test_job') THEN + PERFORM cron.unschedule('upgrade_test_job'); + END IF; +END $$; +SELECT cron.schedule('upgrade_test_job', '5 4 * * *', 'SELECT 1'); EOSQL pre_count=$(run_sql -A -t -c "SELECT count(*) FROM public._upgrade_test;" | tr -d '[:space:]') @@ -115,7 +131,7 @@ echo "" run_sql < {"data": ...} +# disabled => {"errors":[{"message":"pg_graphql extension is not enabled."}]} +# Assert the status AND the response shape, so a non-200, non-JSON, or empty body +# (a real gateway/runtime failure) is not silently classified as "disabled". +gql_status=$(http_status "$BASE_URL/graphql/v1" \ -H "apikey: $ANON_KEY" \ -H "Content-Type: application/json" \ -d '{"query":"{ __typename }"}') -gql_has_data=$(echo "$gql_resp" | jq -r 'if .data then "true" else "false" end' 2>/dev/null) -check "GraphQL introspection" "true" "$gql_has_data" +gql_body=$(http_body "$BASE_URL/graphql/v1" \ + -H "apikey: $ANON_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query":"{ __typename }"}') +if [ "$gql_status" = "200" ] && echo "$gql_body" | jq -e '.data' >/dev/null 2>&1; then + gql_state="enabled" +elif [ "$gql_status" = "200" ] && echo "$gql_body" | jq -e '.errors' >/dev/null 2>&1; then + gql_state="disabled" +else + gql_state="unhealthy (HTTP $gql_status)" +fi +case "$gql_state" in + enabled | disabled) gql_health="healthy" ;; + *) gql_health="unhealthy" ;; +esac +check "GraphQL endpoint healthy" "healthy" "$gql_health" +echo " (GraphQL is $gql_state)" # --------------------------------------------- # 6. Storage: create bucket, upload >6MB file, download, cleanup diff --git a/docker/utils/upgrade-pg17.sh b/docker/utils/upgrade-pg17.sh index 6bdc97bd3a5..c05a7b25718 100644 --- a/docker/utils/upgrade-pg17.sh +++ b/docker/utils/upgrade-pg17.sh @@ -6,7 +6,7 @@ # Upgrade self-hosted Supabase Postgres from 15 to 17. # # Uses Supabase's pg_upgrade scripts (initiate.sh + complete.sh) inside a -# temporary PG15 container, then swaps data directories and starts Postgres 17. +# temporary PG 15 container, then swaps data directories and starts Postgres 17. # # Usage (must be run as root or with sudo): # cd docker/ @@ -27,8 +27,8 @@ # 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down # 2. rm -rf ./volumes/db/data # 3. mv ./volumes/db/data.bak.pg15 ./volumes/db/data -# 4. docker compose run --rm db chown -R postgres:postgres /etc/postgresql-custom/ -# 5. docker compose up -d +# 4. docker compose -f docker-compose.yml -f docker-compose.pg15.yml run --rm db chown -R postgres:postgres /etc/postgresql-custom/ +# 5. docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d # # Ensure we're running under bash (not sh/zsh/dash). @@ -50,10 +50,18 @@ done # --- Configuration ---------------------------------------------------------- # Image used for the upgrade tarball + complete.sh container. -# Must share glibc with PG15 (the extracted ELF binaries run inside PG15). +# Must share glibc with PG 15 (the extracted ELF binaries run inside PG15). +# Pinned to .063: later images bumped glibc, which breaks the ELF extraction. PG17_UPGRADE_IMAGE="supabase/postgres:17.6.1.063" # Tag in supabase/postgres repo matching the upgrade image (for downloading scripts) PG17_SCRIPTS_REF="17.6.1.063" + +# Final Postgres 17 image that runs after the upgrade. Pinned here (not read from +# the compose files). Keep in sync with the db image in docker-compose.yml (and +# docker-compose.pg17.yml). Used for pulling, chowning data/db-config to the +# target's postgres UID, and running the post-upgrade migrations. +PG17_TARGET_IMAGE="supabase/postgres:17.6.1.136" + DB_CONTAINER="supabase-db" UPGRADE_CONTAINER="supabase-pg-upgrade" COMPLETE_CONTAINER="supabase-pg-complete" @@ -96,7 +104,7 @@ trap cleanup EXIT on_interrupt() { echo "" warn "Interrupted. Cleaning up..." - # If db-config was chowned to PG17, restore for PG15 rollback + # If db-config was chowned to PG17, restore for PG 15 rollback if [ -n "$db_config_vol" ] && [ -n "$current_image" ]; then docker run --rm -v "${db_config_vol}:/vol" "$current_image" \ chown -R postgres:postgres /vol/ 2>/dev/null || true @@ -158,10 +166,6 @@ preflight() { | grep -E '^db-config$|_db-config$' | head -n 1) [ -n "$db_config_vol" ] || die "Could not find db-config volume. Is Supabase running?" - # Read the target PG17 image from the compose override (what the user will run) - PG17_TARGET_IMAGE=$(grep 'image:.*postgres' docker-compose.pg17.yml | awk '{print $2}' | head -n 1) - [ -n "$PG17_TARGET_IMAGE" ] || die "Could not read image from docker-compose.pg17.yml." - pg_password=$(grep '^POSTGRES_PASSWORD=' .env | cut -d '=' -f 2- | sed "s/^['\"]//;s/['\"]$//" | head -n 1) [ -n "$pg_password" ] || die "POSTGRES_PASSWORD not set in .env." @@ -187,8 +191,8 @@ preflight() { warn " 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down" warn " 2. rm -rf $DATA_DIR" warn " 3. mv $BACKUP_DIR $DATA_DIR" - warn " 4. docker compose run --rm db chown -R postgres:postgres /etc/postgresql-custom/" - warn " 5. docker compose up -d" + warn " 4. docker compose -f docker-compose.yml -f docker-compose.pg15.yml run --rm db chown -R postgres:postgres /etc/postgresql-custom/" + warn " 5. docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d" echo "" warn "Continuing will DELETE the existing backup permanently." confirm "Delete $BACKUP_DIR and start a fresh upgrade?" @@ -265,8 +269,8 @@ pull_image() { # --- Step 2: Build upgrade tarball ----------------------------------------- # -# Extracts PG17 binaries, libraries, share data, and upgrade scripts from -# the PG17 Docker image into a tarball that initiate.sh can consume. +# Extracts PG 17 binaries, libraries, share data, and upgrade scripts from +# the PG 17 Docker image into a tarball that initiate.sh can consume. # # The tarball uses the "non-nix" layout (17/bin, 17/lib, 17/share - no # nix_flake_version file), so initiate.sh sets LD_LIBRARY_PATH to find @@ -281,7 +285,7 @@ build_tarball() { echo " Staging directory: $staging_dir" # Download upgrade scripts from the supabase/postgres repo (pinned to PG17_SCRIPTS_REF). - # These are no longer bundled in the latest PG17 Docker images. + # These are no longer bundled in the latest PG 17 Docker images. info "Downloading upgrade scripts (ref: $PG17_SCRIPTS_REF)" local scripts_base="https://raw.githubusercontent.com/supabase/postgres/${PG17_SCRIPTS_REF}/ansible/files/admin_api_scripts/pg_upgrade_scripts" mkdir -p "$staging_dir/scripts" @@ -470,7 +474,7 @@ run_upgrade() { ' wait_for_healthy "$UPGRADE_CONTAINER" - # initiate.sh expects the PG17 binaries tarball at /tmp/persistent/pg_upgrade_bin.tar.gz + # initiate.sh expects the PG 17 binaries tarball at /tmp/persistent/pg_upgrade_bin.tar.gz # (hardcoded path - copied there during container setup above). # # Env vars for the unwrapped nix ELF binaries in the tarball: @@ -502,7 +506,7 @@ run_upgrade() { docker rm -f "$UPGRADE_CONTAINER" >/dev/null 2>&1 || true } -# --- Step 6: Run complete.sh in a native PG17 container ------------------- +# --- Step 6: Run complete.sh in a native PG 17 container ------------------- # # complete.sh applies post-upgrade patches (pg_net grants, vault re-encryption, # pg_cron, predefined roles, vacuumdb, etc.). We run it in a PG17 container @@ -525,8 +529,8 @@ run_complete() { info "Preparing complete.sh environment" # Save original db-config ownership so we can restore it if complete.sh fails. - # complete.sh needs PG17 ownership to start postgres, but if it fails the - # user needs to fall back to PG15 which uses a different uid. + # complete.sh needs PG 17 ownership to start postgres, but if it fails the + # user needs to fall back to PG 15 which uses a different uid. docker exec "$COMPLETE_CONTAINER" bash -c ' stat -c "%u:%g" /etc/postgresql-custom/pgsodium_root.key 2>/dev/null > /tmp/dbconfig_owner || true ' @@ -570,7 +574,7 @@ run_complete() { warn "complete.sh failed. Postgres log:" docker exec "$COMPLETE_CONTAINER" cat /tmp/postgres.log 2>/dev/null || true echo "" - # Restore db-config ownership so PG15 can start for rollback + # Restore db-config ownership so PG 15 can start for rollback warn "Restoring db-config ownership for PG15..." local orig_owner orig_owner=$(docker exec "$COMPLETE_CONTAINER" cat /tmp/dbconfig_owner 2>/dev/null || true) @@ -582,7 +586,7 @@ run_complete() { echo " Your Postgres 15 data is unchanged (data swap has not happened yet)." echo " To restart Postgres 15:" echo " rm -rf $MIGRATION_DIR" - echo " docker compose up -d" + echo " docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d" echo "" die "complete.sh failed (status: $status)" fi @@ -609,7 +613,7 @@ swap_data() { start_pg17() { info "Starting Supabase with Postgres 17" - # Ensure db-config volume has correct ownership and structure for PG17. + # Ensure db-config volume has correct ownership and structure for PG 17. # complete.sh does this too, but just in case of partial # failures from previous runs. docker run --rm -v "${db_config_vol}:/vol" "$PG17_TARGET_IMAGE" sh -c ' @@ -641,15 +645,28 @@ start_pg17() { # --- Step 9: Apply migrations not covered by complete.sh ------------------- # -# These PG17 migrations run on fresh installs via initdb but not after +# These PG 17 migrations run on fresh installs via initdb but not after # pg_upgrade (init scripts don't rerun when PG_VERSION already exists). -# complete.sh doesn't cover them either. +# complete.sh doesn't cover them either. On the platform a tracked migration +# runner (dbmate) applies them; self-hosted has no such tracking, so we apply +# the idempotent, safe-on-an-existing-DB ones directly here. # # Source: postgres/migrations/db/migrations/ # - 20250710151649_supabase_read_only_user_default_transaction_read_only.sql # - 20251001204436_predefined_role_grants.sql (supabase_etl_admin + pg_monitor) # - 20251105172723_grant_pg_reload_conf_to_postgres.sql # - 20251121132723_correct_search_path_pgbouncer.sql +# - 20260211120934_supabase_privileged_role.sql (target image's supautils.conf +# references this role; it must exist or supautils config is broken) +# - 20260413000000_fix-authenticator-session-preload-libraries.sql +# - 20260421000001_rescope_pg_graphql_access_trigger.sql +# +# NOT applied: +# - 20260421000000_pg_graphql-off-by-default.sql: drops pg_graphql. Safe on a +# fresh install, but destructive on an existing DB that uses pg_graphql. +# +# This step also reconciles extension versions (complete.sh ran with the .063 +# binaries) and the pg_cron 1.6 -> 1.6.4 version label. See below. apply_role_migrations() { info "Applying Postgres 17 migrations" @@ -663,7 +680,7 @@ apply_role_migrations() { -c "ALTER DATABASE \"$db\" REFRESH COLLATION VERSION;" || true done - # Create supabase_etl_admin role (doesn't exist in PG15 images). + # Create supabase_etl_admin role (doesn't exist in PG 15 images). # Must be created before running predefined_role_grants.sql which # assumes it exists. run_sql_on "$DB_CONTAINER" -c " @@ -677,7 +694,7 @@ apply_role_migrations() { END \$\$;" || true - # Run the migration files directly from the PG17 container image. + # Run the migration files directly from the PG 17 container image. # They're idempotent (IF EXISTS / IF NOT EXISTS guards). local migration_dir="/docker-entrypoint-initdb.d/migrations" local migrations=" @@ -685,6 +702,9 @@ apply_role_migrations() { 20251001204436_predefined_role_grants.sql 20251105172723_grant_pg_reload_conf_to_postgres.sql 20251121132723_correct_search_path_pgbouncer.sql + 20260211120934_supabase_privileged_role.sql + 20260413000000_fix-authenticator-session-preload-libraries.sql + 20260421000001_rescope_pg_graphql_access_trigger.sql " for m in $migrations; do @@ -695,6 +715,54 @@ apply_role_migrations() { psql -h localhost -U supabase_admin -d postgres -v ON_ERROR_STOP=1 \ -f "${migration_dir}/${m}" || warn " $m failed (non-fatal)" done + + # Reconcile extension versions to the target image. pg_upgrade generates + # update_extensions.sql, but complete.sh runs it inside the .063 container, so + # extensions only get updated to .063's versions. The final image ships newer + # .so files, leaving the catalog version behind what's loaded. + # + # Mirror what update_extensions.sql does, but against the running target image + # and for every installed extension: ALTER EXTENSION ... UPDATE brings each one + # up to the image's default version. This is generic on purpose - it tracks the + # image (matching platform behavior) and stays correct across future image + # bumps without maintaining a hardcoded list. ALTER ... UPDATE is a no-op when + # already at the default. Per-extension exceptions are caught so one extension + # with no update path (e.g. pg_cron 1.6 -> 1.6.4, handled below) does not abort + # the rest. + info "Reconciling extension versions to the target image" + run_sql_on "$DB_CONTAINER" -c " + DO \$\$ + DECLARE r record; + BEGIN + FOR r IN SELECT extname FROM pg_extension LOOP + BEGIN + EXECUTE format('ALTER EXTENSION %I UPDATE', r.extname); + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'skipped extension %: %', r.extname, SQLERRM; + END; + END LOOP; + END + \$\$;" || warn "extension version reconcile had errors (non-fatal)" + + # pg_cron: PG 15 registers the extension as '1.6'; PG 17 images package it as + # '1.6.4' with no '1.6' -> '1.6.4' update path, so ALTER EXTENSION ... UPDATE + # has no path to follow. complete.sh only fixes this when pg_cron is owned by + # 'postgres' (it drops + recreates). Manually-enabled instances are often + # owned by supabase_admin, where that path is skipped. Reconcile the catalog + # label directly: the loaded .so and the SQL objects are already + # 1.6.4-equivalent (no SQL delta between 1.6 and 1.6.4). No DROP, so any + # scheduled jobs are untouched. + run_sql_on "$DB_CONTAINER" -c " + DO \$\$ + DECLARE want text; + BEGIN + SELECT default_version INTO want FROM pg_available_extensions WHERE name = 'pg_cron'; + IF want IS NOT NULL + AND EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_cron' AND extversion <> want) THEN + UPDATE pg_extension SET extversion = want WHERE extname = 'pg_cron'; + END IF; + END + \$\$;" || warn "pg_cron version reconcile had errors (non-fatal)" } # --- Step 10: Verify ------------------------------------------------------ @@ -726,8 +794,8 @@ verify() { echo " 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down" echo " 2. rm -rf $DATA_DIR" echo " 3. mv $BACKUP_DIR $DATA_DIR" - echo " 4. docker compose run --rm db chown -R postgres:postgres /etc/postgresql-custom/" - echo " 5. docker compose up -d" + echo " 4. docker compose -f docker-compose.yml -f docker-compose.pg15.yml run --rm db chown -R postgres:postgres /etc/postgresql-custom/" + echo " 5. docker compose -f docker-compose.yml -f docker-compose.pg15.yml up -d" echo "" }