diff --git a/Dockerfile b/Dockerfile index 1d5780d..52472ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,21 @@ -ARG PG_MAJOR_PREVIOUS=14 -ARG PG_MAJOR=15 +ARG PG_MAJOR_PREVIOUS=15 +ARG PG_MAJOR=17 +ARG TIMESCALE_VERSION=2.22 -FROM timescaledev/timescaledb-ha:pg15-multi as trimmed -MAINTAINER support@openremote.io +FROM timescale/timescaledb-ha:pg17-ts${TIMESCALE_VERSION} AS trimmed +LABEL maintainer="support@openremote.io" USER root +# install fd to find files to speed up chown and chgrp +RUN apt-get update && apt-get install -y fd-find && rm -rf /var/lib/apt/lists/* + # Give postgres user the same UID and GID as the old alpine postgres image to simplify migration of existing DB RUN usermod -u 70 postgres \ && groupmod -g 70 postgres \ - && (find / -group 1000 -exec chgrp -h postgres {} \; || true) \ - && (find / -user 1000 -exec chown -h postgres {} \; || true) + && (fd / -group 1000 -exec chgrp -h postgres {} \; || true) \ + && (fd / -user 1000 -exec chown -h postgres {} \; || true) # Set PGDATA to the same location as our old alpine image RUN mkdir -p /var/lib/postgresql && mv /home/postgres/pgdata/* /var/lib/postgresql/ && chown -R postgres:postgres /var/lib/postgresql @@ -28,8 +32,9 @@ RUN chmod +x /docker-entrypoint-initdb.d/* # Below is mostly copied from https://github.com/timescale/timescaledb-docker-ha/blob/master/Dockerfile (with OR specific entrypoint, # workdir and OR env defaults) -# Get multi all image -FROM timescaledev/timescaledb-ha:pg15-multi-all as trimmed-all +# Get the -all variant which contains multiple PostgreSQL versions +# According to TimescaleDB docs: "timescale/timescaledb-ha images have the files necessary to run previous versions" +FROM timescale/timescaledb-ha:pg17-ts${TIMESCALE_VERSION}-all AS trimmed-all ## Create a smaller Docker image from the builder image FROM scratch diff --git a/Dockerfile.alpine b/Dockerfile.alpine deleted file mode 100644 index 404ecd7..0000000 --- a/Dockerfile.alpine +++ /dev/null @@ -1,44 +0,0 @@ -# ----------------------------------------------------------------------------------------------- -# POSTGIS and TimescaleDB (inc. toolkit for hyperfunctions) image built for aarch64 support -# using alpine base image. -# -# timescale/timescaledb-ha image is ubuntu based and only currently supports amd64; they are -# working on ARM64 support in timescaledev/timescaledb-ha see: -# -# https://github.com/timescale/timescaledb-docker-ha/pull/355 -# -# See this issue for POSTGIS base image aarch64 support discussion: -# -# https://github.com/postgis/docker-postgis/issues/216 -# ------- ---------------------------------------------------------------------------------------- - -# We get POSTGIS and timescale+toolkit from this image -FROM timescaledev/timescaledb-ha:pg15-multi as timescale-ha - - -# This base image is alpine based - timescale toolkit requires glibc 2.3+ so we install it into alpine image -# This still doesn't work as timescale code is compiled against glibc and some references don't match with gcompat - -FROM timescale/timescaledb:latest-pg15 -MAINTAINER support@openremote.io - -ENV GLIBC_VERSION 2.35-r0 -ENV TZ ${TZ:-Europe/Amsterdam} -ENV PGTZ ${PGTZ:-Europe/Amsterdam} -ENV POSTGRES_DB ${POSTGRES_DB:-openremote} -ENV POSTGRES_USER ${POSTGRES_USER:-postgres} -ENV POSTGRES_PASSWORD ${POSTGRES_PASSWORD:-postgres} - -# Add glibc -RUN apk add gcompat - - -COPY --from=timescale-ha /usr/lib/postgresql/15/lib/bitcode/postgis-3/ /usr/local/lib/postgresql/bitcode/ -COPY --from=timescale-ha /usr/lib/postgresql/15/lib/postgis* /usr/local/lib/postgresql/ -COPY --from=timescale-ha /docker-entrypoint-initdb.d/010_install_timescaledb_toolkit.sh /docker-entrypoint-initdb.d/010_install_timescaledb_toolkit.sh -COPY --from=timescale-ha /usr/lib/postgresql/15/lib/timescaledb* /usr/local/lib/postgresql/ -COPY --from=timescale-ha /usr/bin/timescale* /usr/local/bin/ -COPY --from=timescale-ha /usr/share/postgresql/15/extension/postgis* /usr/local/share/postgresql/extension/ -COPY --from=timescale-ha /usr/share/postgresql/15/extension/timescale* /usr/local/share/postgresql/extension/ - -HEALTHCHECK --interval=3s --timeout=3s --start-period=2s --retries=30 CMD pg_isready diff --git a/or-entrypoint.sh b/or-entrypoint.sh index 5cb8eeb..6ff92a3 100644 --- a/or-entrypoint.sh +++ b/or-entrypoint.sh @@ -61,6 +61,8 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then ######################################################################################## # Do upgrade checks - Adapted from https://github.com/pgautoupgrade/docker-pgautoupgrade + # IMPORTANT: TimescaleDB must be upgraded BEFORE PostgreSQL upgrade + # See: https://docs.tigerdata.com/self-hosted/latest/upgrades/major-upgrade/ ######################################################################################## # Get the version of the PostgreSQL data files @@ -75,8 +77,79 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then echo "---------------------------------------------------------------------------------" fi - # Try and upgrade if needed + # STEP 1: Upgrade TimescaleDB on OLD PostgreSQL version (if needed) + # This must happen BEFORE pg_upgrade so both old and new PG have the same TS version if [ "$DB_VERSION" != "$PG_MAJOR" ] && [ "$OR_DISABLE_AUTO_UPGRADE" != "true" ]; then + echo "=================================================================================" + echo "STEP 1: Upgrading TimescaleDB on PostgreSQL ${DB_VERSION} before PG upgrade..." + echo "=================================================================================" + + # Start temporary server on OLD PostgreSQL version + echo "Starting temporary PostgreSQL ${DB_VERSION} server..." + + # Temporarily update PATH to use old PostgreSQL version + OLD_PATH=$PATH + export PATH="/usr/lib/postgresql/${DB_VERSION}/bin:$PATH" + + docker_temp_server_start "$@" + + # Don't automatically abort on non-0 exit status, just in case timescaledb extension isn't installed + set +e + + # Get the latest TimescaleDB version available + TS_VERSION_REGEX="\-\-([0-9|\.]+)\." + TS_SCRIPT_NAME=$(find /usr/share/postgresql/$PG_MAJOR/extension/ -type f -name "timescaledb--*.sql" | sort | tail -n 1) + if [ "$TS_SCRIPT_NAME" != "" ] && [[ $TS_SCRIPT_NAME =~ $TS_VERSION_REGEX ]]; then + TARGET_TS_VERSION=${BASH_REMATCH[1]} + echo "Target TimescaleDB version available: ${TARGET_TS_VERSION}" + + # Upgrade TimescaleDB in ALL databases that have it installed + # This is critical because template1, postgres, and user databases may all have TimescaleDB + # We must include template databases because template1 often has TimescaleDB installed + echo "Finding all databases with TimescaleDB extension..." + DATABASES=$(docker_process_sql -X -t -c "SELECT datname FROM pg_database WHERE datallowconn;" | grep -v "^$") + + for DB in $DATABASES; do + echo "Checking database: $DB" + HAS_TS=$(docker_process_sql -X -d "$DB" -c "SELECT 1 FROM pg_extension WHERE extname='timescaledb';" | grep -v "^$" | wc -l) + + if [ "$HAS_TS" -gt 0 ]; then + CURRENT_TS_VERSION=$(docker_process_sql -X -d "$DB" -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb';" | grep -v extversion | grep -v row | tr -d ' ') + echo " Database $DB has TimescaleDB ${CURRENT_TS_VERSION}, upgrading..." + docker_process_sql -X -d "$DB" -c "ALTER EXTENSION timescaledb UPDATE;" + NEW_TS_VERSION=$(docker_process_sql -X -d "$DB" -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb';" | grep -v extversion | grep -v row | tr -d ' ') + echo " Upgraded: ${CURRENT_TS_VERSION} -> ${NEW_TS_VERSION}" + + # Also upgrade toolkit if present + HAS_TOOLKIT=$(docker_process_sql -X -d "$DB" -c "SELECT 1 FROM pg_extension WHERE extname='timescaledb_toolkit';" | grep -v "^$" | wc -l) + if [ "$HAS_TOOLKIT" -gt 0 ]; then + echo " Upgrading timescaledb_toolkit in $DB..." + docker_process_sql -X -d "$DB" -c "ALTER EXTENSION timescaledb_toolkit UPDATE;" + fi + fi + done + + echo "TimescaleDB upgrade complete in all databases" + fi + + # Return error handling back to automatically aborting on non-0 exit status + set -e + + docker_temp_server_stop + + # Restore PATH + export PATH=$OLD_PATH + + echo "=================================================================================" + echo "STEP 1 Complete: TimescaleDB upgraded on PostgreSQL ${DB_VERSION}" + echo "=================================================================================" + fi + + # STEP 2: Upgrade PostgreSQL if needed + if [ "$DB_VERSION" != "$PG_MAJOR" ] && [ "$OR_DISABLE_AUTO_UPGRADE" != "true" ]; then + echo "=================================================================================" + echo "STEP 2: Upgrading PostgreSQL from ${DB_VERSION} to ${PG_MAJOR}..." + echo "=================================================================================" echo "---------------------------------------------------------------------------------" echo "Postgres major version is newer than the existing DB, performing auto upgrade..." @@ -193,7 +266,25 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then echo "---------------------------------------" echo "Running pg_upgrade command, from $(pwd)" echo "---------------------------------------" - pg_upgrade --link -b /usr/lib/postgresql/${DB_VERSION}/bin -B /usr/lib/postgresql/${PG_MAJOR}/bin -d $OLD -D $NEW + # Specify socket directories for both old and new clusters + # Use --check first to see what issues exist + echo "Running pg_upgrade --check first..." + pg_upgrade --check \ + -b /usr/lib/postgresql/${DB_VERSION}/bin \ + -B /usr/lib/postgresql/${PG_MAJOR}/bin \ + -d $OLD \ + -D $NEW \ + -o "-c unix_socket_directories='${PGSOCKET}'" \ + -O "-c unix_socket_directories='${PGSOCKET}'" || true + + echo "Running actual pg_upgrade with --link..." + pg_upgrade --link \ + -b /usr/lib/postgresql/${DB_VERSION}/bin \ + -B /usr/lib/postgresql/${PG_MAJOR}/bin \ + -d $OLD \ + -D $NEW \ + -o "-c unix_socket_directories='${PGSOCKET}'" \ + -O "-c unix_socket_directories='${PGSOCKET}'" echo "--------------------------------------" echo "Running pg_upgrade command is complete" echo "--------------------------------------" @@ -238,14 +329,15 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then echo "Removing left over database files is complete" echo "---------------------------------------------" - echo "**********************************************************" - echo "Automatic upgrade process finished with no errors reported" - echo "**********************************************************" + echo "=================================================================================" + echo "STEP 2 Complete: PostgreSQL upgraded from ${DB_VERSION} to ${PG_MAJOR}" + echo "=================================================================================" # Return the error handling back to automatically aborting on non-0 exit status set -e fi + # STEP 3: Upgrade TimescaleDB on NEW PostgreSQL version (if needed) # Do timescale upgrade if needed - First look for latest extension version number in extension files echo "----------------------------------------------------------" echo "Checking latest available TimescaleDB extension version..." @@ -295,14 +387,23 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then fi fi - if [ "$DO_TS_UPGRADE" == "true" ] && [ "$OR_DISABLE_AUTO_UPGRADE" == "true" ]; then - echo "----------------------------------------------------------------------------------" - echo "TimescaleDB upgrade can be performed but OR_DISABLE_AUTO_UPGRADE=true so skipping!" - echo "----------------------------------------------------------------------------------" - fi + # Only do this upgrade if we're NOT in the middle of a PostgreSQL upgrade + # (TimescaleDB was already upgraded in STEP 1 before pg_upgrade) + if [ "$DB_VERSION" == "$PG_MAJOR" ]; then + if [ "$DO_TS_UPGRADE" == "true" ] && [ "$OR_DISABLE_AUTO_UPGRADE" == "true" ]; then + echo "----------------------------------------------------------------------------------" + echo "TimescaleDB upgrade can be performed but OR_DISABLE_AUTO_UPGRADE=true so skipping!" + echo "----------------------------------------------------------------------------------" + fi - if [ "${OR_DISABLE_AUTO_UPGRADE}" == "true" ]; then + if [ "${OR_DISABLE_AUTO_UPGRADE}" == "true" ]; then + DO_TS_UPGRADE=false + fi + else + # PostgreSQL version mismatch means we're in upgrade mode + # TimescaleDB upgrade already happened in STEP 1 DO_TS_UPGRADE=false + echo "Skipping TimescaleDB upgrade check - already upgraded in STEP 1 before pg_upgrade" fi @@ -335,26 +436,34 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then echo "-------------------------" docker_temp_server_start "$@" + # STEP 3: Upgrade TimescaleDB on new PostgreSQL version (if needed) # Cannot do this on a running DB as the extension is configured to preload if [ "$DO_TS_UPGRADE" == "true" ]; then - echo "------------------------" - echo "Performing TS upgrade..." - echo "------------------------" + echo "=================================================================================" + echo "STEP 3: Upgrading TimescaleDB on PostgreSQL ${PG_MAJOR}..." + echo "=================================================================================" + + # Get current version + CURRENT_TS_VERSION=$(docker_process_sql -X -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb';" | grep -v extversion | grep -v row | tr -d ' ') + echo "Current TimescaleDB version: ${CURRENT_TS_VERSION}" + echo "Target TimescaleDB version: ${TS_VERSION}" # Don't automatically abort on non-0 exit status, just in case timescaledb extension isn't installed on the DB set +e docker_process_sql -X -c "ALTER EXTENSION timescaledb UPDATE;" if [ $? -eq 0 ]; then + NEW_TS_VERSION=$(docker_process_sql -X -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb';" | grep -v extversion | grep -v row | tr -d ' ') + echo "TimescaleDB upgraded: ${CURRENT_TS_VERSION} -> ${NEW_TS_VERSION}" docker_process_sql -c "CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit; ALTER EXTENSION timescaledb_toolkit UPDATE;" fi # Return the error handling back to automatically aborting on non-0 exit status set -e - echo "-------------------" - echo "TS upgrade complete" - echo "-------------------" + echo "=================================================================================" + echo "STEP 3 Complete: TimescaleDB upgraded on PostgreSQL ${PG_MAJOR}" + echo "=================================================================================" echo "$TS_VERSION" > "${PGDATA}/OR_TS_VERSION" fi @@ -377,4 +486,4 @@ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then fi fi -exec /docker-entrypoint.sh $@ +exec /usr/local/bin/docker-entrypoint.sh $@