diff --git a/test/1.synthetic.sh b/test/1.synthetic.sh index 8ac2e98bad6df3c61c54c08abd2ea561a3d22adc..8454392a302af6ce44416aded16229188c652e22 100644 --- a/test/1.synthetic.sh +++ b/test/1.synthetic.sh @@ -3,7 +3,9 @@ set -euxo pipefail TAG=${TAG:-${CI_COMMIT_REF_SLUG}} IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}" -POSTGRES_VERSION="${POSTGRES_VERSION:-13}" + +# Environment variables for replacement rules +export POSTGRES_VERSION="${POSTGRES_VERSION:-13}" DIR=${0%/*} @@ -58,17 +60,18 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${CI_COMMIT_BRANCH:-mast --output "${configDir}/server.yml" # Edit the following options -sed -ri 's/^(\s*)(debug:.*$)/\1debug: true/' "${configDir}/server.yml" -sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml" -sed -ri 's/^(\s*)(- logicalDump$)/\1/' "${configDir}/server.yml" -sed -ri 's/^(\s*)(- logicalRestore$)/\1/' "${configDir}/server.yml" -# replace postgres version -sed -ri "s/:13/:${POSTGRES_VERSION}/g" "${configDir}/server.yml" - +yq eval -i ' + .global.debug = true | + .global.telemetry.enabled = false | + .localUI.enabled = false | + del(.retrieval.jobs[] | select(. == "logicalDump")) | + del(.retrieval.jobs[] | select(. == "logicalRestore")) | + .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) +' "${configDir}/server.yml" # logerrors is not supported in PostgreSQL 9.6 if [ "${POSTGRES_VERSION}" = "9.6" ]; then - sed -ri 's/^(\s*)(shared_preload_libraries:.*$)/\1shared_preload_libraries: "pg_stat_statements, auto_explain"/' "${configDir}/server.yml" + yq eval -i '.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain"' "${configDir}/server.yml" fi ## Launch Database Lab server diff --git a/test/2.logical_generic.sh b/test/2.logical_generic.sh index f2c541abc12321ceea826118e58dc5ffe2b03ea2..c134cee0d5581d8ac3c4830173dd680358dd9a5a 100644 --- a/test/2.logical_generic.sh +++ b/test/2.logical_generic.sh @@ -3,12 +3,14 @@ set -euxo pipefail TAG="${TAG:-"master"}" IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}" -SOURCE_DBNAME="${SOURCE_DBNAME:-test}" -SOURCE_HOST="${SOURCE_HOST:-172.17.0.1}" -SOURCE_PORT="${SOURCE_PORT:-7432}" -SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}" -SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}" -POSTGRES_VERSION="${POSTGRES_VERSION:-13}" + +# Environment variables for replacement rules +export SOURCE_DBNAME="${SOURCE_DBNAME:-test}" +export SOURCE_HOST="${SOURCE_HOST:-172.17.0.1}" +export SOURCE_PORT="${SOURCE_PORT:-7432}" +export SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}" +export SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}" +export POSTGRES_VERSION="${POSTGRES_VERSION:-13}" DIR=${0%/*} @@ -55,19 +57,21 @@ metaDir="$HOME/.dblab/engine/meta" # Copy the contents of configuration example mkdir -p "${configDir}" -curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.example.logical_generic.yml \ +curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${CI_COMMIT_BRANCH:-master}"/configs/config.example.logical_generic.yml \ --output "${configDir}/server.yml" # Edit the following options -sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml" -sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml" -sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${SOURCE_DBNAME}/" "${configDir}/server.yml" -sed -ri "s/^(\s*)(host: 34.56.78.90$)/\1host: ${SOURCE_HOST}/" "${configDir}/server.yml" -sed -ri "s/^(\s*)(port: 5432$)/\1port: ${SOURCE_PORT}/" "${configDir}/server.yml" -sed -ri "s/^(\s*)(username: postgres$)/\1username: ${SOURCE_USERNAME}/" "${configDir}/server.yml" -sed -ri "s/^(\s*)(password:.*$)/\1password: ${SOURCE_PASSWORD}/" "${configDir}/server.yml" -# replace postgres version -sed -ri "s/:13/:${POSTGRES_VERSION}/g" "${configDir}/server.yml" +yq eval -i ' + .global.debug = true | + .global.telemetry.enabled = false | + .localUI.enabled = false | + .retrieval.spec.logicalDump.options.source.connection.dbname = strenv(SOURCE_DBNAME) | + .retrieval.spec.logicalDump.options.source.connection.host = strenv(SOURCE_HOST) | + .retrieval.spec.logicalDump.options.source.connection.port = env(SOURCE_PORT) | + .retrieval.spec.logicalDump.options.source.connection.username = strenv(SOURCE_USERNAME) | + .retrieval.spec.logicalDump.options.source.connection.password = strenv(SOURCE_PASSWORD) | + .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) +' "${configDir}/server.yml" ## Launch Database Lab server sudo docker run \ diff --git a/test/3.physical_walg.sh b/test/3.physical_walg.sh index 29c70ac551aabab2cad5e90f58a38cd4e6c87625..b3f7f1d71189e402aebc327b80307acc311ccc30 100644 --- a/test/3.physical_walg.sh +++ b/test/3.physical_walg.sh @@ -3,17 +3,18 @@ set -euxo pipefail TAG="${TAG:-"master"}" IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}" -POSTGRES_VERSION="${POSTGRES_VERSION:-13}" -WALG_BACKUP_NAME="${WALG_BACKUP_NAME:-"LATEST"}" -# AWS +# Environment variables for replacement rules +export POSTGRES_VERSION="${POSTGRES_VERSION:-13}" +export WALG_BACKUP_NAME="${WALG_BACKUP_NAME:-"LATEST"}" +## AWS set +euxo pipefail # ---- do not display secrets -AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-""}" -AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-""}" -WALG_S3_PREFIX="${WALG_S3_PREFIX:-""}" -# GS -WALG_GS_PREFIX="${WALG_GS_PREFIX:-""}" -GOOGLE_APPLICATION_CREDENTIALS="${GOOGLE_APPLICATION_CREDENTIALS:-""}" +export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-""}" +export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-""}" +export WALG_S3_PREFIX="${WALG_S3_PREFIX:-""}" +## GS +export WALG_GS_PREFIX="${WALG_GS_PREFIX:-""}" +export GOOGLE_APPLICATION_CREDENTIALS="${GOOGLE_APPLICATION_CREDENTIALS:-""}" # check variables [ -z "${WALG_S3_PREFIX}" ] && [ -z "${WALG_GS_PREFIX}" ] && echo "Variables not specified" && exit 1 set -euxo pipefail # ---- @@ -33,29 +34,37 @@ metaDir="$HOME/.dblab/engine/meta" # Copy the contents of configuration example mkdir -p "${configDir}" -curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.example.physical_walg.yml \ +curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${CI_COMMIT_BRANCH:-master}"/configs/config.example.physical_walg.yml \ --output "${configDir}/server.yml" # Edit the following options -sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml" -sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml" -# set WAL-G envs -sed -ri "s/^(\s*)(backupName:.*$)/\1backupName: ${WALG_BACKUP_NAME}/" "${configDir}/server.yml" +yq eval -i ' + .global.debug = true | + .global.telemetry.enabled = false | + .localUI.enabled = false | + .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) | + .retrieval.spec.physicalRestore.options.walg.backupName = strenv(WALG_BACKUP_NAME) | + .retrieval.spec.physicalRestore.options.sync.configs.shared_buffers = "512MB" | + .retrieval.spec.physicalSnapshot.options.skipStartSnapshot = true +' "${configDir}/server.yml" + set +euxo pipefail # ---- do not display secrets if [ -n "${WALG_S3_PREFIX}" ] ; then -sed -ri "s/^(\s*)(WALG_GS_PREFIX:.*$)/\1AWS_ACCESS_KEY_ID: \"${AWS_ACCESS_KEY_ID}\" \n AWS_SECRET_ACCESS_KEY: \"${AWS_SECRET_ACCESS_KEY}\"\n WALG_S3_PREFIX: \"${WALG_S3_PREFIX}\"/" "${configDir}/server.yml" -sed -i "/GOOGLE_APPLICATION_CREDENTIALS/d" "${configDir}/server.yml" + yq eval -i ' + del(.retrieval.spec.physicalRestore.options.envs.WALG_GS_PREFIX) | + del(.retrieval.spec.physicalRestore.options.envs.GOOGLE_APPLICATION_CREDENTIALS) | + .retrieval.spec.physicalRestore.options.envs.AWS_ACCESS_KEY_ID = strenv(AWS_ACCESS_KEY_ID) | + .retrieval.spec.physicalRestore.options.envs.AWS_SECRET_ACCESS_KEY = strenv(AWS_SECRET_ACCESS_KEY) | + .retrieval.spec.physicalRestore.options.envs.WALG_S3_PREFIX = strenv(WALG_S3_PREFIX) +' "${configDir}/server.yml" + elif [ -n "${WALG_GS_PREFIX}" ] ; then -sed -ri "s/^(\s*)(WALG_GS_PREFIX:.*$)/\1WALG_GS_PREFIX: \"${WALG_GS_PREFIX}\"/" "${configDir}/server.yml" -sed -ri "s/^(\s*)(GOOGLE_APPLICATION_CREDENTIALS:.*$)/\1GOOGLE_APPLICATION_CREDENTIALS: \"${GOOGLE_APPLICATION_CREDENTIALS}\"/" "${configDir}/server.yml" + yq eval -i ' + .retrieval.spec.physicalRestore.options.envs.WALG_GS_PREFIX = strenv(WALG_GS_PREFIX) | + .retrieval.spec.physicalRestore.options.envs.GOOGLE_APPLICATION_CREDENTIALS = strenv(GOOGLE_APPLICATION_CREDENTIALS) +' "${configDir}/server.yml" fi set -euxo pipefail # ---- -# replace postgres version -sed -ri "s/:13/:${POSTGRES_VERSION}/g" "${configDir}/server.yml" -# reduce shared_buffers (optional) -sed -ri "s/^(\s*)(shared_buffers:.*$)/\1shared_buffers: 512MB/" "${configDir}/server.yml" -# skip snapshotting on start to replace some Postgres parameters after PGDATA receiving -sed -ri "s/^(\s*)(skipStartSnapshot:.*$)/\1skipStartSnapshot: true/" "${configDir}/server.yml" ## Launch Database Lab server sudo docker run \ diff --git a/test/4.physical_basebackup.sh b/test/4.physical_basebackup.sh index a51bb09bcf84e871db207f07f9b57970be2cae46..3ed06b563b0572a8b483c903bd65b326d04e6246 100644 --- a/test/4.physical_basebackup.sh +++ b/test/4.physical_basebackup.sh @@ -3,11 +3,13 @@ set -euxo pipefail TAG="${TAG:-"master"}" IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}" -SOURCE_HOST="${SOURCE_HOST:-172.17.0.1}" -SOURCE_PORT="${SOURCE_PORT:-7432}" -SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}" -SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}" -POSTGRES_VERSION="${POSTGRES_VERSION:-13}" + +# Environment variables for replacement rules +export SOURCE_HOST="${SOURCE_HOST:-172.17.0.1}" +export SOURCE_PORT="${SOURCE_PORT:-7432}" +export SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}" +export SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}" +export POSTGRES_VERSION="${POSTGRES_VERSION:-13}" DIR=${0%/*} @@ -59,18 +61,24 @@ metaDir="$HOME/.dblab/engine/meta" # Copy the contents of configuration example mkdir -p "${configDir}" -curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.example.physical_generic.yml \ +curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${CI_COMMIT_BRANCH:-master}"/configs/config.example.physical_generic.yml \ --output "${configDir}/server.yml" # Edit the following options -sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml" -sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml" -sed -ri "s/^(\s*)(PGUSER:.*$)/\1PGUSER: ${SOURCE_USERNAME}/" "${configDir}/server.yml" -sed -ri "s/^(\s*)(PGPASSWORD:.*$)/\1PGPASSWORD: ${SOURCE_PASSWORD}/" "${configDir}/server.yml" -sed -ri "s/^(\s*)(PGHOST:.*$)/\1PGHOST: ${SOURCE_HOST}/" "${configDir}/server.yml" -sed -ri "s/^(\s*)(PGPORT:.*$)/\1PGPORT: ${SOURCE_PORT}/" "${configDir}/server.yml" -# replace postgres version -sed -ri "s/:13/:${POSTGRES_VERSION}/g" "${configDir}/server.yml" +yq eval -i ' + .global.debug = true | + .global.telemetry.enabled = false | + .localUI.enabled = false | + .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) | + .retrieval.spec.physicalRestore.options.envs.PGUSER = strenv(SOURCE_USERNAME) | + .retrieval.spec.physicalRestore.options.envs.PGPASSWORD = strenv(SOURCE_PASSWORD) | + .retrieval.spec.physicalRestore.options.envs.PGHOST = strenv(SOURCE_HOST) | + .retrieval.spec.physicalRestore.options.envs.PGPORT = env(SOURCE_PORT) | + .retrieval.spec.physicalSnapshot.options.envs.PGUSER = strenv(SOURCE_USERNAME) | + .retrieval.spec.physicalSnapshot.options.envs.PGPASSWORD = strenv(SOURCE_PASSWORD) | + .retrieval.spec.physicalSnapshot.options.envs.PGHOST = strenv(SOURCE_HOST) | + .retrieval.spec.physicalSnapshot.options.envs.PGPORT = env(SOURCE_PORT) +' "${configDir}/server.yml" ## Launch Database Lab server sudo docker run \ diff --git a/test/5.logical_rds.sh b/test/5.logical_rds.sh index 9308affdaec89ef4de8bbc80fd9e7ee7a7f349ee..8c6d13de715ef2d267c2060de33aaab08912ef98 100644 --- a/test/5.logical_rds.sh +++ b/test/5.logical_rds.sh @@ -3,14 +3,16 @@ set -euxo pipefail TAG="${TAG:-"master"}" IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}" -POSTGRES_VERSION="${POSTGRES_VERSION:-13}" -SOURCE_DBNAME="${SOURCE_DBNAME:-"test"}" -SOURCE_USERNAME="${SOURCE_USERNAME:-"test_user"}" -AWS_REGION="${AWS_REGION:-"us-east-2"}" -RDS_DB_IDENTIFIER="${RDS_DB_IDENTIFIER:-"logical-rds-test1"}" + +# Environment variables for replacement rules +export POSTGRES_VERSION="${POSTGRES_VERSION:-13}" +export SOURCE_DBNAME="${SOURCE_DBNAME:-"test"}" +export SOURCE_USERNAME="${SOURCE_USERNAME:-"test_user"}" +export AWS_REGION="${AWS_REGION:-"us-east-2"}" +export RDS_DB_IDENTIFIER="${RDS_DB_IDENTIFIER:-"logical-rds-test1"}" set +euxo pipefail # ---- do not display secrets -AWS_ACCESS_KEY="${AWS_ACCESS_KEY:-""}" -AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-""}" +export AWS_ACCESS_KEY="${AWS_ACCESS_KEY:-""}" +export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-""}" set -euxo pipefail # ---- DIR=${0%/*} @@ -28,18 +30,20 @@ metaDir="$HOME/.dblab/engine/meta" # Copy the contents of configuration example mkdir -p "${configDir}" -curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.example.logical_rds_iam.yml \ +curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${CI_COMMIT_BRANCH:-master}"/configs/config.example.logical_rds_iam.yml \ --output "${configDir}/server.yml" # Edit the following options -sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml" -sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml" -sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${SOURCE_DBNAME}/" "${configDir}/server.yml" -sed -ri "s/^(\s*)(username: test_user.*$)/\1username: \"${SOURCE_USERNAME}\"/" "${configDir}/server.yml" -sed -ri "s/^(\s*)(awsRegion:.*$)/\1awsRegion: \"${AWS_REGION}\"/" "${configDir}/server.yml" -sed -ri "s/^(\s*)(dbInstanceIdentifier:.*$)/\1dbInstanceIdentifier: \"${RDS_DB_IDENTIFIER}\"/" "${configDir}/server.yml" -# replace postgres version -sed -ri "s/:13/:${POSTGRES_VERSION}/g" "${configDir}/server.yml" +yq eval -i ' + .global.debug = true | + .global.telemetry.enabled = false | + .localUI.enabled = false | + .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) | + .retrieval.spec.logicalDump.options.source.connection.dbname = strenv(SOURCE_DBNAME) | + .retrieval.spec.logicalDump.options.source.connection.username = strenv(SOURCE_USERNAME) | + .retrieval.spec.logicalDump.options.source.rdsIam.awsRegion = strenv(AWS_REGION) | + .retrieval.spec.logicalDump.options.source.rdsIam.dbInstanceIdentifier = strenv(RDS_DB_IDENTIFIER) +' "${configDir}/server.yml" # Download AWS RDS certificate curl https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem \ diff --git a/test/_prerequisites.ubuntu.sh b/test/_prerequisites.ubuntu.sh index 2bd2d26ce77e1e78c65189c364ecf408af42d0f8..249fae0105f231a9b5d5d6aff5e7d451ca4f08af 100644 --- a/test/_prerequisites.ubuntu.sh +++ b/test/_prerequisites.ubuntu.sh @@ -29,3 +29,12 @@ sudo apt-get install -y \ # Install psql sudo apt-get install -y \ postgresql-client + +# Install yq +if ! command -v yq &> /dev/null; then + sudo curl -L https://github.com/mikefarah/yq/releases/download/v4.14.2/yq_linux_amd64 -o /usr/bin/yq && sudo chmod +x /usr/bin/yq +fi +# TODO: Uncomment after upgrade the runner machine +# sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 +# sudo add-apt-repository ppa:rmescandon/yq +# sudo apt-get update && sudo apt-get install yq -y