diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8de38a0b9caf8f860adfd7e89637aafa5d2f4b06..ada9b8dbbfdb37387056ce6220098d700d4a367a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,13 @@ stages: - prepare - test - - final + - bgmigrations default: timeout: 300m variables: - RAILS_ENV: "test" + RAILS_ENV: "development" GITLAB_PATH: "/gitlab" GITLAB_COMMIT_SHA: "master" IMAGE_NAME: "gitlab-com-database-testing:${GITLAB_COMMIT_SHA}" @@ -60,32 +60,53 @@ db:migrations: alias: postgres - name: ${DOCKER_REGISTRY}/redis:4.0-alpine alias: redis - artifacts: - paths: - - migration-stats.json - expire_in: 4 weeks - when: always - when: manual - allow_failure: true script: - cd ${GITLAB_PATH} - ./prepare.sh - bundle exec rake gitlab:db:migration_testing[${CI_PROJECT_DIR}/migration-stats.json] -notify-upstream: - stage: final +scheduler: + stage: bgmigrations + image: ${DOCKER_REGISTRY}/$IMAGE_NAME tags: - - builder - needs: - - job: db:migrations - artifacts: true - # Also store artifacts here so we can easily generate direct links to them - artifacts: - paths: - - migration-stats.json - expire_in: 4 weeks - when: always + - worker + variables: + DBLAB_HOST_KEYS: $DBLAB_HOST_KEYS + DBLAB_SSH_KEY: $DBLAB_SSH_KEY + DBLAB_TOKEN: $DBLAB_TOKEN + DBLAB_USER: $DBLAB_USER + DBLAB_PASSWORD: $DBLAB_PASSWORD + DBLAB_ENVIRONMENT: $DBLAB_ENVIRONMENT + DBLAB_CLONE_ID: "database-testing-${CI_PIPELINE_ID}" + services: + - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest + alias: postgres + - name: ${DOCKER_REGISTRY}/redis:4.0-alpine + alias: redis + script: + - cd ${GITLAB_PATH} + - ./prepare.sh + - bundle exec rake gitlab:bgmigrations:scheduler + +executor: + stage: bgmigrations + image: ${DOCKER_REGISTRY}/$IMAGE_NAME + tags: + - worker + variables: + DBLAB_HOST_KEYS: $DBLAB_HOST_KEYS + DBLAB_SSH_KEY: $DBLAB_SSH_KEY + DBLAB_TOKEN: $DBLAB_TOKEN + DBLAB_USER: $DBLAB_USER + DBLAB_PASSWORD: $DBLAB_PASSWORD + DBLAB_ENVIRONMENT: $DBLAB_ENVIRONMENT + DBLAB_CLONE_ID: "database-testing-${CI_PIPELINE_ID}" + services: + - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest + alias: postgres + - name: ${DOCKER_REGISTRY}/redis:4.0-alpine + alias: redis script: - - cd notifier - - bundle install - - bundle exec ruby notifier.rb send ../migration-stats.json + - cd ${GITLAB_PATH} + - ./prepare.sh + - bundle exec rake gitlab:bgmigrations:executor diff --git a/docker/dblab-ssh/entrypoint.sh b/docker/dblab-ssh/entrypoint.sh index b3461707071cb1835c98ea30284313b3a20c0110..7270655106b3cc69f4a7ed1e30392ef7a1afa2c4 100644 --- a/docker/dblab-ssh/entrypoint.sh +++ b/docker/dblab-ssh/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -x if [ -z "${DBLAB_SSH_KEY}" ] || [ "${DBLAB_SSH_KEY}" = "unset" ]; then echo "DBLAB_SSH_KEY not set" @@ -18,16 +18,14 @@ dblab init --url http://127.0.0.1:2344 --token ${DBLAB_TOKEN} --environment-id $ dblab_info=$(dblab clone create --id ${DBLAB_CLONE_ID} --username ${DBLAB_USER} --password ${DBLAB_PASSWORD}) +if [ $? -ne 0 ]; then + # Clone exists already - we need the port information to open the tunnel + dblab_info=$(dblab clone status ${DBLAB_CLONE_ID}) +fi + port=$(echo $dblab_info | jq -r .db.port) id=$(echo $dblab_info | jq -r .id) -dblab_cleanup() { - echo "Cleaning up and destroying dblab clone $id" - dblab clone destroy $id -} - -trap dblab_cleanup TERM INT EXIT - echo "Opening port forwarding on port ${port}" # This blocks and opens port forwarding for Postgres diff --git a/docker/gitlab/Dockerfile b/docker/gitlab/Dockerfile index a659bc43594a8dd1f5d52f47df73091c39286c90..cf1f592f65f605fc938d39a8de02ae4b29d6fe5c 100644 --- a/docker/gitlab/Dockerfile +++ b/docker/gitlab/Dockerfile @@ -24,7 +24,6 @@ RUN bundle install ARG GITLAB_COMMIT_SHA RUN git fetch origin; git checkout ${GITLAB_COMMIT_SHA} -RUN git merge origin/master RUN bundle install RUN bundle check diff --git a/docker/gitlab/patches/0002-Disable-automatic-schema-dumping-after-migration.patch b/docker/gitlab/patches/0002-Disable-automatic-schema-dumping-after-migration.patch index 2eeb0c7ac5d390c08f33dbb148f4a74dab2d567d..f554807255a867eba23a619b9ee8863d758b88bc 100644 --- a/docker/gitlab/patches/0002-Disable-automatic-schema-dumping-after-migration.patch +++ b/docker/gitlab/patches/0002-Disable-automatic-schema-dumping-after-migration.patch @@ -1,25 +1,22 @@ -From 0bd84872e18d4183310ef44ac790cbb0861d51e0 Mon Sep 17 00:00:00 2001 +From 47984da16be2db9bc307b74a7e5d2f400b1c24c0 Mon Sep 17 00:00:00 2001 From: Andreas Brandl -Date: Tue, 29 Dec 2020 23:24:20 +0100 -Subject: [PATCH] Disable automatic schema dumping after migration +Date: Wed, 24 Feb 2021 15:44:25 +0100 +Subject: [PATCH] Disable schema dump --- - config/environments/test.rb | 2 ++ + config/environments/development.rb | 2 ++ 1 file changed, 2 insertions(+) -diff --git a/config/environments/test.rb b/config/environments/test.rb -index 4e359cd308c..0a08c1b7b98 100644 ---- a/config/environments/test.rb -+++ b/config/environments/test.rb -@@ -50,6 +50,8 @@ - - config.active_job.queue_adapter = :test - -+ config.active_record.dump_schema_after_migration = false -+ - if ENV['CI'] && !ENV['RAILS_ENABLE_TEST_LOG'] - config.logger = ActiveSupport::TaggedLogging.new(Logger.new(nil)) - config.log_level = :fatal --- -2.29.0 +diff --git a/config/environments/development.rb b/config/environments/development.rb +index 50d394859bc..38c69fbe205 100644 +--- a/config/environments/development.rb ++++ b/config/environments/development.rb +@@ -71,4 +71,6 @@ + config.middleware.delete BetterErrors::Middleware + end ++ ++ config.active_record.dump_schema_after_migration = false + end +-- +2.30.0 diff --git a/docker/gitlab/prepare.sh b/docker/gitlab/prepare.sh index e0b4f0ab58daeadbba848c92b166ebfa2b93c069..35e43ee2c2b1fcb32bbe4764faab52be78e211ed 100755 --- a/docker/gitlab/prepare.sh +++ b/docker/gitlab/prepare.sh @@ -4,7 +4,7 @@ cp config/gitlab.yml.example config/gitlab.yml sed -i 's/bin_path: \/usr\/bin\/git/bin_path: \/usr\/local\/bin\/git/' config/gitlab.yml cat > config/database.yml <<-EOF -test: &test +development: &development adapter: postgresql encoding: unicode database: gitlabhq_dblab