diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fd9e3c0687443822b7778182fc681173efeeff08..828b243de548e0d979341e70fafd88d90e3a8d20 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,3 +56,9 @@ run pipeline: - apk add --update curl jq script: - ./misc/trigger-test-pipeline.sh + +run pipeline decomposed: + extends: + - run pipeline + variables: + RUN_DECOMPOSED: 'true' \ No newline at end of file diff --git a/db-testing.yml b/db-testing.yml index 2643427e4fab550826f468c5f5767081be467bd1..292e19d26a722a2280678e8863f13704a585fc85 100644 --- a/db-testing.yml +++ b/db-testing.yml @@ -17,6 +17,14 @@ variables: tags: - builder +.rules:if-decomposed: + rules: + - if: '$RUN_DECOMPOSED == "true"' + +.rules:if-not-decomposed: + rules: + - if: '$RUN_DECOMPOSED == null || $RUN_DECOMPOSED != "true"' + image:gitlab: extends: .image:base script: @@ -42,7 +50,7 @@ image:redis: - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY}/${IMAGE_NAME} - docker push ${DOCKER_REGISTRY}/${IMAGE_NAME} -db:migrations: +.db:migrations:base: stage: test image: ${DOCKER_REGISTRY}/$IMAGE_NAME tags: @@ -56,11 +64,7 @@ db:migrations: DBLAB_API_PORT: $DBLAB_API_PORT DBLAB_ENVIRONMENT: $DBLAB_ENVIRONMENT DBLAB_CLONE_ID: "database-testing-${CI_PIPELINE_ID}" - services: - - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest - alias: postgres - - name: ${DOCKER_REGISTRY}/redis:4.0-alpine - alias: redis + CLONE_DETAILS_DB_NAME: "postgres" # temporary to make clone_details work with multiple databases artifacts: paths: - migration-testing/ @@ -74,15 +78,43 @@ db:migrations: - bundle exec rake gitlab:db:migration_testing:up | tee tmp/migration-testing/up/full-migration-output.log || true - mv tmp/migration-testing/ ${CI_PROJECT_DIR} - bundle exec ruby list_migrations.rb > ${CI_PROJECT_DIR}/migration-testing/migrations.json - - bundle exec ruby clone_details.rb postgres > ${CI_PROJECT_DIR}/migration-testing/clone-details.json + - bundle exec ruby clone_details.rb ${CLONE_DETAILS_DB_NAME} > ${CI_PROJECT_DIR}/migration-testing/clone-details.json + +db:migrations: + extends: + - .db:migrations:base + - .rules:if-not-decomposed + variables: + ALL_DB_HOSTS: "postgres" + services: + - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest + alias: postgres + - name: ${DOCKER_REGISTRY}/redis:4.0-alpine + alias: redis + +db:migrations decomposed: + extends: + - .db:migrations:base + - .rules:if-decomposed + variables: + ALL_DB_HOSTS: "postgres-main,postgres-ci" + CLONE_DETAILS_DB_NAME: "postgres-main" # Temporary to make clone_details script work + services: + - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest + alias: postgres-main + variables: + DBLAB_CLONE_ID: "database-testing-${CI_PIPELINE_ID}-main" + - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest + alias: postgres-ci + variables: + DBLAB_CLONE_ID: "database-testing-${CI_PIPELINE_ID}-ci" + - name: ${DOCKER_REGISTRY}/redis:4.0-alpine + alias: redis notify-upstream: stage: final tags: - builder - needs: - - job: db:migrations - artifacts: true # Also store artifacts here so we can easily generate direct links to them artifacts: paths: diff --git a/docker/gitlab/prepare.sh b/docker/gitlab/prepare.sh index 70cdde1d169aceedc28bd41e36f1fb6b0e7c4f4b..bfa6459e18f997f2c60ce7f6e620e623bb20ab40 100755 --- a/docker/gitlab/prepare.sh +++ b/docker/gitlab/prepare.sh @@ -3,7 +3,36 @@ cp config/gitlab.yml.example config/gitlab.yml sed -i 's/bin_path: \/usr\/bin\/git/bin_path: \/usr\/local\/bin\/git/' config/gitlab.yml -cat > config/database.yml <<-EOF +if [ "$RUN_DECOMPOSED" = "true" ]; then + echo 'using decomposed database.yml' + + cat > config/database.yml <<-EOF +test: &test + main: + adapter: postgresql + encoding: unicode + database: gitlabhq_dblab + username: ${DBLAB_USER} + password: ${DBLAB_PASSWORD} + host: postgres-main + prepared_statements: false + variables: + statement_timeout: 120s + ci: + adapter: postgresql + encoding: unicode + database: gitlabhq_dblab + username: ${DBLAB_USER} + password: ${DBLAB_PASSWORD} + host: postgres-ci + prepared_statements: false + variables: + statement_timeout: 120s +EOF +else + echo 'using single database database.yml' + + cat > config/database.yml <<-EOF test: &test adapter: postgresql encoding: unicode @@ -16,6 +45,8 @@ test: &test statement_timeout: 120s EOF +fi + if test "$VALIDATION_PIPELINE"; then echo "Applying test patch" git am < /gitlab/patches/testing/New-Table-Migration.patch @@ -44,9 +75,12 @@ sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml ### Preparing PG cluster -if timeout 60s bash -c "until pg_isready --quiet -h postgres -U ${DBLAB_USER} --dbname=gitlabhq_dblab; do sleep 1; done"; then - PGPASSWORD="${DBLAB_PASSWORD}" psql -h postgres -U "${DBLAB_USER}" gitlabhq_dblab < /gitlab/prepare_postgres.sql -else - echo 'Unable to connect to database lab psql for optional configuration' -fi +# These will never have spaces, so it's safe to split this way +for db_host in $(echo "$ALL_DB_HOSTS" | tr ',' '\n'); do + if timeout 60s bash -c "until pg_isready --quiet -h ${db_host} -U ${DBLAB_USER} --dbname=gitlabhq_dblab; do sleep 1; done"; then + PGPASSWORD="${DBLAB_PASSWORD}" psql -h "${db_host}" -U "${DBLAB_USER}" gitlabhq_dblab < /gitlab/prepare_postgres.sql + else + echo "Unable to connect to database lab psql for optional configuration of ${db_host}" + fi +done diff --git a/misc/trigger-test-pipeline.sh b/misc/trigger-test-pipeline.sh index 84a4ea28bdb000e81fe2b7e5c0cd155f98e02165..e6f23eea2083356d23808654c7ea36b3d82f0fe1 100755 --- a/misc/trigger-test-pipeline.sh +++ b/misc/trigger-test-pipeline.sh @@ -23,4 +23,5 @@ curl --fail-with-body \ --form "variables[VALIDATION_PIPELINE]=true" \ --form "variables[GITLAB_COMMIT_SHA]=$SHA" \ --form "variables[TRIGGER_SOURCE]=$CI_JOB_URL" \ + --form "variables[RUN_DECOMPOSED]=$RUN_DECOMPOSED" \ "https://ops.gitlab.net/api/v4/projects/429/trigger/pipeline"