From 435b407e8ea69ca45867b7338192022697e426f7 Mon Sep 17 00:00:00 2001
From: Simon Tomlinson <stomlinson@gitlab.com>
Date: Tue, 25 Jan 2022 10:32:20 -0600
Subject: [PATCH 1/3] Revert "Revert "Merge branch 'run-migrations-decomposed'
 into 'master'""

This reverts commit 8f1cf499c962d6aab0d75335fe16bb33bd0eeca6.
---
 .gitlab-ci.yml                |  8 ++++++
 db-testing.yml                | 50 ++++++++++++++++++++++++++++-------
 docker/gitlab/prepare.sh      | 33 ++++++++++++++++++++++-
 misc/trigger-test-pipeline.sh |  1 +
 4 files changed, 81 insertions(+), 11 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fd9e3c06..985ae5a7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,5 +54,13 @@ run pipeline:
   image: python:alpine
   before_script:
     - apk add --update curl jq
+  variables:
+    RUN_DECOMPOSED: 'false'
   script:
     - ./misc/trigger-test-pipeline.sh
+
+run pipeline decomposed:
+  extends:
+    - run pipeline
+  variables:
+    RUN_DECOMPOSED: 'true'
\ No newline at end of file
diff --git a/db-testing.yml b/db-testing.yml
index 2643427e..edb5ef29 100644
--- a/db-testing.yml
+++ b/db-testing.yml
@@ -17,6 +17,15 @@ variables:
   tags:
     - builder
 
+.rules:if-decomposed:
+  rules:
+    - if: '$RUN_DECOMPOSED == "true"'
+
+.rules:if-not-decomposed:
+  rules:
+    - if: '$RUN_DECOMPOSED == "true"'
+      when: never
+
 image:gitlab:
   extends: .image:base
   script:
@@ -42,7 +51,7 @@ image:redis:
     - docker tag ${IMAGE_NAME} ${DOCKER_REGISTRY}/${IMAGE_NAME}
     - docker push ${DOCKER_REGISTRY}/${IMAGE_NAME}
 
-db:migrations:
+.db:migrations:base:
   stage: test
   image: ${DOCKER_REGISTRY}/$IMAGE_NAME
   tags:
@@ -56,11 +65,7 @@ db:migrations:
     DBLAB_API_PORT: $DBLAB_API_PORT
     DBLAB_ENVIRONMENT: $DBLAB_ENVIRONMENT
     DBLAB_CLONE_ID: "database-testing-${CI_PIPELINE_ID}"
-  services:
-    - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest
-      alias: postgres
-    - name: ${DOCKER_REGISTRY}/redis:4.0-alpine
-      alias: redis
+    CLONE_DETAILS_DB_NAME: "postgres" # temporary to make clone_details work with multiple databases
   artifacts:
     paths:
       - migration-testing/
@@ -74,15 +79,40 @@ db:migrations:
     - bundle exec rake gitlab:db:migration_testing:up | tee tmp/migration-testing/up/full-migration-output.log || true
     - mv tmp/migration-testing/ ${CI_PROJECT_DIR}
     - bundle exec ruby list_migrations.rb > ${CI_PROJECT_DIR}/migration-testing/migrations.json
-    - bundle exec ruby clone_details.rb postgres > ${CI_PROJECT_DIR}/migration-testing/clone-details.json
+    - bundle exec ruby clone_details.rb ${CLONE_DETAILS_DB_NAME} > ${CI_PROJECT_DIR}/migration-testing/clone-details.json
+
+db:migrations:
+  extends:
+    - .db:migrations:base
+    - .rules:if-not-decomposed
+  services:
+    - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest
+      alias: postgres
+    - name: ${DOCKER_REGISTRY}/redis:4.0-alpine
+      alias: redis
+
+db:migrations decomposed:
+  extends:
+    - .db:migrations:base
+    - .rules:if-decomposed
+  variables:
+    CLONE_DETAILS_DB_NAME: "postgres-main" # Temporary to make clone_details script work
+  services:
+    - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest
+      alias: postgres-main
+      variables:
+        DBLAB_CLONE_ID: "database-testing-${CI_PIPELINE_ID}-main"
+    - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest
+      alias: postgres-ci
+      variables:
+        DBLAB_CLONE_ID: "database-testing-${CI_PIPELINE_ID}-ci"
+    - name: ${DOCKER_REGISTRY}/redis:4.0-alpine
+      alias: redis
 
 notify-upstream:
   stage: final
   tags:
     - builder
-  needs:
-    - job: db:migrations
-      artifacts: true
   # Also store artifacts here so we can easily generate direct links to them
   artifacts:
     paths:
diff --git a/docker/gitlab/prepare.sh b/docker/gitlab/prepare.sh
index 70cdde1d..8422edcb 100755
--- a/docker/gitlab/prepare.sh
+++ b/docker/gitlab/prepare.sh
@@ -3,7 +3,36 @@
 cp config/gitlab.yml.example config/gitlab.yml
 sed -i 's/bin_path: \/usr\/bin\/git/bin_path: \/usr\/local\/bin\/git/' config/gitlab.yml
 
-cat > config/database.yml <<-EOF
+if test "$RUN_DECOMPOSED"; then
+  echo 'using decomposed database.yml'
+
+  cat > config/database.yml <<-EOF
+test: &test
+  main:
+    adapter: postgresql
+    encoding: unicode
+    database: gitlabhq_dblab
+    username: ${DBLAB_USER}
+    password: ${DBLAB_PASSWORD}
+    host: postgres-main
+    prepared_statements: false
+    variables:
+      statement_timeout: 120s
+  ci:
+    adapter: postgresql
+    encoding: unicode
+    database: gitlabhq_dblab
+    username: ${DBLAB_USER}
+    password: ${DBLAB_PASSWORD}
+    host: postgres-ci
+    prepared_statements: false
+    variables:
+      statement_timeout: 120s
+EOF
+else
+  echo 'using single database database.yml'
+
+  cat > config/database.yml <<-EOF
 test: &test
   adapter: postgresql
   encoding: unicode
@@ -16,6 +45,8 @@ test: &test
     statement_timeout: 120s
 EOF
 
+fi
+
 if test "$VALIDATION_PIPELINE"; then
   echo "Applying test patch"
   git am < /gitlab/patches/testing/New-Table-Migration.patch
diff --git a/misc/trigger-test-pipeline.sh b/misc/trigger-test-pipeline.sh
index 84a4ea28..e6f23eea 100755
--- a/misc/trigger-test-pipeline.sh
+++ b/misc/trigger-test-pipeline.sh
@@ -23,4 +23,5 @@ curl --fail-with-body \
      --form "variables[VALIDATION_PIPELINE]=true" \
      --form "variables[GITLAB_COMMIT_SHA]=$SHA" \
      --form "variables[TRIGGER_SOURCE]=$CI_JOB_URL" \
+     --form "variables[RUN_DECOMPOSED]=$RUN_DECOMPOSED" \
      "https://ops.gitlab.net/api/v4/projects/429/trigger/pipeline"
-- 
GitLab


From d217a8e489e9a77104c4ee905bd138e281d3f3f5 Mon Sep 17 00:00:00 2001
From: Simon Tomlinson <stomlinson@gitlab.com>
Date: Tue, 25 Jan 2022 10:38:46 -0600
Subject: [PATCH 2/3] Fix if: for non-decomposed pipeline

---
 .gitlab-ci.yml           | 2 --
 db-testing.yml           | 3 +--
 docker/gitlab/prepare.sh | 2 +-
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 985ae5a7..828b243d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,8 +54,6 @@ run pipeline:
   image: python:alpine
   before_script:
     - apk add --update curl jq
-  variables:
-    RUN_DECOMPOSED: 'false'
   script:
     - ./misc/trigger-test-pipeline.sh
 
diff --git a/db-testing.yml b/db-testing.yml
index edb5ef29..5011223a 100644
--- a/db-testing.yml
+++ b/db-testing.yml
@@ -23,8 +23,7 @@ variables:
 
 .rules:if-not-decomposed:
   rules:
-    - if: '$RUN_DECOMPOSED == "true"'
-      when: never
+    - if: '$RUN_DECOMPOSED == null || $RUN_DECOMPOSED != "true"'
 
 image:gitlab:
   extends: .image:base
diff --git a/docker/gitlab/prepare.sh b/docker/gitlab/prepare.sh
index 8422edcb..bc89ff2c 100755
--- a/docker/gitlab/prepare.sh
+++ b/docker/gitlab/prepare.sh
@@ -3,7 +3,7 @@
 cp config/gitlab.yml.example config/gitlab.yml
 sed -i 's/bin_path: \/usr\/bin\/git/bin_path: \/usr\/local\/bin\/git/' config/gitlab.yml
 
-if test "$RUN_DECOMPOSED"; then
+if [ "$RUN_DECOMPOSED" = "true" ]; then
   echo 'using decomposed database.yml'
 
   cat > config/database.yml <<-EOF
-- 
GitLab


From 6d77655385b7971527b90a08b254e4280c4386e8 Mon Sep 17 00:00:00 2001
From: Simon Tomlinson <stomlinson@gitlab.com>
Date: Tue, 25 Jan 2022 11:06:57 -0600
Subject: [PATCH 3/3] configure auto-explain for all databases

---
 db-testing.yml           |  3 +++
 docker/gitlab/prepare.sh | 13 ++++++++-----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/db-testing.yml b/db-testing.yml
index 5011223a..292e19d2 100644
--- a/db-testing.yml
+++ b/db-testing.yml
@@ -84,6 +84,8 @@ db:migrations:
   extends:
     - .db:migrations:base
     - .rules:if-not-decomposed
+  variables:
+    ALL_DB_HOSTS: "postgres"
   services:
     - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest
       alias: postgres
@@ -95,6 +97,7 @@ db:migrations decomposed:
     - .db:migrations:base
     - .rules:if-decomposed
   variables:
+    ALL_DB_HOSTS: "postgres-main,postgres-ci"
     CLONE_DETAILS_DB_NAME: "postgres-main" # Temporary to make clone_details script work
   services:
     - name: ${DOCKER_REGISTRY}/gitlab-com-database-testing-dblab-ssh:latest
diff --git a/docker/gitlab/prepare.sh b/docker/gitlab/prepare.sh
index bc89ff2c..bfa6459e 100755
--- a/docker/gitlab/prepare.sh
+++ b/docker/gitlab/prepare.sh
@@ -75,9 +75,12 @@ sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml
 
 ### Preparing PG cluster
 
-if timeout 60s bash -c "until pg_isready --quiet -h postgres -U ${DBLAB_USER} --dbname=gitlabhq_dblab; do sleep 1; done"; then
-  PGPASSWORD="${DBLAB_PASSWORD}" psql -h postgres -U "${DBLAB_USER}" gitlabhq_dblab < /gitlab/prepare_postgres.sql
-else
-  echo 'Unable to connect to database lab psql for optional configuration'
-fi
+# These will never have spaces, so it's safe to split this way
+for db_host in $(echo "$ALL_DB_HOSTS" | tr ',' '\n'); do
+  if timeout 60s bash -c "until pg_isready --quiet -h ${db_host} -U ${DBLAB_USER} --dbname=gitlabhq_dblab; do sleep 1; done"; then
+    PGPASSWORD="${DBLAB_PASSWORD}" psql -h "${db_host}" -U "${DBLAB_USER}" gitlabhq_dblab < /gitlab/prepare_postgres.sql
+  else
+    echo "Unable to connect to database lab psql for optional configuration of ${db_host}"
+  fi
+done
 
-- 
GitLab