From c43d9e6db274e02614b7393bd71ab97839446f8b Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Fri, 5 Mar 2021 17:03:43 +0100 Subject: [PATCH 1/2] Capture which migrations are this branch only Relates to https://gitlab.com/gitlab-org/database-team/team-tasks/-/issues/157 --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8de38a0b..1657ba5e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,6 +63,7 @@ db:migrations: artifacts: paths: - migration-stats.json + - migrations-on-this-branch.list expire_in: 4 weeks when: always when: manual @@ -71,6 +72,7 @@ db:migrations: - cd ${GITLAB_PATH} - ./prepare.sh - bundle exec rake gitlab:db:migration_testing[${CI_PROJECT_DIR}/migration-stats.json] + - git diff --name-only origin/master...HEAD db/post_migrate db/migrate | grep -Po '(?<=/)([0-9]+)(?=\_)' > ${CI_PROJECT_DIR}/migrations-on-this-branch.list notify-upstream: stage: final @@ -83,6 +85,7 @@ notify-upstream: artifacts: paths: - migration-stats.json + - migrations-on-this-branch.list expire_in: 4 weeks when: always script: -- GitLab From 7468b9b420306fc43a73d35c193633d8d7acfa16 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Mon, 8 Mar 2021 12:36:42 +0100 Subject: [PATCH 2/2] Only display details for migrations from branch --- .gitlab-ci.yml | 2 +- notifier/feedback.rb | 21 +++++++++++++++++-- notifier/notifier.rb | 30 ++++++++++++++++++---------- notifier/templates/feedback.erb | 2 +- notifier/templates/summary_table.erb | 8 ++++---- 5 files changed, 44 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1657ba5e..57816ef1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,4 +91,4 @@ notify-upstream: script: - cd notifier - bundle install - - bundle exec ruby notifier.rb send ../migration-stats.json + - bundle exec ruby notifier.rb send ../migration-stats.json ../migrations-on-this-branch.list diff --git a/notifier/feedback.rb b/notifier/feedback.rb index 7c91b080..76689f7e 100644 --- a/notifier/feedback.rb +++ b/notifier/feedback.rb @@ -6,10 +6,11 @@ require 'pg_query' class Feedback UNKNOWN = ':grey_question:'.freeze - attr_reader :stats + attr_reader :stats, :migration_filter - def initialize(stats) + def initialize(stats, migration_filter: nil) @stats = stats + @migration_filter = migration_filter end def render @@ -18,6 +19,22 @@ class Feedback private + def filtered_migrations + return stats unless migration_filter + + stats.select do |stat| + filtered?(stat) + end + end + + def all_migrations + stats + end + + def filtered?(migration) + migration_filter.call(migration['migration']) + end + def render_details(migration) erb('detail').result(binding) end diff --git a/notifier/notifier.rb b/notifier/notifier.rb index 5cc2a804..a9f968d7 100644 --- a/notifier/notifier.rb +++ b/notifier/notifier.rb @@ -9,18 +9,15 @@ require 'thor' require_relative 'feedback' class Notifier < Thor - desc "send FILE", "send feedback back to merge request" - def send(file) - + desc "send STATS MIGRATIONS", "send feedback back to merge request" + def send(statistics_file, migrations_file) project_path = ENV['TOP_UPSTREAM_SOURCE_PROJECT'] merge_request_id = ENV['TOP_UPSTREAM_MERGE_REQUEST_IID'] raise "Project path missing: Specify TOP_UPSTREAM_SOURCE_PROJECT" unless project_path raise "Upstream merge request id missing: Specify TOP_UPSTREAM_MERGE_REQUEST_IID" unless merge_request_id - - stats = JSON.parse(File.read(file)) - comment = Feedback.new(stats).render + comment = feedback_for(statistics_file, migrations_file).render gitlab = Gitlab.client( endpoint: 'https://gitlab.com/api/v4', @@ -36,15 +33,26 @@ class Notifier < Thor end end - desc "print FILE", "only print feedback" - def print(file) - stats = JSON.parse(File.read(file)) - - puts Feedback.new(stats).render + desc "print STATS MIGRATIONS", "only print feedback" + def print(statistics_file, migrations_file) + puts feedback_for(statistics_file, migrations_file).render end private + def feedback_for(statistics_file, migrations_file) + stats = JSON.parse(File.read(statistics_file)) + migration_filter = migration_filter(migrations_file) + + Feedback.new(stats, migration_filter: migration_filter) + end + + def migration_filter(file) + migrations = Set.new(File.readlines(file).map(&:strip).map(&:to_i)) + + ->(version) { migrations.include?(version) } + end + def ignore_errors yield rescue Gitlab::Error::Error => e diff --git a/notifier/templates/feedback.erb b/notifier/templates/feedback.erb index 03970f4a..d262b4a7 100644 --- a/notifier/templates/feedback.erb +++ b/notifier/templates/feedback.erb @@ -4,7 +4,7 @@ Migrations included in this change have been executed on gitlab.com data for tes <%= render_summary_table %> -% stats.each do |migration| +% filtered_migrations.each do |migration| <%= render_details(migration) %> % end diff --git a/notifier/templates/summary_table.erb b/notifier/templates/summary_table.erb index 1b75b125..cb773759 100644 --- a/notifier/templates/summary_table.erb +++ b/notifier/templates/summary_table.erb @@ -1,5 +1,5 @@ -| Migration | Total runtime | Result | DB size change | -| --------- | ------------- | ------ | -------------- | -% stats.each do |m| -| <%= m['migration'] %> | <%= walltime(m) %> | <%= success(m) %> | <%= total_size_change(m) %> | +| Migration | Total runtime | Result | DB size change | Note | +| --------- | ------------- | ------ | -------------- | ---- | +% all_migrations.each do |m| +| <%= m['migration'] %> | <%= walltime(m) %> | <%= success(m) %> | <%= total_size_change(m) %> | <%= (!filtered?(m)) ? '(unrelated to this MR, but merged and pending on GitLab.com)' : '' %> | % end -- GitLab