diff --git a/notifier/charts/execution_histogram.rb b/notifier/charts/execution_histogram.rb index 5afe02ae7ffa9e9ecc4e6adfe07487b56c53d504..ec5e8aa18054379f5ba9bc46898a3a76b80a2b2c 100644 --- a/notifier/charts/execution_histogram.rb +++ b/notifier/charts/execution_histogram.rb @@ -13,12 +13,12 @@ module Charts def self.for_result(result) executions = result.migrations_from_branch - .flat_map(&:query_executions) + .flat_map(&:important_query_executions) new(executions, title: 'Runtime Histogram for all migrations') end def self.for_migration(migration) - new(migration.query_executions, title: "Histogram for #{migration.name}") + new(migration.important_query_executions, title: "Histogram for #{migration.name}") end def initialize(query_executions, title:, bucket_cutoffs: DEFAULT_CUTOFFS) diff --git a/notifier/migration.rb b/notifier/migration.rb index 94ee04ce7ee59c061d140a81742212e8716a19e2..10f1a1356f5d3839eaee6c5afd8786801ea8e1c7 100644 --- a/notifier/migration.rb +++ b/notifier/migration.rb @@ -55,6 +55,10 @@ class Migration queries.reject(&:excluded?) end + def important_query_executions + query_executions.reject(&:excluded?) + end + def queries_with_warnings @queries_with_warnings ||= important_queries.select(&:exceeds_time_guidance?) end diff --git a/notifier/query.rb b/notifier/query.rb index 010632ee48f52991095b23a10db6ba337e4bd102..ea13d20a24bc91e24cba8c675232687168bc169e 100644 --- a/notifier/query.rb +++ b/notifier/query.rb @@ -2,28 +2,13 @@ require 'pg_query' require_relative 'query_execution' +require_relative 'query_exclusion' class Query QUERY_GUIDANCE_MILLISECONDS = 100 CONCURRENT_QUERY_GUIDANCE_MILLISECONDS = 5.minutes.in_milliseconds TIMING_GUIDELINES = 'https://docs.gitlab.com/ee/development/query_performance.html#timing-guidelines-for-queries' - # rubocop:disable Layout/LineLength - EXCLUSIONS = [ - "select pg_database_size(current_database()) /*application:test*/", - "SELECT \"schema_migrations\".\"version\" FROM \"schema_migrations\" ORDER BY \"schema_migrations\".\"version\" ASC /*application:test*/", - "select pg_stat_statements_reset() /*application:test*/", - "SELECT c.relname FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = ANY (current_schemas($1)) AND c.relname = $2 AND c.relkind IN ($3,$4)", - "INSERT INTO \"schema_migrations\" (\"version\") VALUES ($1) RETURNING \"version\" /*application:test*/", - "SELECT \"ar_internal_metadata\".* FROM \"ar_internal_metadata\" WHERE \"ar_internal_metadata\".\"key\" = $1 LIMIT $2 /*application:test*/", - "SELECT pg_try_advisory_lock($1)", - "SELECT pg_advisory_unlock($1)", - "SELECT current_database()", - "SELECT $1", - "SELECT current_schema" - ].map(&:downcase).freeze - # rubocop:enable Layout/LineLength - attr_accessor :query, :calls, :total_time, :max_time, :mean_time, :rows, :executions def initialize(pgss_row) @@ -75,20 +60,6 @@ class Query end def excluded? - return true if query.downcase.start_with?(/commit|show|reset|begin|release savepoint|savepoint|set|rollback/) - return true if query.include?('/* pgssignore */') - return true if EXCLUSIONS.include?(query.downcase) - - tables = PgQuery.parse(query).tables - - return false if tables.empty? - - return true if tables.all? { |table| table.start_with?('pg_') } - return true if tables.uniq == ['ar_internal_metadata'] - - false - rescue PgQuery::ParseError => e - warn "Query parse error:\n#{e}\nFor query: #{query}" - false + QueryExclusion.exclude?(query) end end diff --git a/notifier/query_exclusion.rb b/notifier/query_exclusion.rb new file mode 100644 index 0000000000000000000000000000000000000000..97c355cd476d7cd62ad0eb5fea85a5d67aa524b5 --- /dev/null +++ b/notifier/query_exclusion.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'pg_query' + +class QueryExclusion + # rubocop:disable Layout/LineLength + EXCLUSIONS = [ + "select pg_database_size(current_database()) /*application:test*/", + "SELECT \"schema_migrations\".\"version\" FROM \"schema_migrations\" ORDER BY \"schema_migrations\".\"version\" ASC /*application:test*/", + "select pg_stat_statements_reset() /*application:test*/", + "SELECT c.relname FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = ANY (current_schemas($1)) AND c.relname = $2 AND c.relkind IN ($3,$4)", + "INSERT INTO \"schema_migrations\" (\"version\") VALUES ($1) RETURNING \"version\" /*application:test*/", + "SELECT \"ar_internal_metadata\".* FROM \"ar_internal_metadata\" WHERE \"ar_internal_metadata\".\"key\" = $1 LIMIT $2 /*application:test*/", + "SELECT pg_try_advisory_lock($1)", + "SELECT pg_advisory_unlock($1)", + "SELECT current_database()", + "SELECT $1", + "SELECT current_schema" + ].map(&:downcase).freeze + # rubocop:enable Layout/LineLength + + def self.exclude?(sql) + normalized_query = PgQuery.normalize(sql).downcase + return true if normalized_query.start_with?(/commit|show|reset|begin|release savepoint|savepoint|set|rollback/) + return true if normalized_query.include?('/* pgssignore */') + return true if EXCLUSIONS.include?(normalized_query) + + tables = PgQuery.parse(normalized_query).tables + + return false if tables.empty? + + return true if tables.all? { |table| table.start_with?('pg_') } + return true if tables.uniq == ['ar_internal_metadata'] + + false + rescue PgQuery::ParseError => e + warn "Query parse error:\n#{e}\nFor query: #{query}" + false + end +end diff --git a/notifier/query_execution.rb b/notifier/query_execution.rb index e7a6df6402614c889af1eb90d96792a5aea2195b..a4cb89c242d69875f0f47ea6d63c55ff26245911 100644 --- a/notifier/query_execution.rb +++ b/notifier/query_execution.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative 'query_exclusion' + class QueryExecution attr_accessor :sql, :start_time, :end_time, :binds @@ -9,6 +11,10 @@ class QueryExecution @end_time = Time.rfc3339(end_time) end + def excluded? + QueryExclusion.exclude?(@sql) + end + def duration ActiveSupport::Duration.build(end_time - start_time) end diff --git a/notifier/spec/fixtures/migration-testing/expected-comment.txt b/notifier/spec/fixtures/migration-testing/expected-comment.txt index 884393bd01b47cdec2f622c4e338c6d7e89027f7..bbc73038cdc0aab110a24fdc78d42c2021b55e9c 100644 --- a/notifier/spec/fixtures/migration-testing/expected-comment.txt +++ b/notifier/spec/fixtures/migration-testing/expected-comment.txt @@ -24,8 +24,8 @@ Migrations included in this change have been executed on gitlab.com data for tes | Query Runtime | Count | |---------------|-------| |0 seconds - 0.01 seconds | 0 | -|0.01 seconds - 0.1 seconds | 61 | -|0.1 seconds - 1 second | 5 | +|0.01 seconds - 0.1 seconds | 3 | +|0.1 seconds - 1 second | 1 | |1 second - 5 minutes | 0 | |5 minutes + | 0 | @@ -57,8 +57,8 @@ Migrations included in this change have been executed on gitlab.com data for tes | Query Runtime | Count | |---------------|-------| |0 seconds - 0.01 seconds | 0 | -|0.01 seconds - 0.1 seconds | 28 | -|0.1 seconds - 1 second | 3 | +|0.01 seconds - 0.1 seconds | 2 | +|0.1 seconds - 1 second | 1 | |1 second - 5 minutes | 0 | |5 minutes + | 0 | @@ -87,8 +87,8 @@ Migrations included in this change have been executed on gitlab.com data for tes | Query Runtime | Count | |---------------|-------| |0 seconds - 0.01 seconds | 0 | -|0.01 seconds - 0.1 seconds | 19 | -|0.1 seconds - 1 second | 1 | +|0.01 seconds - 0.1 seconds | 1 | +|0.1 seconds - 1 second | 0 | |1 second - 5 minutes | 0 | |5 minutes + | 0 | @@ -101,19 +101,6 @@ Migrations included in this change have been executed on gitlab.com data for tes * Database size change: +0.00 B -
-Histogram for MigrationThrowsException - -| Query Runtime | Count | -|---------------|-------| -|0 seconds - 0.01 seconds | 0 | -|0.01 seconds - 0.1 seconds | 14 | -|0.1 seconds - 1 second | 1 | -|1 second - 5 minutes | 0 | -|5 minutes + | 0 | - -
- --- diff --git a/notifier/spec/query_exclusion_spec.rb b/notifier/spec/query_exclusion_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..71fc1c6bc4663516dc91ab0f70bcb9295032c249 --- /dev/null +++ b/notifier/spec/query_exclusion_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe QueryExclusion do + it 'is true if query is on the excluded list' do + query = described_class::EXCLUSIONS.first + + expect(described_class.exclude?(query)).to be true + end + + it 'is true if query includes /* pgssignore /*' do + query = "SELECT 1 /* pgssignore */" + + expect(described_class.exclude?(query)).to be true + end + + it 'is true if query table is ar_internal_metadata' do + query = "SELECT * from ar_internal_metadata;" + + expect(described_class.exclude?(query)).to be true + end + + it 'is true if query table starts with pg_' do + query = "SELECT * from pg_internal_nonsense;" + + expect(described_class.exclude?(query)).to be true + end + + it 'is false for other queries' do + query = "SELECT * from user;" + + expect(described_class.exclude?(query)).to be false + end +end diff --git a/notifier/spec/query_execution_spec.rb b/notifier/spec/query_execution_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..15ccfac3657f0ce34e6cf89dd72dd78a2b257181 --- /dev/null +++ b/notifier/spec/query_execution_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe QueryExecution do + def make_execution(sql:, binds: [], start_time: Time.current.rfc3339, end_time: Time.current.rfc3339) + QueryExecution.new({ "sql" => sql, "binds" => binds, "start_time" => start_time, "end_time" => end_time }) + end + + describe '#excluded?' do + it 'delegates to QueryExclusion' do + execution = make_execution(sql: 'foo') + result = double + expect(QueryExclusion).to receive(:exclude?).with(execution.sql).and_return(result) + expect(execution.excluded?).to eq(result) + end + end +end diff --git a/notifier/spec/query_spec.rb b/notifier/spec/query_spec.rb index 2a7128bf244962e70922ea125fd0b8a76ee28786..fb87aa61ac6265628609aa5b3a4a3d175772e29b 100644 --- a/notifier/spec/query_spec.rb +++ b/notifier/spec/query_spec.rb @@ -20,34 +20,10 @@ RSpec.describe Query do end describe 'excluded?' do - it 'is true if query is on the excluded list' do - subject.query = described_class::EXCLUSIONS.first - - expect(subject.excluded?).to be true - end - - it 'is true if query includes /* pgssignore /*' do - subject.query = "#{subject.query} /* pgssignore */" - - expect(subject.excluded?).to be true - end - - it 'is true if query table is ar_internal_metadata' do - subject.query = "SELECT * from ar_internal_metadata;" - - expect(subject.excluded?).to be true - end - - it 'is true if query table starts with pg_' do - subject.query = "SELECT * from pg_internal_nonsense;" - - expect(subject.excluded?).to be true - end - - it 'is false for other queries' do - subject.query = "SELECT * from user;" - - expect(subject.excluded?).to be false + it 'delegates to QueryExclusion' do + result = double + expect(QueryExclusion).to receive(:exclude?).with(subject.query).and_return(result) + expect(subject.excluded?).to eq(result) end end @@ -103,7 +79,7 @@ RSpec.describe Query do end it 'parse it successfully' do - expect { query.formatted_query }.not_to raise_error PgQuery::ParseError + expect { query.formatted_query }.not_to raise_error end end end