Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 10a082b

Browse files
anarazelanarazel
committed
Add tests for pg_stat_io
Author: Melanie Plageman <melanieplageman@gmail.com> Reviewed-by: Andres Freund <andres@anarazel.de> Reviewed-by: Justin Pryzby <pryzby@telsasoft.com> Discussion: https://postgr.es/m/20200124195226.lth52iydq2n2uilq@alap3.anarazel.de
1 parent 9c83bbc commit 10a082b

File tree

4 files changed

+431
-0
lines changed

4 files changed

+431
-0
lines changed

‎contrib/amcheck/expected/check_heap.out

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,22 @@ SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'ALL-VISIBLE');
6666
INSERT INTO heaptest (a, b)
6767
(SELECT gs, repeat('x', gs)
6868
FROM generate_series(1,50) gs);
69+
-- pg_stat_io test:
70+
-- verify_heapam always uses a BAS_BULKREAD BufferAccessStrategy, whereas a
71+
-- sequential scan does so only if the table is large enough when compared to
72+
-- shared buffers (see initscan()). CREATE DATABASE ... also unconditionally
73+
-- uses a BAS_BULKREAD strategy, but we have chosen to use a tablespace and
74+
-- verify_heapam to provide coverage instead of adding another expensive
75+
-- operation to the main regression test suite.
76+
--
77+
-- Create an alternative tablespace and move the heaptest table to it, causing
78+
-- it to be rewritten and all the blocks to reliably evicted from shared
79+
-- buffers -- guaranteeing actual reads when we next select from it.
80+
SET allow_in_place_tablespaces = true;
81+
CREATE TABLESPACE regress_test_stats_tblspc LOCATION '';
82+
SELECT sum(reads) AS stats_bulkreads_before
83+
FROM pg_stat_io WHERE io_context = 'bulkread' \gset
84+
ALTER TABLE heaptest SET TABLESPACE regress_test_stats_tblspc;
6985
-- Check that valid options are not rejected nor corruption reported
7086
-- for a non-empty table
7187
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none');
@@ -88,6 +104,23 @@ SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock :=
88104
-------+--------+--------+-----
89105
(0 rows)
90106

107+
-- verify_heapam should have read in the page written out by
108+
-- ALTER TABLE ... SET TABLESPACE ...
109+
-- causing an additional bulkread, which should be reflected in pg_stat_io.
110+
SELECT pg_stat_force_next_flush();
111+
pg_stat_force_next_flush
112+
--------------------------
113+
114+
(1 row)
115+
116+
SELECT sum(reads) AS stats_bulkreads_after
117+
FROM pg_stat_io WHERE io_context = 'bulkread' \gset
118+
SELECT :stats_bulkreads_after > :stats_bulkreads_before;
119+
?column?
120+
----------
121+
t
122+
(1 row)
123+
91124
CREATE ROLE regress_heaptest_role;
92125
-- verify permissions are checked (error due to function not callable)
93126
SET ROLE regress_heaptest_role;
@@ -195,6 +228,7 @@ ERROR: cannot check relation "test_foreign_table"
195228
DETAIL: This operation is not supported for foreign tables.
196229
-- cleanup
197230
DROP TABLE heaptest;
231+
DROP TABLESPACE regress_test_stats_tblspc;
198232
DROP TABLE test_partition;
199233
DROP TABLE test_partitioned;
200234
DROP OWNED BY regress_heaptest_role; -- permissions

‎contrib/amcheck/sql/check_heap.sql

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,44 @@ SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'NONE');
2020
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'ALL-FROZEN');
2121
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'ALL-VISIBLE');
2222

23+
2324
-- Add some data so subsequent tests are not entirely trivial
2425
INSERT INTO heaptest (a, b)
2526
(SELECT gs, repeat('x', gs)
2627
FROM generate_series(1,50) gs);
2728

29+
-- pg_stat_io test:
30+
-- verify_heapam always uses a BAS_BULKREAD BufferAccessStrategy, whereas a
31+
-- sequential scan does so only if the table is large enough when compared to
32+
-- shared buffers (see initscan()). CREATE DATABASE ... also unconditionally
33+
-- uses a BAS_BULKREAD strategy, but we have chosen to use a tablespace and
34+
-- verify_heapam to provide coverage instead of adding another expensive
35+
-- operation to the main regression test suite.
36+
--
37+
-- Create an alternative tablespace and move the heaptest table to it, causing
38+
-- it to be rewritten and all the blocks to reliably evicted from shared
39+
-- buffers -- guaranteeing actual reads when we next select from it.
40+
SET allow_in_place_tablespaces = true;
41+
CREATE TABLESPACE regress_test_stats_tblspc LOCATION '';
42+
SELECT sum(reads) AS stats_bulkreads_before
43+
FROM pg_stat_io WHERE io_context = 'bulkread' \gset
44+
ALTER TABLE heaptest SET TABLESPACE regress_test_stats_tblspc;
45+
2846
-- Check that valid options are not rejected nor corruption reported
2947
-- for a non-empty table
3048
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none');
3149
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen');
3250
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible');
3351
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0);
3452

53+
-- verify_heapam should have read in the page written out by
54+
-- ALTER TABLE ... SET TABLESPACE ...
55+
-- causing an additional bulkread, which should be reflected in pg_stat_io.
56+
SELECT pg_stat_force_next_flush();
57+
SELECT sum(reads) AS stats_bulkreads_after
58+
FROM pg_stat_io WHERE io_context = 'bulkread' \gset
59+
SELECT :stats_bulkreads_after > :stats_bulkreads_before;
60+
3561
CREATE ROLE regress_heaptest_role;
3662

3763
-- verify permissions are checked (error due to function not callable)
@@ -110,6 +136,7 @@ SELECT * FROM verify_heapam('test_foreign_table',
110136

111137
-- cleanup
112138
DROP TABLE heaptest;
139+
DROP TABLESPACE regress_test_stats_tblspc;
113140
DROP TABLE test_partition;
114141
DROP TABLE test_partitioned;
115142
DROP OWNED BY regress_heaptest_role; -- permissions

‎src/test/regress/expected/stats.out

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1126,4 +1126,232 @@ SELECT pg_stat_get_subscription_stats(NULL);
11261126

11271127
(1 row)
11281128

1129+
-- Test that the following operations are tracked in pg_stat_io:
1130+
-- - reads of target blocks into shared buffers
1131+
-- - writes of shared buffers to permanent storage
1132+
-- - extends of relations using shared buffers
1133+
-- - fsyncs done to ensure the durability of data dirtying shared buffers
1134+
-- There is no test for blocks evicted from shared buffers, because we cannot
1135+
-- be sure of the state of shared buffers at the point the test is run.
1136+
-- Create a regular table and insert some data to generate IOCONTEXT_NORMAL
1137+
-- extends.
1138+
SELECT sum(extends) AS io_sum_shared_before_extends
1139+
FROM pg_stat_io WHERE io_context = 'normal' AND io_object = 'relation' \gset
1140+
CREATE TABLE test_io_shared(a int);
1141+
INSERT INTO test_io_shared SELECT i FROM generate_series(1,100)i;
1142+
SELECT pg_stat_force_next_flush();
1143+
pg_stat_force_next_flush
1144+
--------------------------
1145+
1146+
(1 row)
1147+
1148+
SELECT sum(extends) AS io_sum_shared_after_extends
1149+
FROM pg_stat_io WHERE io_context = 'normal' AND io_object = 'relation' \gset
1150+
SELECT :io_sum_shared_after_extends > :io_sum_shared_before_extends;
1151+
?column?
1152+
----------
1153+
t
1154+
(1 row)
1155+
1156+
-- After a checkpoint, there should be some additional IOCONTEXT_NORMAL writes
1157+
-- and fsyncs.
1158+
SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs
1159+
FROM pg_stat_io
1160+
WHERE io_context = 'normal' AND io_object = 'relation' \gset io_sum_shared_before_
1161+
-- See comment above for rationale for two explicit CHECKPOINTs.
1162+
CHECKPOINT;
1163+
CHECKPOINT;
1164+
SELECT sum(writes) AS writes, sum(fsyncs) AS fsyncs
1165+
FROM pg_stat_io
1166+
WHERE io_context = 'normal' AND io_object = 'relation' \gset io_sum_shared_after_
1167+
SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes;
1168+
?column?
1169+
----------
1170+
t
1171+
(1 row)
1172+
1173+
SELECT current_setting('fsync') = 'off'
1174+
OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs;
1175+
?column?
1176+
----------
1177+
t
1178+
(1 row)
1179+
1180+
-- Change the tablespace so that the table is rewritten directly, then SELECT
1181+
-- from it to cause it to be read back into shared buffers.
1182+
SELECT sum(reads) AS io_sum_shared_before_reads
1183+
FROM pg_stat_io WHERE io_context = 'normal' AND io_object = 'relation' \gset
1184+
ALTER TABLE test_io_shared SET TABLESPACE regress_tblspace;
1185+
-- SELECT from the table so that the data is read into shared buffers and
1186+
-- io_context 'normal', io_object 'relation' reads are counted.
1187+
SELECT COUNT(*) FROM test_io_shared;
1188+
count
1189+
-------
1190+
100
1191+
(1 row)
1192+
1193+
SELECT pg_stat_force_next_flush();
1194+
pg_stat_force_next_flush
1195+
--------------------------
1196+
1197+
(1 row)
1198+
1199+
SELECT sum(reads) AS io_sum_shared_after_reads
1200+
FROM pg_stat_io WHERE io_context = 'normal' AND io_object = 'relation' \gset
1201+
SELECT :io_sum_shared_after_reads > :io_sum_shared_before_reads;
1202+
?column?
1203+
----------
1204+
t
1205+
(1 row)
1206+
1207+
DROP TABLE test_io_shared;
1208+
-- Test that the follow IOCONTEXT_LOCAL IOOps are tracked in pg_stat_io:
1209+
-- - eviction of local buffers in order to reuse them
1210+
-- - reads of temporary table blocks into local buffers
1211+
-- - writes of local buffers to permanent storage
1212+
-- - extends of temporary tables
1213+
-- Set temp_buffers to its minimum so that we can trigger writes with fewer
1214+
-- inserted tuples. Do so in a new session in case temporary tables have been
1215+
-- accessed by previous tests in this session.
1216+
\c
1217+
SET temp_buffers TO 100;
1218+
CREATE TEMPORARY TABLE test_io_local(a int, b TEXT);
1219+
SELECT sum(extends) AS extends, sum(evictions) AS evictions, sum(writes) AS writes
1220+
FROM pg_stat_io
1221+
WHERE io_context = 'normal' AND io_object = 'temp relation' \gset io_sum_local_before_
1222+
-- Insert tuples into the temporary table, generating extends in the stats.
1223+
-- Insert enough values that we need to reuse and write out dirty local
1224+
-- buffers, generating evictions and writes.
1225+
INSERT INTO test_io_local SELECT generate_series(1, 5000) as id, repeat('a', 200);
1226+
-- Ensure the table is large enough to exceed our temp_buffers setting.
1227+
SELECT pg_relation_size('test_io_local') / current_setting('block_size')::int8 > 100;
1228+
?column?
1229+
----------
1230+
t
1231+
(1 row)
1232+
1233+
SELECT sum(reads) AS io_sum_local_before_reads
1234+
FROM pg_stat_io WHERE io_context = 'normal' AND io_object = 'temp relation' \gset
1235+
-- Read in evicted buffers, generating reads.
1236+
SELECT COUNT(*) FROM test_io_local;
1237+
count
1238+
-------
1239+
5000
1240+
(1 row)
1241+
1242+
SELECT pg_stat_force_next_flush();
1243+
pg_stat_force_next_flush
1244+
--------------------------
1245+
1246+
(1 row)
1247+
1248+
SELECT sum(evictions) AS evictions,
1249+
sum(reads) AS reads,
1250+
sum(writes) AS writes,
1251+
sum(extends) AS extends
1252+
FROM pg_stat_io
1253+
WHERE io_context = 'normal' AND io_object = 'temp relation' \gset io_sum_local_after_
1254+
SELECT :io_sum_local_after_evictions > :io_sum_local_before_evictions,
1255+
:io_sum_local_after_reads > :io_sum_local_before_reads,
1256+
:io_sum_local_after_writes > :io_sum_local_before_writes,
1257+
:io_sum_local_after_extends > :io_sum_local_before_extends;
1258+
?column? | ?column? | ?column? | ?column?
1259+
----------+----------+----------+----------
1260+
t | t | t | t
1261+
(1 row)
1262+
1263+
-- Change the tablespaces so that the temporary table is rewritten to other
1264+
-- local buffers, exercising a different codepath than standard local buffer
1265+
-- writes.
1266+
ALTER TABLE test_io_local SET TABLESPACE regress_tblspace;
1267+
SELECT pg_stat_force_next_flush();
1268+
pg_stat_force_next_flush
1269+
--------------------------
1270+
1271+
(1 row)
1272+
1273+
SELECT sum(writes) AS io_sum_local_new_tblspc_writes
1274+
FROM pg_stat_io WHERE io_context = 'normal' AND io_object = 'temp relation' \gset
1275+
SELECT :io_sum_local_new_tblspc_writes > :io_sum_local_after_writes;
1276+
?column?
1277+
----------
1278+
t
1279+
(1 row)
1280+
1281+
RESET temp_buffers;
1282+
-- Test that reuse of strategy buffers and reads of blocks into these reused
1283+
-- buffers while VACUUMing are tracked in pg_stat_io.
1284+
-- Set wal_skip_threshold smaller than the expected size of
1285+
-- test_io_vac_strategy so that, even if wal_level is minimal, VACUUM FULL will
1286+
-- fsync the newly rewritten test_io_vac_strategy instead of writing it to WAL.
1287+
-- Writing it to WAL will result in the newly written relation pages being in
1288+
-- shared buffers -- preventing us from testing BAS_VACUUM BufferAccessStrategy
1289+
-- reads.
1290+
SET wal_skip_threshold = '1 kB';
1291+
SELECT sum(reuses) AS reuses, sum(reads) AS reads
1292+
FROM pg_stat_io WHERE io_context = 'vacuum' \gset io_sum_vac_strategy_before_
1293+
CREATE TABLE test_io_vac_strategy(a int, b int) WITH (autovacuum_enabled = 'false');
1294+
INSERT INTO test_io_vac_strategy SELECT i, i from generate_series(1, 8000)i;
1295+
-- Ensure that the next VACUUM will need to perform IO by rewriting the table
1296+
-- first with VACUUM (FULL).
1297+
VACUUM (FULL) test_io_vac_strategy;
1298+
VACUUM (PARALLEL 0) test_io_vac_strategy;
1299+
SELECT pg_stat_force_next_flush();
1300+
pg_stat_force_next_flush
1301+
--------------------------
1302+
1303+
(1 row)
1304+
1305+
SELECT sum(reuses) AS reuses, sum(reads) AS reads
1306+
FROM pg_stat_io WHERE io_context = 'vacuum' \gset io_sum_vac_strategy_after_
1307+
SELECT :io_sum_vac_strategy_after_reads > :io_sum_vac_strategy_before_reads,
1308+
:io_sum_vac_strategy_after_reuses > :io_sum_vac_strategy_before_reuses;
1309+
?column? | ?column?
1310+
----------+----------
1311+
t | t
1312+
(1 row)
1313+
1314+
RESET wal_skip_threshold;
1315+
-- Test that extends done by a CTAS, which uses a BAS_BULKWRITE
1316+
-- BufferAccessStrategy, are tracked in pg_stat_io.
1317+
SELECT sum(extends) AS io_sum_bulkwrite_strategy_extends_before
1318+
FROM pg_stat_io WHERE io_context = 'bulkwrite' \gset
1319+
CREATE TABLE test_io_bulkwrite_strategy AS SELECT i FROM generate_series(1,100)i;
1320+
SELECT pg_stat_force_next_flush();
1321+
pg_stat_force_next_flush
1322+
--------------------------
1323+
1324+
(1 row)
1325+
1326+
SELECT sum(extends) AS io_sum_bulkwrite_strategy_extends_after
1327+
FROM pg_stat_io WHERE io_context = 'bulkwrite' \gset
1328+
SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_extends_before;
1329+
?column?
1330+
----------
1331+
t
1332+
(1 row)
1333+
1334+
-- Test IO stats reset
1335+
SELECT pg_stat_have_stats('io', 0, 0);
1336+
pg_stat_have_stats
1337+
--------------------
1338+
t
1339+
(1 row)
1340+
1341+
SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) AS io_stats_pre_reset
1342+
FROM pg_stat_io \gset
1343+
SELECT pg_stat_reset_shared('io');
1344+
pg_stat_reset_shared
1345+
----------------------
1346+
1347+
(1 row)
1348+
1349+
SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) AS io_stats_post_reset
1350+
FROM pg_stat_io \gset
1351+
SELECT :io_stats_post_reset < :io_stats_pre_reset;
1352+
?column?
1353+
----------
1354+
t
1355+
(1 row)
1356+
11291357
-- End of Stats Test

0 commit comments

Comments
 (0)