Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit e255b64

Browse files
MasahikoSawadaMasahikoSawada
committed
Add tests for XID wraparound.
The test module includes helper functions to quickly burn through lots of XIDs. They are used in the tests, and are also handy for manually testing XID wraparound. Since these tests are very expensive the entire suite is disabled by default. It requires to set PG_TEST_EXTRA to run it. Reviewed-by: Daniel Gustafsson, John Naylor, Michael Paquier Reviewed-by: vignesh C Author: Heikki Linnakangas, Masahiko Sawada, Andres Freund Discussion: https://www.postgresql.org/message-id/CAD21AoDVhkXp8HjpFO-gp3TgL6tCKcZQNxn04m01VAtcSi-5sA%40mail.gmail.com
1 parent a243569 commit e255b64

13 files changed

+644
-1
lines changed

‎doc/src/sgml/regress.sgml

+10
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,16 @@ make check-world PG_TEST_EXTRA='kerberos ldap ssl load_balance'
314314
</para>
315315
</listitem>
316316
</varlistentry>
317+
318+
<varlistentry>
319+
<term><literal>xid_wraparound</literal></term>
320+
<listitem>
321+
<para>
322+
Runs the test suite under <filename>src/test/module/xid_wrapround</filename>.
323+
Not enabled by default because it is resource intensive.
324+
</para>
325+
</listitem>
326+
</varlistentry>
317327
</variablelist>
318328

319329
Tests for features that are not supported by the current build

‎src/test/modules/Makefile

+2-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ SUBDIRS = \
3434
test_shm_mq \
3535
test_slru \
3636
unsafe_tests \
37-
worker_spi
37+
worker_spi \
38+
xid_wraparound
3839

3940
ifeq ($(with_ssl),openssl)
4041
SUBDIRS += ssl_passphrase_callback

‎src/test/modules/meson.build

+1
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,4 @@ subdir('test_shm_mq')
3232
subdir('test_slru')
3333
subdir('unsafe_tests')
3434
subdir('worker_spi')
35+
subdir('xid_wraparound')

‎src/test/modules/xid_wraparound/.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Generated subdirectories
2+
/log/
3+
/results/
4+
/tmp_check/

‎src/test/modules/xid_wraparound/Makefile

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# src/test/modules/xid_wraparound/Makefile
2+
3+
MODULE_big = xid_wraparound
4+
OBJS = \
5+
$(WIN32RES) \
6+
xid_wraparound.o
7+
PGFILEDESC = "xid_wraparound - tests for XID wraparound"
8+
9+
EXTENSION = xid_wraparound
10+
DATA = xid_wraparound--1.0.sql
11+
12+
TAP_TESTS = 1
13+
14+
ifdef USE_PGXS
15+
PG_CONFIG = pg_config
16+
PGXS := $(shell $(PG_CONFIG) --pgxs)
17+
include $(PGXS)
18+
else
19+
subdir = src/test/modules/xid_wraparound
20+
top_builddir = ../../../..
21+
include $(top_builddir)/src/Makefile.global
22+
include $(top_srcdir)/contrib/contrib-global.mk
23+
endif

‎src/test/modules/xid_wraparound/README

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
This module contains tests for XID wraparound. The tests use two
2+
helper functions to quickly consume lots of XIDs, to reach XID
3+
wraparound faster.

‎src/test/modules/xid_wraparound/meson.build

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Copyright (c) 2023, PostgreSQL Global Development Group
2+
3+
xid_wraparound_sources = files(
4+
'xid_wraparound.c',
5+
)
6+
7+
if host_system == 'windows'
8+
xid_wraparound_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
9+
'--NAME', 'xid_wraparound',
10+
'--FILEDESC', 'xid_wraparound - tests for XID wraparound',])
11+
endif
12+
13+
xid_wraparound = shared_module('xid_wraparound',
14+
xid_wraparound_sources,
15+
kwargs: pg_mod_args,
16+
)
17+
testprep_targets += xid_wraparound
18+
19+
install_data(
20+
'xid_wraparound.control',
21+
'xid_wraparound--1.0.sql',
22+
kwargs: contrib_data_args,
23+
)
24+
25+
tests += {
26+
'name': 'xid_wraparound',
27+
'sd': meson.current_source_dir(),
28+
'bd': meson.current_build_dir(),
29+
'tap': {
30+
'tests': [
31+
't/001_emergency_vacuum.pl',
32+
't/002_limits.pl',
33+
't/003_wraparounds.pl',
34+
],
35+
},
36+
}

‎src/test/modules/xid_wraparound/t/001_emergency_vacuum.pl

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
# Copyright (c) 2023, PostgreSQL Global Development Group
2+
3+
# Test wraparound emergency autovacuum.
4+
use strict;
5+
use warnings;
6+
use PostgreSQL::Test::Cluster;
7+
use PostgreSQL::Test::Utils;
8+
use Test::More;
9+
10+
if ($ENV{PG_TEST_EXTRA} !~ /\bxid_wraparound\b/)
11+
{
12+
plan skip_all => "test xid_wraparound not enabled in PG_TEST_EXTRA";
13+
}
14+
15+
# Initialize node
16+
my $node = PostgreSQL::Test::Cluster->new('main');
17+
18+
$node->init;
19+
$node->append_conf(
20+
'postgresql.conf', qq[
21+
autovacuum = off # run autovacuum only when to anti wraparound
22+
autovacuum_naptime = 1s
23+
# so it's easier to verify the order of operations
24+
autovacuum_max_workers = 1
25+
log_autovacuum_min_duration = 0
26+
]);
27+
$node->start;
28+
$node->safe_psql('postgres', 'CREATE EXTENSION xid_wraparound');
29+
30+
# Create tables for a few different test scenarios
31+
$node->safe_psql(
32+
'postgres', qq[
33+
CREATE TABLE large(id serial primary key, data text, filler text default repeat(random()::text, 10));
34+
INSERT INTO large(data) SELECT generate_series(1,30000);
35+
36+
CREATE TABLE large_trunc(id serial primary key, data text, filler text default repeat(random()::text, 10));
37+
INSERT INTO large_trunc(data) SELECT generate_series(1,30000);
38+
39+
CREATE TABLE small(id serial primary key, data text, filler text default repeat(random()::text, 10));
40+
INSERT INTO small(data) SELECT generate_series(1,15000);
41+
42+
CREATE TABLE small_trunc(id serial primary key, data text, filler text default repeat(random()::text, 10));
43+
INSERT INTO small_trunc(data) SELECT generate_series(1,15000);
44+
45+
CREATE TABLE autovacuum_disabled(id serial primary key, data text) WITH (autovacuum_enabled=false);
46+
INSERT INTO autovacuum_disabled(data) SELECT generate_series(1,1000);
47+
]);
48+
49+
# Bump the query timeout to avoid false negatives on slow test systems.
50+
my $psql_timeout_secs = 4 * $PostgreSQL::Test::Utils::timeout_default;
51+
52+
# Start a background session, which holds a transaction open, preventing
53+
# autovacuum from advancing relfrozenxid and datfrozenxid.
54+
my $background_psql = $node->background_psql(
55+
'postgres',
56+
on_error_stop => 0,
57+
timeout => $psql_timeout_secs);
58+
$background_psql->set_query_timer_restart();
59+
$background_psql->query_safe(
60+
qq[
61+
BEGIN;
62+
DELETE FROM large WHERE id % 2 = 0;
63+
DELETE FROM large_trunc WHERE id > 10000;
64+
DELETE FROM small WHERE id % 2 = 0;
65+
DELETE FROM small_trunc WHERE id > 1000;
66+
DELETE FROM autovacuum_disabled WHERE id % 2 = 0;
67+
]);
68+
69+
# Consume 2 billion XIDs, to get us very close to wraparound
70+
$node->safe_psql('postgres',
71+
qq[SELECT consume_xids_until('2000000000'::xid8)]);
72+
73+
# Make sure the latest completed XID is advanced
74+
$node->safe_psql('postgres', qq[INSERT INTO small(data) SELECT 1]);
75+
76+
# Check that all databases became old enough to trigger failsafe.
77+
my $ret = $node->safe_psql(
78+
'postgres',
79+
qq[
80+
SELECT datname,
81+
age(datfrozenxid) > current_setting('vacuum_failsafe_age')::int as old
82+
FROM pg_database ORDER BY 1
83+
]);
84+
is( $ret, "postgres|t
85+
template0|t
86+
template1|t", "all tables became old");
87+
88+
my $log_offset = -s $node->logfile;
89+
90+
# Finish the old transaction, to allow vacuum freezing to advance
91+
# relfrozenxid and datfrozenxid again.
92+
$background_psql->query_safe(qq[COMMIT]);
93+
$background_psql->quit;
94+
95+
# Wait until autovacuum processed all tables and advanced the
96+
# system-wide oldest-XID.
97+
$node->poll_query_until(
98+
'postgres', qq[
99+
SELECT NOT EXISTS (
100+
SELECT *
101+
FROM pg_database
102+
WHERE age(datfrozenxid) > current_setting('autovacuum_freeze_max_age')::int)
103+
]) or die "timeout waiting for all databases to be vacuumed";
104+
105+
# Check if these tables are vacuumed.
106+
$ret = $node->safe_psql(
107+
'postgres', qq[
108+
SELECT relname, age(relfrozenxid) > current_setting('autovacuum_freeze_max_age')::int
109+
FROM pg_class
110+
WHERE relname IN ('large', 'large_trunc', 'small', 'small_trunc', 'autovacuum_disabled')
111+
ORDER BY 1
112+
]);
113+
114+
is( $ret, "autovacuum_disabled|f
115+
large|f
116+
large_trunc|f
117+
small|f
118+
small_trunc|f", "all tables are vacuumed");
119+
120+
# Check if vacuum failsafe was triggered for each table.
121+
my $log_contents = slurp_file($node->logfile, $log_offset);
122+
foreach my $tablename ('large', 'large_trunc', 'small', 'small_trunc',
123+
'autovacuum_disabled')
124+
{
125+
like(
126+
$log_contents,
127+
qr/bypassing nonessential maintenance of table "postgres.public.$tablename" as a failsafe after \d+ index scans/,
128+
"failsafe vacuum triggered for $tablename");
129+
}
130+
131+
$node->stop;
132+
done_testing();

‎src/test/modules/xid_wraparound/t/002_limits.pl

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
# Copyright (c) 2023, PostgreSQL Global Development Group
2+
#
3+
# Test XID wraparound limits.
4+
#
5+
# When you get close to XID wraparound, you start to get warnings, and
6+
# when you get even closer, the system refuses to assign any more XIDs
7+
# until the oldest databases have been vacuumed and datfrozenxid has
8+
# been advanced.
9+
10+
use strict;
11+
use warnings;
12+
use PostgreSQL::Test::Cluster;
13+
use PostgreSQL::Test::Utils;
14+
use Test::More;
15+
use Time::HiRes qw(usleep);
16+
17+
if ($ENV{PG_TEST_EXTRA} !~ /\bxid_wraparound\b/)
18+
{
19+
plan skip_all => "test xid_wraparound not enabled in PG_TEST_EXTRA";
20+
}
21+
22+
my $ret;
23+
24+
# Initialize node
25+
my $node = PostgreSQL::Test::Cluster->new('wraparound');
26+
27+
$node->init;
28+
$node->append_conf(
29+
'postgresql.conf', qq[
30+
autovacuum = off # run autovacuum only to prevent wraparound
31+
autovacuum_naptime = 1s
32+
log_autovacuum_min_duration = 0
33+
]);
34+
$node->start;
35+
$node->safe_psql('postgres', 'CREATE EXTENSION xid_wraparound');
36+
37+
# Create a test table
38+
$node->safe_psql(
39+
'postgres', qq[
40+
CREATE TABLE wraparoundtest(t text);
41+
INSERT INTO wraparoundtest VALUES ('start');
42+
]);
43+
44+
# Bump the query timeout to avoid false negatives on slow test systems.
45+
my $psql_timeout_secs = 4 * $PostgreSQL::Test::Utils::timeout_default;
46+
47+
# Start a background session, which holds a transaction open, preventing
48+
# autovacuum from advancing relfrozenxid and datfrozenxid.
49+
my $background_psql = $node->background_psql(
50+
'postgres',
51+
on_error_stop => 0,
52+
timeout => $psql_timeout_secs);
53+
$background_psql->query_safe(
54+
qq[
55+
BEGIN;
56+
INSERT INTO wraparoundtest VALUES ('oldxact');
57+
]);
58+
59+
# Consume 2 billion transactions, to get close to wraparound
60+
$node->safe_psql('postgres', qq[SELECT consume_xids(1000000000)]);
61+
$node->safe_psql('postgres',
62+
qq[INSERT INTO wraparoundtest VALUES ('after 1 billion')]);
63+
64+
$node->safe_psql('postgres', qq[SELECT consume_xids(1000000000)]);
65+
$node->safe_psql('postgres',
66+
qq[INSERT INTO wraparoundtest VALUES ('after 2 billion')]);
67+
68+
# We are now just under 150 million XIDs away from wraparound.
69+
# Continue consuming XIDs, in batches of 10 million, until we get
70+
# the warning:
71+
#
72+
# WARNING: database "postgres" must be vacuumed within 3000024 transactions
73+
# HINT: To avoid a database shutdown, execute a database-wide VACUUM in that database.
74+
# You might also need to commit or roll back old prepared transactions, or drop stale replication slots.
75+
my $stderr;
76+
my $warn_limit = 0;
77+
for my $i (1 .. 15)
78+
{
79+
$node->psql(
80+
'postgres', qq[SELECT consume_xids(10000000)],
81+
stderr => \$stderr,
82+
on_error_die => 1);
83+
84+
if ($stderr =~
85+
/WARNING: database "postgres" must be vacuumed within [0-9]+ transactions/
86+
)
87+
{
88+
# Reached the warn-limit
89+
$warn_limit = 1;
90+
last;
91+
}
92+
}
93+
ok($warn_limit == 1, "warn-limit reached");
94+
95+
# We can still INSERT, despite the warnings.
96+
$node->safe_psql('postgres',
97+
qq[INSERT INTO wraparoundtest VALUES ('reached warn-limit')]);
98+
99+
# Keep going. We'll hit the hard "stop" limit.
100+
$ret = $node->psql(
101+
'postgres',
102+
qq[SELECT consume_xids(100000000)],
103+
stderr => \$stderr);
104+
like(
105+
$stderr,
106+
qr/ERROR: database is not accepting commands that assign new XIDs to avoid wraparound data loss in database "postgres"/,
107+
"stop-limit");
108+
109+
# Finish the old transaction, to allow vacuum freezing to advance
110+
# relfrozenxid and datfrozenxid again.
111+
$background_psql->query_safe(qq[COMMIT]);
112+
$background_psql->quit;
113+
114+
# VACUUM, to freeze the tables and advance datfrozenxid.
115+
#
116+
# Autovacuum does this for the other databases, and would do it for
117+
# 'postgres' too, but let's test manual VACUUM.
118+
#
119+
$node->safe_psql('postgres', 'VACUUM');
120+
121+
# Wait until autovacuum has processed the other databases and advanced
122+
# the system-wide oldest-XID.
123+
$ret =
124+
$node->poll_query_until('postgres',
125+
qq[INSERT INTO wraparoundtest VALUES ('after VACUUM')],
126+
'INSERT 0 1');
127+
128+
# Check the table contents
129+
$ret = $node->safe_psql('postgres', qq[SELECT * from wraparoundtest]);
130+
is( $ret, "start
131+
oldxact
132+
after 1 billion
133+
after 2 billion
134+
reached warn-limit
135+
after VACUUM");
136+
137+
$node->stop;
138+
done_testing();

0 commit comments

Comments
 (0)