Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 87198ce

Browse files
tristan957Commitfest Bot
tristan957
authored and
Commitfest Bot
committed
Add contrib/fsync_checker
fsync_checker is an extension which overrides the global storage manager to check for volatile relations, those which have been written but not synced to disk.
1 parent f13653e commit 87198ce

File tree

6 files changed

+280
-0
lines changed

6 files changed

+280
-0
lines changed

‎contrib/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ SUBDIRS = \
1919
dict_int \
2020
dict_xsyn \
2121
earthdistance \
22+
fsync_checker \
2223
file_fdw \
2324
fuzzystrmatch \
2425
hstore \

‎contrib/fsync_checker/fsync_checker.control

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# fsync_checker extension
2+
comment = 'SMGR extension for checking volatile writes'
3+
default_version = '1.0'
4+
module_pathname = '$libdir/fsync_checker'
5+
relocatable = true

‎contrib/fsync_checker/fsync_checker_smgr.c

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
#include "postgres.h"
2+
3+
#include "access/xlog.h"
4+
#include "fmgr.h"
5+
#include "miscadmin.h"
6+
#include "storage/ipc.h"
7+
#include "storage/lwlock.h"
8+
#include "storage/shmem.h"
9+
#include "storage/smgr.h"
10+
#include "storage/md.h"
11+
#include "utils/hsearch.h"
12+
13+
PG_MODULE_MAGIC;
14+
15+
typedef struct
16+
{
17+
RelFileLocator locator;
18+
ForkNumber forknum;
19+
} VolatileRelnKey;
20+
21+
typedef struct
22+
{
23+
VolatileRelnKey key;
24+
XLogRecPtr lsn;
25+
} VolatileRelnEntry;
26+
27+
void _PG_init(void);
28+
29+
static void fsync_checker_extend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
30+
const void *buffer, bool skipFsync);
31+
static void fsync_checker_immedsync(SMgrRelation reln, ForkNumber forknum);
32+
static void fsync_checker_writev(SMgrRelation reln, ForkNumber forknum,
33+
BlockNumber blocknum, const void **buffers,
34+
BlockNumber nblocks, bool skipFsync);
35+
static void fsync_checker_writeback(SMgrRelation reln, ForkNumber forknum,
36+
BlockNumber blocknum, BlockNumber nblocks);
37+
static void fsync_checker_zeroextend(SMgrRelation reln, ForkNumber forknum,
38+
BlockNumber blocknum, int nblocks, bool skipFsync);
39+
40+
static void fsync_checker_checkpoint_create(const CheckPoint *checkPoint);
41+
static void fsync_checker_shmem_request(void);
42+
static void fsync_checker_shmem_startup(void);
43+
44+
static void add_reln(SMgrRelation reln, ForkNumber forknum);
45+
static void remove_reln(SMgrRelation reln, ForkNumber forknum);
46+
47+
static SMgrId fsync_checker_smgr_id;
48+
static const struct f_smgr fsync_checker_smgr = {
49+
.name = "fsync_checker",
50+
.smgr_init = mdinit,
51+
.smgr_shutdown = NULL,
52+
.smgr_open = mdopen,
53+
.smgr_close = mdclose,
54+
.smgr_create = mdcreate,
55+
.smgr_exists = mdexists,
56+
.smgr_unlink = mdunlink,
57+
.smgr_extend = fsync_checker_extend,
58+
.smgr_zeroextend = fsync_checker_zeroextend,
59+
.smgr_prefetch = mdprefetch,
60+
.smgr_maxcombine = mdmaxcombine,
61+
.smgr_readv = mdreadv,
62+
.smgr_writev = fsync_checker_writev,
63+
.smgr_writeback = fsync_checker_writeback,
64+
.smgr_nblocks = mdnblocks,
65+
.smgr_truncate = mdtruncate,
66+
.smgr_immedsync = fsync_checker_immedsync,
67+
.smgr_registersync = mdregistersync,
68+
};
69+
70+
static HTAB *volatile_relns;
71+
static LWLock *volatile_relns_lock;
72+
static shmem_request_hook_type prev_shmem_request_hook;
73+
static shmem_startup_hook_type prev_shmem_startup_hook;
74+
static checkpoint_create_hook_type prev_checkpoint_create_hook;
75+
76+
void
77+
_PG_init(void)
78+
{
79+
prev_checkpoint_create_hook = checkpoint_create_hook;
80+
checkpoint_create_hook = fsync_checker_checkpoint_create;
81+
82+
prev_shmem_request_hook = shmem_request_hook;
83+
shmem_request_hook = fsync_checker_shmem_request;
84+
85+
prev_shmem_startup_hook = shmem_startup_hook;
86+
shmem_startup_hook = fsync_checker_shmem_startup;
87+
88+
/*
89+
* Relation size of 0 means we can just defer to md, but it would be nice
90+
* to just expose this functionality, so if I needed my own relation, I
91+
* could use MdSmgrRelation as the parent.
92+
*/
93+
fsync_checker_smgr_id = smgr_register(&fsync_checker_smgr, 0);
94+
95+
storage_manager_id = fsync_checker_smgr_id;
96+
}
97+
98+
static void
99+
fsync_checker_checkpoint_create(const CheckPoint *checkPoint)
100+
{
101+
long num_entries;
102+
HASH_SEQ_STATUS status;
103+
VolatileRelnEntry *entry;
104+
105+
if (prev_checkpoint_create_hook)
106+
prev_checkpoint_create_hook(checkPoint);
107+
108+
LWLockAcquire(volatile_relns_lock, LW_EXCLUSIVE);
109+
110+
hash_seq_init(&status, volatile_relns);
111+
112+
num_entries = hash_get_num_entries(volatile_relns);
113+
elog(INFO, "Analyzing %ld volatile relations", num_entries);
114+
while ((entry = hash_seq_search(&status)))
115+
{
116+
if (entry->lsn < checkPoint->redo)
117+
{
118+
RelPathStr path;
119+
120+
path = relpathperm(entry->key.locator, entry->key.forknum);
121+
122+
elog(WARNING, "Relation not previously synced: %s", path.str);
123+
}
124+
}
125+
126+
LWLockRelease(volatile_relns_lock);
127+
}
128+
129+
static void
130+
fsync_checker_shmem_request(void)
131+
{
132+
if (prev_shmem_request_hook)
133+
prev_shmem_request_hook();
134+
135+
RequestAddinShmemSpace(hash_estimate_size(1024, sizeof(VolatileRelnEntry)));
136+
RequestNamedLWLockTranche("fsync_checker volatile relns lock", 1);
137+
}
138+
139+
static void
140+
fsync_checker_shmem_startup(void)
141+
{
142+
HASHCTL ctl;
143+
144+
if (prev_shmem_startup_hook)
145+
prev_shmem_startup_hook();
146+
147+
ctl.keysize = sizeof(VolatileRelnKey);
148+
ctl.entrysize = sizeof(VolatileRelnEntry);
149+
volatile_relns = NULL;
150+
volatile_relns_lock = NULL;
151+
152+
/*
153+
* Create or attach to the shared memory state, including hash table
154+
*/
155+
LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE);
156+
157+
volatile_relns = ShmemInitHash("fsync_checker volatile relns",
158+
1024, 1024, &ctl, HASH_BLOBS | HASH_ELEM);
159+
volatile_relns_lock = &GetNamedLWLockTranche("fsync_checker volatile relns lock")->lock;
160+
161+
LWLockRelease(AddinShmemInitLock);
162+
}
163+
164+
static void
165+
add_reln(SMgrRelation reln, ForkNumber forknum)
166+
{
167+
bool found;
168+
XLogRecPtr lsn;
169+
VolatileRelnKey key;
170+
VolatileRelnEntry *entry;
171+
172+
key.locator = reln->smgr_rlocator.locator;
173+
key.forknum = forknum;
174+
175+
lsn = GetXLogWriteRecPtr();
176+
177+
LWLockAcquire(volatile_relns_lock, LW_EXCLUSIVE);
178+
179+
entry = hash_search(volatile_relns, &key, HASH_ENTER, &found);
180+
if (!found)
181+
entry->lsn = lsn;
182+
183+
LWLockRelease(volatile_relns_lock);
184+
}
185+
186+
static void
187+
remove_reln(SMgrRelation reln, ForkNumber forknum)
188+
{
189+
VolatileRelnKey key;
190+
191+
key.locator = reln->smgr_rlocator.locator;
192+
key.forknum = forknum;
193+
194+
LWLockAcquire(volatile_relns_lock, LW_EXCLUSIVE);
195+
196+
hash_search(volatile_relns, &key, HASH_REMOVE, NULL);
197+
198+
LWLockRelease(volatile_relns_lock);
199+
}
200+
201+
static void
202+
fsync_checker_extend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
203+
const void *buffer, bool skipFsync)
204+
{
205+
if (!SmgrIsTemp(reln) && !skipFsync)
206+
add_reln(reln, forknum);
207+
208+
mdextend(reln, forknum, blocknum, buffer, skipFsync);
209+
}
210+
211+
static void
212+
fsync_checker_immedsync(SMgrRelation reln, ForkNumber forknum)
213+
{
214+
if (!SmgrIsTemp(reln))
215+
remove_reln(reln, forknum);
216+
217+
mdimmedsync(reln, forknum);
218+
}
219+
220+
static void
221+
fsync_checker_writev(SMgrRelation reln, ForkNumber forknum,
222+
BlockNumber blocknum, const void **buffers,
223+
BlockNumber nblocks, bool skipFsync)
224+
{
225+
if (!SmgrIsTemp(reln) && !skipFsync)
226+
add_reln(reln, forknum);
227+
228+
mdwritev(reln, forknum, blocknum, buffers, nblocks, skipFsync);
229+
}
230+
231+
static void
232+
fsync_checker_writeback(SMgrRelation reln, ForkNumber forknum,
233+
BlockNumber blocknum, BlockNumber nblocks)
234+
{
235+
if (!SmgrIsTemp(reln))
236+
remove_reln(reln, forknum);
237+
238+
mdwriteback(reln, forknum, blocknum, nblocks);
239+
}
240+
241+
static void
242+
fsync_checker_zeroextend(SMgrRelation reln, ForkNumber forknum,
243+
BlockNumber blocknum, int nblocks, bool skipFsync)
244+
{
245+
if (!SmgrIsTemp(reln) && !skipFsync)
246+
add_reln(reln, forknum);
247+
248+
mdzeroextend(reln, forknum, blocknum, nblocks, skipFsync);
249+
}

‎contrib/fsync_checker/meson.build

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Copyright (c) 2023, PostgreSQL Global Development Group
2+
3+
fsync_checker_sources = files(
4+
'fsync_checker_smgr.c',
5+
)
6+
7+
if host_system == 'windows'
8+
fsync_checker_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
9+
'--NAME', 'fsync_checker',
10+
'--FILEDESC', 'fsync_checker - SMGR extension for checking volatile relations',])
11+
endif
12+
13+
fsync_checker = shared_module('fsync_checker',
14+
fsync_checker_sources,
15+
kwargs: contrib_mod_args,
16+
)
17+
contrib_targets += fsync_checker
18+
19+
install_data(
20+
'fsync_checker.control',
21+
kwargs: contrib_data_args,
22+
)

‎contrib/meson.build

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ subdir('dict_int')
2828
subdir('dict_xsyn')
2929
subdir('earthdistance')
3030
subdir('file_fdw')
31+
subdir('fsync_checker')
3132
subdir('fuzzystrmatch')
3233
subdir('hstore')
3334
subdir('hstore_plperl')

‎src/tools/pgindent/typedefs.list

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3150,6 +3150,8 @@ ViewStmt
31503150
VirtualTransactionId
31513151
VirtualTupleTableSlot
31523152
VolatileFunctionStatus
3153+
VolatileRelnEntry
3154+
VolatileRelnKey
31533155
Vsrt
31543156
WAIT_ORDER
31553157
WALAvailability

0 commit comments

Comments
 (0)