Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 439f617

Browse files
committed
Add palloc_aligned() to allow aligned memory allocations
This introduces palloc_aligned() and MemoryContextAllocAligned() which allow callers to obtain memory which is allocated to the given size and also aligned to the specified alignment boundary. The alignment boundaries may be any power-of-2 value. Currently, the alignment is capped at 2^26, however, we don't expect values anything like that large. The primary expected use case is to align allocations to perhaps CPU cache line size or to maybe I/O page size. Certain use cases can benefit from having aligned memory by either having better performance or more predictable performance. The alignment is achieved by requesting 'alignto' additional bytes from the underlying allocator function and then aligning the address that is returned to the requested alignment. This obviously does waste some memory, so alignments should be kept as small as what is required. It's also important to note that these alignment bytes eat into the maximum allocation size. So something like: palloc_aligned(MaxAllocSize, 64, 0); will not work as we cannot request MaxAllocSize + 64 bytes. Additionally, because we're just requesting the requested size plus the alignment requirements from the given MemoryContext, if that context is the Slab allocator, then since slab can only provide chunks of the size that's specified when the slab context is created, then this is not going to work. Slab will generate an error to indicate that the requested size is not supported. The alignment that is requested in palloc_aligned() is stored along with the allocated memory. This allows the alignment to remain intact through repalloc() calls. Author: Andres Freund, David Rowley Reviewed-by: Maxim Orlov, Andres Freund, John Naylor Discussion: https://postgr.es/m/CAApHDvpxLPUMV1mhxs6g7GNwCP6Cs6hfnYQL5ffJQTuFAuxt8A%40mail.gmail.com
1 parent 701c881 commit 439f617

File tree

8 files changed

+308
-13
lines changed

8 files changed

+308
-13
lines changed

src/backend/utils/cache/catcache.c

+2-3
Original file line numberDiff line numberDiff line change
@@ -763,7 +763,6 @@ InitCatCache(int id,
763763
{
764764
CatCache *cp;
765765
MemoryContext oldcxt;
766-
size_t sz;
767766
int i;
768767

769768
/*
@@ -807,8 +806,8 @@ InitCatCache(int id,
807806
*
808807
* Note: we rely on zeroing to initialize all the dlist headers correctly
809808
*/
810-
sz = sizeof(CatCache) + PG_CACHE_LINE_SIZE;
811-
cp = (CatCache *) CACHELINEALIGN(palloc0(sz));
809+
cp = (CatCache *) palloc_aligned(sizeof(CatCache), PG_CACHE_LINE_SIZE,
810+
MCXT_ALLOC_ZERO);
812811
cp->cc_bucket = palloc0(nbuckets * sizeof(dlist_head));
813812

814813
/*

src/backend/utils/mmgr/Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ top_builddir = ../../../..
1313
include $(top_builddir)/src/Makefile.global
1414

1515
OBJS = \
16+
alignedalloc.o \
1617
aset.o \
1718
dsa.o \
1819
freepage.o \

src/backend/utils/mmgr/alignedalloc.c

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*-------------------------------------------------------------------------
2+
*
3+
* alignedalloc.c
4+
* Allocator functions to implement palloc_aligned
5+
*
6+
* This is not a fully-fledged MemoryContext type as there is no means to
7+
* create a MemoryContext of this type. The code here only serves to allow
8+
* operations such as pfree() and repalloc() to work correctly on a memory
9+
* chunk that was allocated by palloc_aligned().
10+
*
11+
* Portions Copyright (c) 2022, PostgreSQL Global Development Group
12+
*
13+
* IDENTIFICATION
14+
* src/backend/utils/mmgr/alignedalloc.c
15+
*
16+
*-------------------------------------------------------------------------
17+
*/
18+
19+
#include "postgres.h"
20+
21+
#include "utils/memdebug.h"
22+
#include "utils/memutils_memorychunk.h"
23+
24+
/*
25+
* AlignedAllocFree
26+
* Frees allocated memory; memory is removed from its owning context.
27+
*/
28+
void
29+
AlignedAllocFree(void *pointer)
30+
{
31+
MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
32+
void *unaligned;
33+
34+
Assert(!MemoryChunkIsExternal(chunk));
35+
36+
/* obtain the original (unaligned) allocated pointer */
37+
unaligned = MemoryChunkGetBlock(chunk);
38+
39+
#ifdef MEMORY_CONTEXT_CHECKING
40+
/* Test for someone scribbling on unused space in chunk */
41+
if (!sentinel_ok(pointer, chunk->requested_size))
42+
elog(WARNING, "detected write past chunk end in %s %p",
43+
GetMemoryChunkContext(unaligned)->name, chunk);
44+
#endif
45+
46+
pfree(unaligned);
47+
}
48+
49+
/*
50+
* AlignedAllocRealloc
51+
* Change the allocated size of a chunk and return possibly a different
52+
* pointer to a memory address aligned to the same boundary as the
53+
* originally requested alignment. The contents of 'pointer' will be
54+
* copied into the returned pointer up until 'size'. Any additional
55+
* memory will be uninitialized.
56+
*/
57+
void *
58+
AlignedAllocRealloc(void *pointer, Size size)
59+
{
60+
MemoryChunk *redirchunk = PointerGetMemoryChunk(pointer);
61+
Size alignto = MemoryChunkGetValue(redirchunk);
62+
void *unaligned = MemoryChunkGetBlock(redirchunk);
63+
MemoryContext ctx;
64+
Size old_size;
65+
void *newptr;
66+
67+
/* sanity check this is a power of 2 value */
68+
Assert((alignto & (alignto - 1)) == 0);
69+
70+
/*
71+
* Determine the size of the original allocation. We can't determine this
72+
* exactly as GetMemoryChunkSpace() returns the total space used for the
73+
* allocation, which for contexts like aset includes rounding up to the
74+
* next power of 2. However, this value is just used to memcpy() the old
75+
* data into the new allocation, so we only need to concern ourselves with
76+
* not reading beyond the end of the original allocation's memory. The
77+
* drawback here is that we may copy more bytes than we need to, which
78+
* only amounts to wasted effort. We can safely subtract the extra bytes
79+
* that we requested to allow us to align the pointer. We must also
80+
* subtract the space for the unaligned pointer's MemoryChunk since
81+
* GetMemoryChunkSpace should have included that. This does assume that
82+
* all context types use MemoryChunk as a chunk header.
83+
*/
84+
old_size = GetMemoryChunkSpace(unaligned) -
85+
PallocAlignedExtraBytes(alignto) - sizeof(MemoryChunk);
86+
87+
#ifdef MEMORY_CONTEXT_CHECKING
88+
/* check that GetMemoryChunkSpace returned something realistic */
89+
Assert(old_size >= redirchunk->requested_size);
90+
#endif
91+
92+
ctx = GetMemoryChunkContext(unaligned);
93+
newptr = MemoryContextAllocAligned(ctx, size, alignto, 0);
94+
95+
/*
96+
* We may memcpy beyond the end of the original allocation request size,
97+
* so we must mark the entire allocation as defined.
98+
*/
99+
VALGRIND_MAKE_MEM_DEFINED(pointer, old_size);
100+
memcpy(newptr, pointer, Min(size, old_size));
101+
pfree(unaligned);
102+
103+
return newptr;
104+
}
105+
106+
/*
107+
* AlignedAllocGetChunkContext
108+
* Return the MemoryContext that 'pointer' belongs to.
109+
*/
110+
MemoryContext
111+
AlignedAllocGetChunkContext(void *pointer)
112+
{
113+
MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
114+
115+
Assert(!MemoryChunkIsExternal(chunk));
116+
117+
return GetMemoryChunkContext(MemoryChunkGetBlock(chunk));
118+
}
119+
120+
/*
121+
* AlignedAllocGetChunkSpace
122+
* Given a currently-allocated chunk, determine the total space
123+
* it occupies (including all memory-allocation overhead).
124+
*/
125+
Size
126+
AlignedAllocGetChunkSpace(void *pointer)
127+
{
128+
MemoryChunk *redirchunk = PointerGetMemoryChunk(pointer);
129+
void *unaligned = MemoryChunkGetBlock(redirchunk);
130+
131+
return GetMemoryChunkSpace(unaligned);
132+
}

src/backend/utils/mmgr/mcxt.c

+148-7
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "utils/memdebug.h"
3131
#include "utils/memutils.h"
3232
#include "utils/memutils_internal.h"
33+
#include "utils/memutils_memorychunk.h"
3334

3435

3536
static void BogusFree(void *pointer);
@@ -84,6 +85,21 @@ static const MemoryContextMethods mcxt_methods[] = {
8485
[MCTX_SLAB_ID].check = SlabCheck,
8586
#endif
8687

88+
/* alignedalloc.c */
89+
[MCTX_ALIGNED_REDIRECT_ID].alloc = NULL, /* not required */
90+
[MCTX_ALIGNED_REDIRECT_ID].free_p = AlignedAllocFree,
91+
[MCTX_ALIGNED_REDIRECT_ID].realloc = AlignedAllocRealloc,
92+
[MCTX_ALIGNED_REDIRECT_ID].reset = NULL, /* not required */
93+
[MCTX_ALIGNED_REDIRECT_ID].delete_context = NULL, /* not required */
94+
[MCTX_ALIGNED_REDIRECT_ID].get_chunk_context = AlignedAllocGetChunkContext,
95+
[MCTX_ALIGNED_REDIRECT_ID].get_chunk_space = AlignedAllocGetChunkSpace,
96+
[MCTX_ALIGNED_REDIRECT_ID].is_empty = NULL, /* not required */
97+
[MCTX_ALIGNED_REDIRECT_ID].stats = NULL, /* not required */
98+
#ifdef MEMORY_CONTEXT_CHECKING
99+
[MCTX_ALIGNED_REDIRECT_ID].check = NULL, /* not required */
100+
#endif
101+
102+
87103
/*
88104
* Unused (as yet) IDs should have dummy entries here. This allows us to
89105
* fail cleanly if a bogus pointer is passed to pfree or the like. It
@@ -110,11 +126,6 @@ static const MemoryContextMethods mcxt_methods[] = {
110126
[MCTX_UNUSED4_ID].realloc = BogusRealloc,
111127
[MCTX_UNUSED4_ID].get_chunk_context = BogusGetChunkContext,
112128
[MCTX_UNUSED4_ID].get_chunk_space = BogusGetChunkSpace,
113-
114-
[MCTX_UNUSED5_ID].free_p = BogusFree,
115-
[MCTX_UNUSED5_ID].realloc = BogusRealloc,
116-
[MCTX_UNUSED5_ID].get_chunk_context = BogusGetChunkContext,
117-
[MCTX_UNUSED5_ID].get_chunk_space = BogusGetChunkSpace,
118129
};
119130

120131
/*
@@ -1298,6 +1309,125 @@ palloc_extended(Size size, int flags)
12981309
return ret;
12991310
}
13001311

1312+
/*
1313+
* MemoryContextAllocAligned
1314+
* Allocate 'size' bytes of memory in 'context' aligned to 'alignto'
1315+
* bytes.
1316+
*
1317+
* Currently, we align addresses by requesting additional bytes from the
1318+
* MemoryContext's standard allocator function and then aligning the returned
1319+
* address by the required alignment. This means that the given MemoryContext
1320+
* must support providing us with a chunk of memory that's larger than 'size'.
1321+
* For allocators such as Slab, that's not going to work, as slab only allows
1322+
* chunks of the size that's specified when the context is created.
1323+
*
1324+
* 'alignto' must be a power of 2.
1325+
* 'flags' may be 0 or set the same as MemoryContextAllocExtended().
1326+
*/
1327+
void *
1328+
MemoryContextAllocAligned(MemoryContext context,
1329+
Size size, Size alignto, int flags)
1330+
{
1331+
MemoryChunk *alignedchunk;
1332+
Size alloc_size;
1333+
void *unaligned;
1334+
void *aligned;
1335+
1336+
/* wouldn't make much sense to waste that much space */
1337+
Assert(alignto < (128 * 1024 * 1024));
1338+
1339+
/* ensure alignto is a power of 2 */
1340+
Assert((alignto & (alignto - 1)) == 0);
1341+
1342+
/*
1343+
* If the alignment requirements are less than what we already guarantee
1344+
* then just use the standard allocation function.
1345+
*/
1346+
if (unlikely(alignto <= MAXIMUM_ALIGNOF))
1347+
return MemoryContextAllocExtended(context, size, flags);
1348+
1349+
/*
1350+
* We implement aligned pointers by simply allocating enough memory for
1351+
* the requested size plus the alignment and an additional "redirection"
1352+
* MemoryChunk. This additional MemoryChunk is required for operations
1353+
* such as pfree when used on the pointer returned by this function. We
1354+
* use this redirection MemoryChunk in order to find the pointer to the
1355+
* memory that was returned by the MemoryContextAllocExtended call below.
1356+
* We do that by "borrowing" the block offset field and instead of using
1357+
* that to find the offset into the owning block, we use it to find the
1358+
* original allocated address.
1359+
*
1360+
* Here we must allocate enough extra memory so that we can still align
1361+
* the pointer returned by MemoryContextAllocExtended and also have enough
1362+
* space for the redirection MemoryChunk. Since allocations will already
1363+
* be at least aligned by MAXIMUM_ALIGNOF, we can subtract that amount
1364+
* from the allocation size to save a little memory.
1365+
*/
1366+
alloc_size = size + PallocAlignedExtraBytes(alignto);
1367+
1368+
#ifdef MEMORY_CONTEXT_CHECKING
1369+
/* ensure there's space for a sentinal byte */
1370+
alloc_size += 1;
1371+
#endif
1372+
1373+
/* perform the actual allocation */
1374+
unaligned = MemoryContextAllocExtended(context, alloc_size, flags);
1375+
1376+
/* set the aligned pointer */
1377+
aligned = (void *) TYPEALIGN(alignto, (char *) unaligned +
1378+
sizeof(MemoryChunk));
1379+
1380+
alignedchunk = PointerGetMemoryChunk(aligned);
1381+
1382+
/*
1383+
* We set the redirect MemoryChunk so that the block offset calculation is
1384+
* used to point back to the 'unaligned' allocated chunk. This allows us
1385+
* to use MemoryChunkGetBlock() to find the unaligned chunk when we need
1386+
* to perform operations such as pfree() and repalloc().
1387+
*
1388+
* We store 'alignto' in the MemoryChunk's 'value' so that we know what
1389+
* the alignment was set to should we ever be asked to realloc this
1390+
* pointer.
1391+
*/
1392+
MemoryChunkSetHdrMask(alignedchunk, unaligned, alignto,
1393+
MCTX_ALIGNED_REDIRECT_ID);
1394+
1395+
/* double check we produced a correctly aligned pointer */
1396+
Assert((void *) TYPEALIGN(alignto, aligned) == aligned);
1397+
1398+
#ifdef MEMORY_CONTEXT_CHECKING
1399+
alignedchunk->requested_size = size;
1400+
/* set mark to catch clobber of "unused" space */
1401+
set_sentinel(aligned, size);
1402+
#endif
1403+
1404+
/* Mark the bytes before the redirection header as noaccess */
1405+
VALGRIND_MAKE_MEM_NOACCESS(unaligned,
1406+
(char *) alignedchunk - (char *) unaligned);
1407+
return aligned;
1408+
}
1409+
1410+
/*
1411+
* palloc_aligned
1412+
* Allocate 'size' bytes returning a pointer that's aligned to the
1413+
* 'alignto' boundary.
1414+
*
1415+
* Currently, we align addresses by requesting additional bytes from the
1416+
* MemoryContext's standard allocator function and then aligning the returned
1417+
* address by the required alignment. This means that the given MemoryContext
1418+
* must support providing us with a chunk of memory that's larger than 'size'.
1419+
* For allocators such as Slab, that's not going to work, as slab only allows
1420+
* chunks of the size that's specified when the context is created.
1421+
*
1422+
* 'alignto' must be a power of 2.
1423+
* 'flags' may be 0 or set the same as MemoryContextAllocExtended().
1424+
*/
1425+
void *
1426+
palloc_aligned(Size size, Size alignto, int flags)
1427+
{
1428+
return MemoryContextAllocAligned(CurrentMemoryContext, size, alignto, flags);
1429+
}
1430+
13011431
/*
13021432
* pfree
13031433
* Release an allocated chunk.
@@ -1306,11 +1436,16 @@ void
13061436
pfree(void *pointer)
13071437
{
13081438
#ifdef USE_VALGRIND
1439+
MemoryContextMethodID method = GetMemoryChunkMethodID(pointer);
13091440
MemoryContext context = GetMemoryChunkContext(pointer);
13101441
#endif
13111442

13121443
MCXT_METHOD(pointer, free_p) (pointer);
1313-
VALGRIND_MEMPOOL_FREE(context, pointer);
1444+
1445+
#ifdef USE_VALGRIND
1446+
if (method != MCTX_ALIGNED_REDIRECT_ID)
1447+
VALGRIND_MEMPOOL_FREE(context, pointer);
1448+
#endif
13141449
}
13151450

13161451
/*
@@ -1320,6 +1455,9 @@ pfree(void *pointer)
13201455
void *
13211456
repalloc(void *pointer, Size size)
13221457
{
1458+
#ifdef USE_VALGRIND
1459+
MemoryContextMethodID method = GetMemoryChunkMethodID(pointer);
1460+
#endif
13231461
#if defined(USE_ASSERT_CHECKING) || defined(USE_VALGRIND)
13241462
MemoryContext context = GetMemoryChunkContext(pointer);
13251463
#endif
@@ -1346,7 +1484,10 @@ repalloc(void *pointer, Size size)
13461484
size, cxt->name)));
13471485
}
13481486

1349-
VALGRIND_MEMPOOL_CHANGE(context, pointer, ret, size);
1487+
#ifdef USE_VALGRIND
1488+
if (method != MCTX_ALIGNED_REDIRECT_ID)
1489+
VALGRIND_MEMPOOL_CHANGE(context, pointer, ret, size);
1490+
#endif
13501491

13511492
return ret;
13521493
}

src/backend/utils/mmgr/meson.build

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Copyright (c) 2022, PostgreSQL Global Development Group
22

33
backend_sources += files(
4+
'alignedalloc.c',
45
'aset.c',
56
'dsa.c',
67
'freepage.c',

0 commit comments

Comments
 (0)