Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 40bede5

Browse files
committed
Move pg_lzcompress.c to src/common.
The meta data of PGLZ symbolized by PGLZ_Header is removed, to make the compression and decompression code independent on the backend-only varlena facility. PGLZ_Header is being used to store some meta data related to the data being compressed like the raw length of the uncompressed record or some varlena-related data, making it unpluggable once PGLZ is stored in src/common as it contains some backend-only code paths with the management of varlena structures. The APIs of PGLZ are reworked at the same time to do only compression and decompression of buffers without the meta-data layer, simplifying its use for a more general usage. On-disk format is preserved as well, so there is no incompatibility with previous major versions of PostgreSQL for TOAST entries. Exposing compression and decompression APIs of pglz makes possible its use by extensions and contrib modules. Especially this commit is required for upcoming WAL compression feature so that the WAL reader facility can decompress the WAL data by using pglz_decompress. Michael Paquier, reviewed by me.
1 parent 237795a commit 40bede5

File tree

7 files changed

+124
-102
lines changed

7 files changed

+124
-102
lines changed

src/backend/access/heap/tuptoaster.c

Lines changed: 65 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,36 @@
3535
#include "access/tuptoaster.h"
3636
#include "access/xact.h"
3737
#include "catalog/catalog.h"
38+
#include "common/pg_lzcompress.h"
3839
#include "miscadmin.h"
3940
#include "utils/fmgroids.h"
40-
#include "utils/pg_lzcompress.h"
4141
#include "utils/rel.h"
4242
#include "utils/typcache.h"
4343
#include "utils/tqual.h"
4444

4545

4646
#undef TOAST_DEBUG
4747

48+
/*
49+
* The information at the start of the compressed toast data.
50+
*/
51+
typedef struct toast_compress_header
52+
{
53+
int32 vl_len_; /* varlena header (do not touch directly!) */
54+
int32 rawsize;
55+
} toast_compress_header;
56+
57+
/*
58+
* Utilities for manipulation of header information for compressed
59+
* toast entries.
60+
*/
61+
#define TOAST_COMPRESS_HDRSZ ((int32) sizeof(toast_compress_header))
62+
#define TOAST_COMPRESS_RAWSIZE(ptr) (((toast_compress_header *) ptr)->rawsize)
63+
#define TOAST_COMPRESS_RAWDATA(ptr) \
64+
(((char *) ptr) + TOAST_COMPRESS_HDRSZ)
65+
#define TOAST_COMPRESS_SET_RAWSIZE(ptr, len) \
66+
(((toast_compress_header *) ptr)->rawsize = len)
67+
4868
static void toast_delete_datum(Relation rel, Datum value);
4969
static Datum toast_save_datum(Relation rel, Datum value,
5070
struct varlena * oldexternal, int options);
@@ -53,6 +73,7 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid);
5373
static struct varlena *toast_fetch_datum(struct varlena * attr);
5474
static struct varlena *toast_fetch_datum_slice(struct varlena * attr,
5575
int32 sliceoffset, int32 length);
76+
static struct varlena *toast_decompress_datum(struct varlena * attr);
5677
static int toast_open_indexes(Relation toastrel,
5778
LOCKMODE lock,
5879
Relation **toastidxs,
@@ -138,11 +159,8 @@ heap_tuple_untoast_attr(struct varlena * attr)
138159
/* If it's compressed, decompress it */
139160
if (VARATT_IS_COMPRESSED(attr))
140161
{
141-
PGLZ_Header *tmp = (PGLZ_Header *) attr;
142-
143-
attr = (struct varlena *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
144-
SET_VARSIZE(attr, PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
145-
pglz_decompress(tmp, VARDATA(attr));
162+
struct varlena *tmp = attr;
163+
attr = toast_decompress_datum(tmp);
146164
pfree(tmp);
147165
}
148166
}
@@ -163,11 +181,7 @@ heap_tuple_untoast_attr(struct varlena * attr)
163181
/*
164182
* This is a compressed value inside of the main tuple
165183
*/
166-
PGLZ_Header *tmp = (PGLZ_Header *) attr;
167-
168-
attr = (struct varlena *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
169-
SET_VARSIZE(attr, PGLZ_RAW_SIZE(tmp) + VARHDRSZ);
170-
pglz_decompress(tmp, VARDATA(attr));
184+
attr = toast_decompress_datum(attr);
171185
}
172186
else if (VARATT_IS_SHORT(attr))
173187
{
@@ -234,14 +248,10 @@ heap_tuple_untoast_attr_slice(struct varlena * attr,
234248

235249
if (VARATT_IS_COMPRESSED(preslice))
236250
{
237-
PGLZ_Header *tmp = (PGLZ_Header *) preslice;
238-
Size size = PGLZ_RAW_SIZE(tmp) + VARHDRSZ;
239-
240-
preslice = (struct varlena *) palloc(size);
241-
SET_VARSIZE(preslice, size);
242-
pglz_decompress(tmp, VARDATA(preslice));
251+
struct varlena *tmp = preslice;
252+
preslice = toast_decompress_datum(tmp);
243253

244-
if (tmp != (PGLZ_Header *) attr)
254+
if (tmp != attr)
245255
pfree(tmp);
246256
}
247257

@@ -1228,6 +1238,7 @@ toast_compress_datum(Datum value)
12281238
{
12291239
struct varlena *tmp;
12301240
int32 valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
1241+
int32 len;
12311242

12321243
Assert(!VARATT_IS_EXTERNAL(DatumGetPointer(value)));
12331244
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
@@ -1240,7 +1251,8 @@ toast_compress_datum(Datum value)
12401251
valsize > PGLZ_strategy_default->max_input_size)
12411252
return PointerGetDatum(NULL);
12421253

1243-
tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize));
1254+
tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
1255+
TOAST_COMPRESS_HDRSZ);
12441256

12451257
/*
12461258
* We recheck the actual size even if pglz_compress() reports success,
@@ -1252,10 +1264,15 @@ toast_compress_datum(Datum value)
12521264
* only one header byte and no padding if the value is short enough. So
12531265
* we insist on a savings of more than 2 bytes to ensure we have a gain.
12541266
*/
1255-
if (pglz_compress(VARDATA_ANY(DatumGetPointer(value)), valsize,
1256-
(PGLZ_Header *) tmp, PGLZ_strategy_default) &&
1257-
VARSIZE(tmp) < valsize - 2)
1267+
len = pglz_compress(VARDATA_ANY(DatumGetPointer(value)),
1268+
valsize,
1269+
TOAST_COMPRESS_RAWDATA(tmp),
1270+
PGLZ_strategy_default);
1271+
if (len >= 0 &&
1272+
len + TOAST_COMPRESS_HDRSZ < valsize - 2)
12581273
{
1274+
TOAST_COMPRESS_SET_RAWSIZE(tmp, valsize);
1275+
SET_VARSIZE_COMPRESSED(tmp, len + TOAST_COMPRESS_HDRSZ);
12591276
/* successful compression */
12601277
return PointerGetDatum(tmp);
12611278
}
@@ -2100,6 +2117,32 @@ toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length)
21002117
return result;
21012118
}
21022119

2120+
/* ----------
2121+
* toast_decompress_datum -
2122+
*
2123+
* Decompress a compressed version of a varlena datum
2124+
*/
2125+
static struct varlena *
2126+
toast_decompress_datum(struct varlena * attr)
2127+
{
2128+
struct varlena *result;
2129+
2130+
Assert(VARATT_IS_COMPRESSED(attr));
2131+
2132+
result = (struct varlena *)
2133+
palloc(TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ);
2134+
SET_VARSIZE(result, TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ);
2135+
2136+
if (pglz_decompress(TOAST_COMPRESS_RAWDATA(attr),
2137+
VARSIZE(attr) - TOAST_COMPRESS_HDRSZ,
2138+
VARDATA(result),
2139+
TOAST_COMPRESS_RAWSIZE(attr)) < 0)
2140+
elog(ERROR, "compressed data is corrupted");
2141+
2142+
return result;
2143+
}
2144+
2145+
21032146
/* ----------
21042147
* toast_open_indexes
21052148
*

src/backend/utils/adt/Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ OBJS = acl.o arrayfuncs.o array_selfuncs.o array_typanalyze.o \
2525
jsonfuncs.o like.o lockfuncs.o mac.o misc.o nabstime.o name.o \
2626
network.o network_gist.o network_selfuncs.o \
2727
numeric.o numutils.o oid.o oracle_compat.o \
28-
orderedsetaggs.o pg_lzcompress.o pg_locale.o pg_lsn.o \
29-
pgstatfuncs.o pseudotypes.o quote.o rangetypes.o rangetypes_gist.o \
28+
orderedsetaggs.o pg_locale.o pg_lsn.o pgstatfuncs.o \
29+
pseudotypes.o quote.o rangetypes.o rangetypes_gist.o \
3030
rangetypes_selfuncs.o rangetypes_spgist.o rangetypes_typanalyze.o \
3131
regexp.o regproc.o ri_triggers.o rowtypes.o ruleutils.o \
3232
selfuncs.o tid.o timestamp.o trigfuncs.o \

src/common/Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ include $(top_builddir)/src/Makefile.global
2323
override CPPFLAGS := -DFRONTEND $(CPPFLAGS)
2424
LIBS += $(PTHREAD_LIBS)
2525

26-
OBJS_COMMON = exec.o pgfnames.o psprintf.o relpath.o rmtree.o string.o username.o wait_error.o
26+
OBJS_COMMON = exec.o pg_lzcompress.o pgfnames.o psprintf.o relpath.o \
27+
rmtree.o string.o username.o wait_error.o
2728

2829
OBJS_FRONTEND = $(OBJS_COMMON) fe_memutils.o
2930

src/backend/utils/adt/pg_lzcompress.c renamed to src/common/pg_lzcompress.c

Lines changed: 48 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88
*
99
* Entry routines:
1010
*
11-
* bool
12-
* pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
11+
* int32
12+
* pglz_compress(const char *source, int32 slen, char *dest,
1313
* const PGLZ_Strategy *strategy);
1414
*
1515
* source is the input data to be compressed.
@@ -23,44 +23,43 @@
2323
* the compression algorithm. If NULL, the compiled
2424
* in default strategy is used.
2525
*
26-
* The return value is TRUE if compression succeeded,
27-
* FALSE if not; in the latter case the contents of dest
28-
* are undefined.
26+
* The return value is the number of bytes written in the
27+
* buffer dest, or -1 if compression fails; in the latter
28+
* case the contents of dest are undefined.
2929
*
30-
* void
31-
* pglz_decompress(const PGLZ_Header *source, char *dest)
30+
* int32
31+
* pglz_decompress(const char *source, int32 slen, char *dest,
32+
* int32 rawsize)
3233
*
3334
* source is the compressed input.
3435
*
36+
* slen is the length of the compressed input.
37+
*
3538
* dest is the area where the uncompressed data will be
3639
* written to. It is the callers responsibility to
37-
* provide enough space. The required amount can be
38-
* obtained with the macro PGLZ_RAW_SIZE(source).
40+
* provide enough space.
3941
*
4042
* The data is written to buff exactly as it was handed
4143
* to pglz_compress(). No terminating zero byte is added.
4244
*
43-
* The decompression algorithm and internal data format:
45+
* rawsize is the length of the uncompressed data.
4446
*
45-
* PGLZ_Header is defined as
47+
* The return value is the number of bytes written in the
48+
* buffer dest, or -1 if decompression fails.
4649
*
47-
* typedef struct PGLZ_Header {
48-
* int32 vl_len_;
49-
* int32 rawsize;
50-
* }
50+
* The decompression algorithm and internal data format:
5151
*
52-
* The header is followed by the compressed data itself.
52+
* It is made with the compressed data itself.
5353
*
5454
* The data representation is easiest explained by describing
5555
* the process of decompression.
5656
*
57-
* If VARSIZE(x) == rawsize + sizeof(PGLZ_Header), then the data
57+
* If compressed_size == rawsize, then the data
5858
* is stored uncompressed as plain bytes. Thus, the decompressor
59-
* simply copies rawsize bytes from the location after the
60-
* header to the destination.
59+
* simply copies rawsize bytes to the destination.
6160
*
62-
* Otherwise the first byte after the header tells what to do
63-
* the next 8 times. We call this the control byte.
61+
* Otherwise the first byte tells what to do the next 8 times.
62+
* We call this the control byte.
6463
*
6564
* An unset bit in the control byte means, that one uncompressed
6665
* byte follows, which is copied from input to output.
@@ -169,14 +168,18 @@
169168
*
170169
* Copyright (c) 1999-2015, PostgreSQL Global Development Group
171170
*
172-
* src/backend/utils/adt/pg_lzcompress.c
171+
* src/common/pg_lzcompress.c
173172
* ----------
174173
*/
174+
#ifndef FRONTEND
175175
#include "postgres.h"
176+
#else
177+
#include "postgres_fe.h"
178+
#endif
176179

177180
#include <limits.h>
178181

179-
#include "utils/pg_lzcompress.h"
182+
#include "common/pg_lzcompress.h"
180183

181184

182185
/* ----------
@@ -492,14 +495,15 @@ pglz_find_match(int16 *hstart, const char *input, const char *end,
492495
/* ----------
493496
* pglz_compress -
494497
*
495-
* Compresses source into dest using strategy.
498+
* Compresses source into dest using strategy. Returns the number of
499+
* bytes written in buffer dest, or -1 if compression fails.
496500
* ----------
497501
*/
498-
bool
499-
pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
502+
int32
503+
pglz_compress(const char *source, int32 slen, char *dest,
500504
const PGLZ_Strategy *strategy)
501505
{
502-
unsigned char *bp = ((unsigned char *) dest) + sizeof(PGLZ_Header);
506+
unsigned char *bp = (unsigned char *) dest;
503507
unsigned char *bstart = bp;
504508
int hist_next = 1;
505509
bool hist_recycle = false;
@@ -533,12 +537,7 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
533537
if (strategy->match_size_good <= 0 ||
534538
slen < strategy->min_input_size ||
535539
slen > strategy->max_input_size)
536-
return false;
537-
538-
/*
539-
* Save the original source size in the header.
540-
*/
541-
dest->rawsize = slen;
540+
return -1;
542541

543542
/*
544543
* Limit the match parameters to the supported range.
@@ -611,7 +610,7 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
611610
* allow 4 slop bytes.
612611
*/
613612
if (bp - bstart >= result_max)
614-
return false;
613+
return -1;
615614

616615
/*
617616
* If we've emitted more than first_success_by bytes without finding
@@ -620,7 +619,7 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
620619
* pre-compressed data).
621620
*/
622621
if (!found_match && bp - bstart >= strategy->first_success_by)
623-
return false;
622+
return -1;
624623

625624
/*
626625
* Try to find a match in the history
@@ -664,35 +663,34 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
664663
*ctrlp = ctrlb;
665664
result_size = bp - bstart;
666665
if (result_size >= result_max)
667-
return false;
668-
669-
/*
670-
* Success - need only fill in the actual length of the compressed datum.
671-
*/
672-
SET_VARSIZE_COMPRESSED(dest, result_size + sizeof(PGLZ_Header));
666+
return -1;
673667

674-
return true;
668+
/* success */
669+
return result_size;
675670
}
676671

677672

678673
/* ----------
679674
* pglz_decompress -
680675
*
681-
* Decompresses source into dest.
676+
* Decompresses source into dest. Returns the number of bytes
677+
* decompressed in the destination buffer, or -1 if decompression
678+
* fails.
682679
* ----------
683680
*/
684-
void
685-
pglz_decompress(const PGLZ_Header *source, char *dest)
681+
int32
682+
pglz_decompress(const char *source, int32 slen, char *dest,
683+
int32 rawsize)
686684
{
687685
const unsigned char *sp;
688686
const unsigned char *srcend;
689687
unsigned char *dp;
690688
unsigned char *destend;
691689

692-
sp = ((const unsigned char *) source) + sizeof(PGLZ_Header);
693-
srcend = ((const unsigned char *) source) + VARSIZE(source);
690+
sp = (const unsigned char *) source;
691+
srcend = ((const unsigned char *) source) + slen;
694692
dp = (unsigned char *) dest;
695-
destend = dp + source->rawsize;
693+
destend = dp + rawsize;
696694

697695
while (sp < srcend && dp < destend)
698696
{
@@ -771,9 +769,10 @@ pglz_decompress(const PGLZ_Header *source, char *dest)
771769
* Check we decompressed the right amount.
772770
*/
773771
if (dp != destend || sp != srcend)
774-
elog(ERROR, "compressed data is corrupt");
772+
return -1;
775773

776774
/*
777775
* That's it.
778776
*/
777+
return rawsize;
779778
}

0 commit comments

Comments
 (0)