summary | shortlog | log | commit | commitdiff | tree
raw | patch | inline | side by side (parent: a8a968a)
raw | patch | inline | side by side (parent: a8a968a)
| author | Alvaro Herrera <alvherre@alvh.no-ip.org> | |
| Thu, 5 Oct 2023 08:59:08 +0000 (10:59 +0200) | ||
| committer | Alvaro Herrera <alvherre@alvh.no-ip.org> | |
| Thu, 5 Oct 2023 08:59:08 +0000 (10:59 +0200) |
Previously, the JSON code didn't have to worry too much about freeing
JsonLexContext, because it was never too long-lived. With new features
being added for SQL/JSON this is no longer the case. Add a routine
that knows how to free this struct and apply that to a few places, to
prevent this from becoming problematic.
At the same time, we change the API of makeJsonLexContextCstringLen to
make it receive a pointer to JsonLexContext for callers that want it to
be stack-allocated; it can also be passed as NULL to get the original
behavior of a palloc'ed one.
This also causes an ABI break due to the addition of flags to
JsonLexContext, so we can't easily backpatch it. AFAICS that's not much
of a problem; apparently some leaks might exist in JSON usage of
text-search, for example via json_to_tsvector, but I haven't seen any
complaints about that.
Per Coverity complaint about datum_to_jsonb_internal().
Discussion: https://postgr.es/m/20230808174110.oq3iymllsv6amkih@alvherre.pgsql
JsonLexContext, because it was never too long-lived. With new features
being added for SQL/JSON this is no longer the case. Add a routine
that knows how to free this struct and apply that to a few places, to
prevent this from becoming problematic.
At the same time, we change the API of makeJsonLexContextCstringLen to
make it receive a pointer to JsonLexContext for callers that want it to
be stack-allocated; it can also be passed as NULL to get the original
behavior of a palloc'ed one.
This also causes an ABI break due to the addition of flags to
JsonLexContext, so we can't easily backpatch it. AFAICS that's not much
of a problem; apparently some leaks might exist in JSON usage of
text-search, for example via json_to_tsvector, but I haven't seen any
complaints about that.
Per Coverity complaint about datum_to_jsonb_internal().
Discussion: https://postgr.es/m/20230808174110.oq3iymllsv6amkih@alvherre.pgsql
index 2c620809b209602efa25e3c750d62af6f282b8d7..55413c0fdf2bf1749e2025324c1eee4092c8b9f0 100644 (file)
{
char *json = PG_GETARG_CSTRING(0);
text *result = cstring_to_text(json);
- JsonLexContext *lex;
+ JsonLexContext lex;
/* validate it */
- lex = makeJsonLexContext(result, false);
- if (!pg_parse_json_or_errsave(lex, &nullSemAction, fcinfo->context))
+ makeJsonLexContext(&lex, result, false);
+ if (!pg_parse_json_or_errsave(&lex, &nullSemAction, fcinfo->context))
PG_RETURN_NULL();
/* Internal representation is the same as text */
StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
char *str;
int nbytes;
- JsonLexContext *lex;
+ JsonLexContext lex;
str = pq_getmsgtext(buf, buf->len - buf->cursor, &nbytes);
/* Validate it. */
- lex = makeJsonLexContextCstringLen(str, nbytes, GetDatabaseEncoding(), false);
- pg_parse_json_or_ereport(lex, &nullSemAction);
+ makeJsonLexContextCstringLen(&lex, str, nbytes, GetDatabaseEncoding(),
+ false);
+ pg_parse_json_or_ereport(&lex, &nullSemAction);
PG_RETURN_TEXT_P(cstring_to_text_with_len(str, nbytes));
}
bool
json_validate(text *json, bool check_unique_keys, bool throw_error)
{
- JsonLexContext *lex = makeJsonLexContext(json, check_unique_keys);
+ JsonLexContext lex;
JsonSemAction uniqueSemAction = {0};
JsonUniqueParsingState state;
JsonParseErrorType result;
+ makeJsonLexContext(&lex, json, check_unique_keys);
+
if (check_unique_keys)
{
- state.lex = lex;
+ state.lex = &lex;
state.stack = NULL;
state.id_counter = 0;
state.unique = true;
uniqueSemAction.object_end = json_unique_object_end;
}
- result = pg_parse_json(lex, check_unique_keys ? &uniqueSemAction : &nullSemAction);
+ result = pg_parse_json(&lex, check_unique_keys ? &uniqueSemAction : &nullSemAction);
if (result != JSON_SUCCESS)
{
if (throw_error)
- json_errsave_error(result, lex, NULL);
+ json_errsave_error(result, &lex, NULL);
return false; /* invalid json */
}
return false; /* not unique keys */
}
+ if (check_unique_keys)
+ freeJsonLexContext(&lex);
+
return true; /* ok */
}
json_typeof(PG_FUNCTION_ARGS)
{
text *json = PG_GETARG_TEXT_PP(0);
- JsonLexContext *lex = makeJsonLexContext(json, false);
+ JsonLexContext lex;
char *type;
- JsonTokenType tok;
JsonParseErrorType result;
/* Lex exactly one token from the input and check its type. */
- result = json_lex(lex);
+ makeJsonLexContext(&lex, json, false);
+ result = json_lex(&lex);
if (result != JSON_SUCCESS)
- json_errsave_error(result, lex, NULL);
- tok = lex->token_type;
+ json_errsave_error(result, &lex, NULL);
- switch (tok)
+ switch (lex.token_type)
{
case JSON_TOKEN_OBJECT_START:
type = "object";
type = "null";
break;
default:
- elog(ERROR, "unexpected json token: %d", tok);
+ elog(ERROR, "unexpected json token: %d", lex.token_type);
}
PG_RETURN_TEXT_P(cstring_to_text(type));
index 9781852b0cb583a7b27fe95ee8c7699c1db77de8..b10a60ac66558c6c217db1f0ffc45816b2143b0e 100644 (file)
static inline Datum
jsonb_from_cstring(char *json, int len, bool unique_keys, Node *escontext)
{
- JsonLexContext *lex;
+ JsonLexContext lex;
JsonbInState state;
JsonSemAction sem;
memset(&state, 0, sizeof(state));
memset(&sem, 0, sizeof(sem));
- lex = makeJsonLexContextCstringLen(json, len, GetDatabaseEncoding(), true);
+ makeJsonLexContextCstringLen(&lex, json, len, GetDatabaseEncoding(), true);
state.unique_keys = unique_keys;
state.escontext = escontext;
sem.scalar = jsonb_in_scalar;
sem.object_field_start = jsonb_in_object_field_start;
- if (!pg_parse_json_or_errsave(lex, &sem, escontext))
+ if (!pg_parse_json_or_errsave(&lex, &sem, escontext))
return (Datum) 0;
/* after parsing, the item member has the composed jsonb structure */
case JSONTYPE_JSON:
{
/* parse the json right into the existing result object */
- JsonLexContext *lex;
+ JsonLexContext lex;
JsonSemAction sem;
text *json = DatumGetTextPP(val);
- lex = makeJsonLexContext(json, true);
+ makeJsonLexContext(&lex, json, true);
memset(&sem, 0, sizeof(sem));
sem.scalar = jsonb_in_scalar;
sem.object_field_start = jsonb_in_object_field_start;
- pg_parse_json_or_ereport(lex, &sem);
+ pg_parse_json_or_ereport(&lex, &sem);
+ freeJsonLexContext(&lex);
}
break;
case JSONTYPE_JSONB:
index a4bfa5e4040133e24cd284bfe9ec9d530a99ba76..0bff272f245f44b254ccc9123fa5f11070ad7f89 100644 (file)
* directly.
*/
JsonLexContext *
-makeJsonLexContext(text *json, bool need_escapes)
+makeJsonLexContext(JsonLexContext *lex, text *json, bool need_escapes)
{
/*
* Most callers pass a detoasted datum, but it's not clear that they all
*/
json = pg_detoast_datum_packed(json);
- return makeJsonLexContextCstringLen(VARDATA_ANY(json),
+ return makeJsonLexContextCstringLen(lex,
+ VARDATA_ANY(json),
VARSIZE_ANY_EXHDR(json),
GetDatabaseEncoding(),
need_escapes);
if (SRF_IS_FIRSTCALL())
{
text *json = PG_GETARG_TEXT_PP(0);
- JsonLexContext *lex = makeJsonLexContext(json, true);
+ JsonLexContext lex;
JsonSemAction *sem;
MemoryContext oldcontext;
state = palloc(sizeof(OkeysState));
sem = palloc0(sizeof(JsonSemAction));
- state->lex = lex;
+ state->lex = makeJsonLexContext(&lex, json, true);
state->result_size = 256;
state->result_count = 0;
state->sent_count = 0;
sem->object_field_start = okeys_object_field_start;
/* remainder are all NULL, courtesy of palloc0 above */
- pg_parse_json_or_ereport(lex, sem);
+ pg_parse_json_or_ereport(&lex, sem);
/* keys are now in state->result */
- pfree(lex->strval->data);
- pfree(lex->strval);
- pfree(lex);
+ freeJsonLexContext(&lex);
pfree(sem);
MemoryContextSwitchTo(oldcontext);
int npath,
bool normalize_results)
{
- JsonLexContext *lex = makeJsonLexContext(json, true);
JsonSemAction *sem = palloc0(sizeof(JsonSemAction));
GetState *state = palloc0(sizeof(GetState));
Assert(npath >= 0);
- state->lex = lex;
+ state->lex = makeJsonLexContext(NULL, json, true);
+
/* is it "_as_text" variant? */
state->normalize_results = normalize_results;
state->npath = npath;
sem->array_element_end = get_array_element_end;
}
- pg_parse_json_or_ereport(lex, sem);
+ pg_parse_json_or_ereport(state->lex, sem);
+ freeJsonLexContext(state->lex);
return state->tresult;
}
{
text *json = PG_GETARG_TEXT_PP(0);
AlenState *state;
- JsonLexContext *lex;
+ JsonLexContext lex;
JsonSemAction *sem;
- lex = makeJsonLexContext(json, false);
state = palloc0(sizeof(AlenState));
- sem = palloc0(sizeof(JsonSemAction));
-
+ state->lex = makeJsonLexContext(&lex, json, false);
/* palloc0 does this for us */
#if 0
state->count = 0;
#endif
- state->lex = lex;
+ sem = palloc0(sizeof(JsonSemAction));
sem->semstate = (void *) state;
sem->object_start = alen_object_start;
sem->scalar = alen_scalar;
sem->array_element_start = alen_array_element_start;
- pg_parse_json_or_ereport(lex, sem);
+ pg_parse_json_or_ereport(state->lex, sem);
PG_RETURN_INT32(state->count);
}
each_worker(FunctionCallInfo fcinfo, bool as_text)
{
text *json = PG_GETARG_TEXT_PP(0);
- JsonLexContext *lex;
+ JsonLexContext lex;
JsonSemAction *sem;
ReturnSetInfo *rsi;
EachState *state;
- lex = makeJsonLexContext(json, true);
state = palloc0(sizeof(EachState));
sem = palloc0(sizeof(JsonSemAction));
state->normalize_results = as_text;
state->next_scalar = false;
- state->lex = lex;
+ state->lex = makeJsonLexContext(&lex, json, true);
state->tmp_cxt = AllocSetContextCreate(CurrentMemoryContext,
"json_each temporary cxt",
ALLOCSET_DEFAULT_SIZES);
- pg_parse_json_or_ereport(lex, sem);
+ pg_parse_json_or_ereport(&lex, sem);
MemoryContextDelete(state->tmp_cxt);
+ freeJsonLexContext(&lex);
PG_RETURN_NULL();
}
elements_worker(FunctionCallInfo fcinfo, const char *funcname, bool as_text)
{
text *json = PG_GETARG_TEXT_PP(0);
-
- /* elements only needs escaped strings when as_text */
- JsonLexContext *lex = makeJsonLexContext(json, as_text);
+ JsonLexContext lex;
JsonSemAction *sem;
ReturnSetInfo *rsi;
ElementsState *state;
+ /* elements only needs escaped strings when as_text */
+ makeJsonLexContext(&lex, json, as_text);
+
state = palloc0(sizeof(ElementsState));
sem = palloc0(sizeof(JsonSemAction));
@@ -2323,14 +2322,15 @@ elements_worker(FunctionCallInfo fcinfo, const char *funcname, bool as_text)
state->function_name = funcname;
state->normalize_results = as_text;
state->next_scalar = false;
- state->lex = lex;
+ state->lex = &lex;
state->tmp_cxt = AllocSetContextCreate(CurrentMemoryContext,
"json_array_elements temporary cxt",
ALLOCSET_DEFAULT_SIZES);
- pg_parse_json_or_ereport(lex, sem);
+ pg_parse_json_or_ereport(&lex, sem);
MemoryContextDelete(state->tmp_cxt);
+ freeJsonLexContext(&lex);
PG_RETURN_NULL();
}
PopulateArrayState state;
JsonSemAction sem;
- state.lex = makeJsonLexContextCstringLen(json, len, GetDatabaseEncoding(), true);
+ state.lex = makeJsonLexContextCstringLen(NULL, json, len,
+ GetDatabaseEncoding(), true);
state.ctx = ctx;
memset(&sem, 0, sizeof(sem));
/* number of dimensions should be already known */
Assert(ctx->ndims > 0 && ctx->dims);
- pfree(state.lex);
+ freeJsonLexContext(state.lex);
}
/*
HASHCTL ctl;
HTAB *tab;
JHashState *state;
- JsonLexContext *lex = makeJsonLexContextCstringLen(json, len, GetDatabaseEncoding(), true);
JsonSemAction *sem;
ctl.keysize = NAMEDATALEN;
state->function_name = funcname;
state->hash = tab;
- state->lex = lex;
+ state->lex = makeJsonLexContextCstringLen(NULL, json, len,
+ GetDatabaseEncoding(), true);
sem->semstate = (void *) state;
sem->array_start = hash_array_start;
sem->object_field_start = hash_object_field_start;
sem->object_field_end = hash_object_field_end;
- pg_parse_json_or_ereport(lex, sem);
+ pg_parse_json_or_ereport(state->lex, sem);
+
+ freeJsonLexContext(state->lex);
return tab;
}
if (is_json)
{
text *json = PG_GETARG_TEXT_PP(json_arg_num);
- JsonLexContext *lex;
+ JsonLexContext lex;
JsonSemAction *sem;
sem = palloc0(sizeof(JsonSemAction));
- lex = makeJsonLexContext(json, true);
+ makeJsonLexContext(&lex, json, true);
sem->semstate = (void *) state;
sem->array_start = populate_recordset_array_start;
sem->object_start = populate_recordset_object_start;
sem->object_end = populate_recordset_object_end;
- state->lex = lex;
+ state->lex = &lex;
- pg_parse_json_or_ereport(lex, sem);
+ pg_parse_json_or_ereport(&lex, sem);
+
+ freeJsonLexContext(&lex);
+ state->lex = NULL;
}
else
{
{
text *json = PG_GETARG_TEXT_PP(0);
StripnullState *state;
- JsonLexContext *lex;
+ JsonLexContext lex;
JsonSemAction *sem;
- lex = makeJsonLexContext(json, true);
state = palloc0(sizeof(StripnullState));
sem = palloc0(sizeof(JsonSemAction));
+ state->lex = makeJsonLexContext(&lex, json, true);
state->strval = makeStringInfo();
state->skip_next_null = false;
- state->lex = lex;
sem->semstate = (void *) state;
sem->object_start = sn_object_start;
sem->array_element_start = sn_array_element_start;
sem->object_field_start = sn_object_field_start;
- pg_parse_json_or_ereport(lex, sem);
+ pg_parse_json_or_ereport(&lex, sem);
PG_RETURN_TEXT_P(cstring_to_text_with_len(state->strval->data,
state->strval->len));
iterate_json_values(text *json, uint32 flags, void *action_state,
JsonIterateStringValuesAction action)
{
- JsonLexContext *lex = makeJsonLexContext(json, true);
+ JsonLexContext lex;
JsonSemAction *sem = palloc0(sizeof(JsonSemAction));
IterateJsonStringValuesState *state = palloc0(sizeof(IterateJsonStringValuesState));
- state->lex = lex;
+ state->lex = makeJsonLexContext(&lex, json, true);
state->action = action;
state->action_state = action_state;
state->flags = flags;
sem->scalar = iterate_values_scalar;
sem->object_field_start = iterate_values_object_field_start;
- pg_parse_json_or_ereport(lex, sem);
+ pg_parse_json_or_ereport(&lex, sem);
+ freeJsonLexContext(&lex);
}
/*
transform_json_string_values(text *json, void *action_state,
JsonTransformStringValuesAction transform_action)
{
- JsonLexContext *lex = makeJsonLexContext(json, true);
+ JsonLexContext lex;
JsonSemAction *sem = palloc0(sizeof(JsonSemAction));
TransformJsonStringValuesState *state = palloc0(sizeof(TransformJsonStringValuesState));
- state->lex = lex;
+ state->lex = makeJsonLexContext(&lex, json, true);
state->strval = makeStringInfo();
state->action = transform_action;
state->action_state = action_state;
sem->array_element_start = transform_string_values_array_element_start;
sem->object_field_start = transform_string_values_object_field_start;
- pg_parse_json_or_ereport(lex, sem);
+ pg_parse_json_or_ereport(&lex, sem);
+ freeJsonLexContext(&lex);
return cstring_to_text_with_len(state->strval->data, state->strval->len);
}
@@ -5670,19 +5677,19 @@ transform_string_values_scalar(void *state, char *token, JsonTokenType tokentype
JsonTokenType
json_get_first_token(text *json, bool throw_error)
{
- JsonLexContext *lex;
+ JsonLexContext lex;
JsonParseErrorType result;
- lex = makeJsonLexContext(json, false);
+ makeJsonLexContext(&lex, json, false);
/* Lex exactly one token from the input and check its type. */
- result = json_lex(lex);
+ result = json_lex(&lex);
if (result == JSON_SUCCESS)
- return lex->token_type;
+ return lex.token_type;
if (throw_error)
- json_errsave_error(result, lex, NULL);
+ json_errsave_error(result, &lex, NULL);
return JSON_TOKEN_INVALID; /* invalid json */
}
index 2379f7be7bb922268c95420964759ee5fde5cfc2..f0acd9f1e704adf7385c4bac17a55e0e316b0f21 100644 (file)
parse.saw_version_field = false;
/* Create a JSON lexing context. */
- lex = makeJsonLexContextCstringLen(buffer, size, PG_UTF8, true);
+ lex = makeJsonLexContextCstringLen(NULL, buffer, size, PG_UTF8, true);
/* Set up semantic actions. */
sem.semstate = &parse;
/* Verify the manifest checksum. */
verify_manifest_checksum(&parse, buffer, size);
+
+ freeJsonLexContext(lex);
}
/*
diff --git a/src/common/jsonapi.c b/src/common/jsonapi.c
index 2e86589cfd8dfe08f05030bdb6ed0810218ec53c..18cd78b86f65efa05010375b5de60ac7bb78735e 100644 (file)
--- a/src/common/jsonapi.c
+++ b/src/common/jsonapi.c
/*
* makeJsonLexContextCstringLen
+ * Initialize the given JsonLexContext object, or create one
*
- * lex constructor, with or without StringInfo object for de-escaped lexemes.
+ * If a valid 'lex' pointer is given, it is initialized. This can
+ * be used for stack-allocated structs, saving overhead. If NULL is
+ * given, a new struct is allocated.
*
- * Without is better as it makes the processing faster, so only make one
- * if really required.
+ * If need_escapes is true, ->strval stores the unescaped lexemes.
+ * Unescaping is expensive, so only request it when necessary.
+ *
+ * If need_escapes is true or lex was given as NULL, then caller is
+ * responsible for freeing the returned struct, either by calling
+ * freeJsonLexContext() or (in backend environment) via memory context
+ * cleanup.
*/
JsonLexContext *
-makeJsonLexContextCstringLen(char *json, int len, int encoding, bool need_escapes)
+makeJsonLexContextCstringLen(JsonLexContext *lex, char *json,
+ int len, int encoding, bool need_escapes)
{
- JsonLexContext *lex = palloc0(sizeof(JsonLexContext));
+ if (lex == NULL)
+ {
+ lex = palloc0(sizeof(JsonLexContext));
+ lex->flags |= JSONLEX_FREE_STRUCT;
+ }
+ else
+ memset(lex, 0, sizeof(JsonLexContext));
lex->input = lex->token_terminator = lex->line_start = json;
lex->line_number = 1;
lex->input_length = len;
lex->input_encoding = encoding;
if (need_escapes)
+ {
lex->strval = makeStringInfo();
+ lex->flags |= JSONLEX_FREE_STRVAL;
+ }
+
return lex;
}
+/*
+ * Free memory in a JsonLexContext. There's no need for this if a *lex
+ * pointer was given when the object was made and need_escapes was false,
+ * or (in backend environment) a memory context delete/reset is imminent.
+ */
+void
+freeJsonLexContext(JsonLexContext *lex)
+{
+ if (lex->flags & JSONLEX_FREE_STRVAL)
+ {
+ pfree(lex->strval->data);
+ pfree(lex->strval);
+ }
+ if (lex->flags & JSONLEX_FREE_STRUCT)
+ pfree(lex);
+}
+
/*
* pg_parse_json
*
index 4310084b2bdb659941c0efaf10e449040a9343d6..1207e542f7a04a1a4d1003b6f0de54f2123bc864 100644 (file)
* token_terminator and prev_token_terminator point to the character
* AFTER the end of the token, i.e. where there would be a nul byte
* if we were using nul-terminated strings.
+ *
+ * JSONLEX_FREE_STRUCT/STRVAL are used to drive freeJsonLexContext.
*/
+#define JSONLEX_FREE_STRUCT (1 << 0)
+#define JSONLEX_FREE_STRVAL (1 << 1)
typedef struct JsonLexContext
{
char *input;
char *prev_token_terminator;
JsonTokenType token_type;
int lex_level;
+ bits32 flags;
int line_number; /* line number, starting from 1 */
char *line_start; /* where that line starts within input */
StringInfo strval;
int *elements);
/*
- * constructor for JsonLexContext, with or without strval element.
- * If supplied, the strval element will contain a de-escaped version of
- * the lexeme. However, doing this imposes a performance penalty, so
- * it should be avoided if the de-escaped lexeme is not required.
+ * initializer for JsonLexContext.
+ *
+ * If a valid 'lex' pointer is given, it is initialized. This can be used
+ * for stack-allocated structs, saving overhead. If NULL is given, a new
+ * struct is allocated.
+ *
+ * If need_escapes is true, ->strval stores the unescaped lexemes.
+ * Unescaping is expensive, so only request it when necessary.
+ *
+ * If need_escapes is true or lex was given as NULL, then the caller is
+ * responsible for freeing the returned struct, either by calling
+ * freeJsonLexContext() or (in backend environment) via memory context
+ * cleanup.
*/
-extern JsonLexContext *makeJsonLexContextCstringLen(char *json,
+extern JsonLexContext *makeJsonLexContextCstringLen(JsonLexContext *lex,
+ char *json,
int len,
int encoding,
bool need_escapes);
+extern void freeJsonLexContext(JsonLexContext *lex);
/* lex one token */
extern JsonParseErrorType json_lex(JsonLexContext *lex);
index c677ac8ff782e542547e7c1ea1009f7e25bbdcef..1c6d2be0252fec49605993f90b0e15938de55537 100644 (file)
/* an action that will be applied to each value in transform_json(b)_values functions */
typedef text *(*JsonTransformStringValuesAction) (void *state, char *elem_value, int elem_len);
-/* build a JsonLexContext from a text datum */
-extern JsonLexContext *makeJsonLexContext(text *json, bool need_escapes);
+/* build a JsonLexContext from a text datum; see also freeJsonLexContext */
+extern JsonLexContext *makeJsonLexContext(JsonLexContext *lex, text *json, bool need_escapes);
/* try to parse json, and errsave(escontext) on failure */
extern bool pg_parse_json_or_errsave(JsonLexContext *lex, JsonSemAction *sem,