Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit ba3e6e2

Browse files
adunstanadunstan
committed
Post review fixes for test_json_parser test module
. Add missing copytight notices . improve code coverage . put work files in a temp directory in the standard location . improve error checking in C code . indent perl files with perltidy . add some comments per comments from Michael Paquier Discussion: https://postgr.es/m/ZiC3-cdFys4-6xSk@paquier.xyz
1 parent b7d35d3 commit ba3e6e2

File tree

8 files changed

+133
-48
lines changed

8 files changed

+133
-48
lines changed

‎src/test/modules/test_json_parser/t/001_test_json_parser_incremental.pl

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,29 @@
11

2+
# Copyright (c) 2024, PostgreSQL Global Development Group
3+
4+
# Test the incremental (table-driven) json parser.
5+
6+
27
use strict;
38
use warnings;
49

510
use PostgreSQL::Test::Utils;
611
use Test::More;
712
use FindBin;
813

9-
use File::Temp qw(tempfile);
10-
1114
my $test_file = "$FindBin::RealBin/../tiny.json";
1215

1316
my $exe = "test_json_parser_incremental";
1417

18+
# Test the usage error
19+
my ($stdout, $stderr) = run_command([ $exe, "-c", 10 ]);
20+
like($stderr, qr/Usage:/, 'error message if not enough arguments');
21+
22+
# Test that we get success for small chunk sizes from 64 down to 1.
23+
1524
for (my $size = 64; $size > 0; $size--)
1625
{
17-
my ($stdout, $stderr) = run_command( [$exe, "-c", $size, $test_file] );
26+
($stdout, $stderr) = run_command([ $exe, "-c", $size, $test_file ]);
1827

1928
like($stdout, qr/SUCCESS/, "chunk size $size: test succeeds");
2029
is($stderr, "", "chunk size $size: no error output");

‎src/test/modules/test_json_parser/t/002_inline.pl

Lines changed: 70 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
2+
# Copyright (c) 2021-2024, PostgreSQL Global Development Group
3+
4+
# Test success or failure of the incremental (table-driven) JSON parser
5+
# for a variety of small inputs.
6+
17
use strict;
28
use warnings;
39

@@ -6,6 +12,8 @@
612

713
use File::Temp qw(tempfile);
814

15+
my $dir = PostgreSQL::Test::Utils::tempdir;
16+
917
sub test
1018
{
1119
local $Test::Builder::Level = $Test::Builder::Level + 1;
@@ -14,27 +22,32 @@ sub test
1422
my $exe = "test_json_parser_incremental";
1523
my $chunk = length($json);
1624

25+
# Test the input with chunk sizes from max(input_size, 64) down to 1
26+
1727
if ($chunk > 64)
1828
{
1929
$chunk = 64;
2030
}
2131

22-
my ($fh, $fname) = tempfile(UNLINK => 1);
32+
my ($fh, $fname) = tempfile(DIR => $dir);
2333
print $fh "$json";
2434
close($fh);
2535

26-
foreach my $size (reverse(1..$chunk))
36+
foreach my $size (reverse(1 .. $chunk))
2737
{
28-
my ($stdout, $stderr) = run_command( [$exe, "-c", $size, $fname] );
38+
my ($stdout, $stderr) = run_command([ $exe, "-c", $size, $fname ]);
2939

3040
if (defined($params{error}))
3141
{
32-
unlike($stdout, qr/SUCCESS/, "$name, chunk size $size: test fails");
33-
like($stderr, $params{error}, "$name, chunk size $size: correct error output");
42+
unlike($stdout, qr/SUCCESS/,
43+
"$name, chunk size $size: test fails");
44+
like($stderr, $params{error},
45+
"$name, chunk size $size: correct error output");
3446
}
3547
else
3648
{
37-
like($stdout, qr/SUCCESS/, "$name, chunk size $size: test succeeds");
49+
like($stdout, qr/SUCCESS/,
50+
"$name, chunk size $size: test succeeds");
3851
is($stderr, "", "$name, chunk size $size: no error output");
3952
}
4053
}
@@ -58,26 +71,60 @@ sub test
5871
test("interrupted escapes", '"\\\\\\"\\\\\\\\\\"\\\\"');
5972
test("whitespace", ' "" ');
6073

61-
test("unclosed empty object", "{", error => qr/input string ended unexpectedly/);
74+
test("unclosed empty object",
75+
"{", error => qr/input string ended unexpectedly/);
6276
test("bad key", "{{", error => qr/Expected string or "}", but found "\{"/);
6377
test("bad key", "{{}", error => qr/Expected string or "}", but found "\{"/);
64-
test("numeric key", "{1234: 2}", error => qr/Expected string or "}", but found "1234"/);
65-
test("second numeric key", '{"a": "a", 1234: 2}', error => qr/Expected string, but found "1234"/);
66-
test("unclosed object with pair", '{"key": "value"', error => qr/input string ended unexpectedly/);
67-
test("missing key value", '{"key": }', error => qr/Expected JSON value, but found "}"/);
68-
test("missing colon", '{"key" 12345}', error => qr/Expected ":", but found "12345"/);
69-
test("missing comma", '{"key": 12345 12345}', error => qr/Expected "," or "}", but found "12345"/);
70-
test("overnested array", "[" x 6401, error => qr/maximum permitted depth is 6400/);
71-
test("overclosed array", "[]]", error => qr/Expected end of input, but found "]"/);
72-
test("unexpected token in array", "[ }}} ]", error => qr/Expected array element or "]", but found "}"/);
78+
test("numeric key", "{1234: 2}",
79+
error => qr/Expected string or "}", but found "1234"/);
80+
test(
81+
"second numeric key",
82+
'{"a": "a", 1234: 2}',
83+
error => qr/Expected string, but found "1234"/);
84+
test(
85+
"unclosed object with pair",
86+
'{"key": "value"',
87+
error => qr/input string ended unexpectedly/);
88+
test("missing key value",
89+
'{"key": }', error => qr/Expected JSON value, but found "}"/);
90+
test(
91+
"missing colon",
92+
'{"key" 12345}',
93+
error => qr/Expected ":", but found "12345"/);
94+
test(
95+
"missing comma",
96+
'{"key": 12345 12345}',
97+
error => qr/Expected "," or "}", but found "12345"/);
98+
test("overnested array",
99+
"[" x 6401, error => qr/maximum permitted depth is 6400/);
100+
test("overclosed array",
101+
"[]]", error => qr/Expected end of input, but found "]"/);
102+
test("unexpected token in array",
103+
"[ }}} ]", error => qr/Expected array element or "]", but found "}"/);
73104
test("junk punctuation", "[ ||| ]", error => qr/Token "|" is invalid/);
74-
test("missing comma in array", "[123 123]", error => qr/Expected "," or "]", but found "123"/);
105+
test("missing comma in array",
106+
"[123 123]", error => qr/Expected "," or "]", but found "123"/);
75107
test("misspelled boolean", "tru", error => qr/Token "tru" is invalid/);
76-
test("misspelled boolean in array", "[tru]", error => qr/Token "tru" is invalid/);
77-
test("smashed top-level scalar", "12zz", error => qr/Token "12zz" is invalid/);
78-
test("smashed scalar in array", "[12zz]", error => qr/Token "12zz" is invalid/);
79-
test("unknown escape sequence", '"hello\vworld"', error => qr/Escape sequence "\\v" is invalid/);
80-
test("unescaped control", "\"hello\tworld\"", error => qr/Character with value 0x09 must be escaped/);
81-
test("incorrect escape count", '"\\\\\\\\\\\\\\"', error => qr/Token ""\\\\\\\\\\\\\\"" is invalid/);
108+
test(
109+
"misspelled boolean in array",
110+
"[tru]",
111+
error => qr/Token "tru" is invalid/);
112+
test("smashed top-level scalar", "12zz",
113+
error => qr/Token "12zz" is invalid/);
114+
test(
115+
"smashed scalar in array",
116+
"[12zz]",
117+
error => qr/Token "12zz" is invalid/);
118+
test(
119+
"unknown escape sequence",
120+
'"hello\vworld"',
121+
error => qr/Escape sequence "\\v" is invalid/);
122+
test("unescaped control",
123+
"\"hello\tworld\"",
124+
error => qr/Character with value 0x09 must be escaped/);
125+
test(
126+
"incorrect escape count",
127+
'"\\\\\\\\\\\\\\"',
128+
error => qr/Token ""\\\\\\\\\\\\\\"" is invalid/);
82129

83130
done_testing();

‎src/test/modules/test_json_parser/t/003_test_semantic.pl

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
2+
# Copyright (c) 2021-2024, PostgreSQL Global Development Group
3+
4+
# Test the incremental JSON parser with semantic routines, and compare the
5+
# output with the expected output.
6+
17
use strict;
28
use warnings;
39

@@ -12,17 +18,18 @@
1218

1319
my $exe = "test_json_parser_incremental";
1420

15-
my ($stdout, $stderr) = run_command( [$exe, "-s", $test_file] );
21+
my ($stdout, $stderr) = run_command([ $exe, "-s", $test_file ]);
1622

1723
is($stderr, "", "no error output");
1824

19-
my ($fh, $fname) = tempfile();
25+
my $dir = PostgreSQL::Test::Utils::tempdir;
26+
my ($fh, $fname) = tempfile(DIR => $dir);
2027

21-
print $fh $stdout,"\n";
28+
print $fh $stdout, "\n";
2229

2330
close($fh);
2431

25-
($stdout, $stderr) = run_command(["diff", "-u", $fname, $test_out]);
32+
($stdout, $stderr) = run_command([ "diff", "-u", $fname, $test_out ]);
2633

2734
is($stdout, "", "no output diff");
2835
is($stderr, "", "no diff error");

‎src/test/modules/test_json_parser/t/004_test_parser_perf.pl

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,11 @@
11

2+
# Copyright (c) 2021-2024, PostgreSQL Global Development Group
3+
4+
# Test the JSON parser performance tester. Here we are just checking that
5+
# the performance tester can run, both with the standard parser and the
6+
# incremental parser. An actual performance test will run with thousands
7+
# of iterations onstead of just one.
8+
29
use strict;
310
use warnings;
411

@@ -14,21 +21,22 @@
1421

1522
my $contents = slurp_file($test_file);
1623

17-
my ($fh, $fname) = tempfile(UNLINK => 1);
24+
my $dir = PostgreSQL::Test::Utils::tempdir;
25+
my ($fh, $fname) = tempfile(DIR => $dir);
1826

1927
# repeat the input json file 50 times in an array
2028

21-
print $fh, '[', $contents , ",$contents" x 49 , ']';
29+
print $fh, '[', $contents, ",$contents" x 49, ']';
2230

2331
close($fh);
2432

2533
# but only do one iteration
2634

27-
my ($result) = run_log([ $exe, "1", $fname ] );
35+
my ($result) = run_log([ $exe, "1", $fname ]);
2836

29-
ok($result == 0, "perf test runs with RD parser");
37+
ok($result == 0, "perf test runs with recursive descent parser");
3038

31-
$result = run_log([ $exe, "-i" , "1", $fname ]);
39+
$result = run_log([ $exe, "-i", "1", $fname ]);
3240

3341
ok($result == 0, "perf test runs with table driven parser");
3442

‎src/test/modules/test_json_parser/test_json_parser_incremental.c

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
* If the "-c SIZE" option is provided, that chunk size is used instead
1616
* of the default of 60.
1717
*
18+
* If the -s flag is given, the program does semantic processing. This should
19+
* just mirror back the json, albeit with white space changes.
20+
*
1821
* The argument specifies the file containing the JSON input.
1922
*
2023
*-------------------------------------------------------------------------
@@ -28,6 +31,7 @@
2831
#include <unistd.h>
2932

3033
#include "common/jsonapi.h"
34+
#include "common/logging.h"
3135
#include "lib/stringinfo.h"
3236
#include "mb/pg_wchar.h"
3337
#include "pg_getopt.h"
@@ -92,10 +96,7 @@ main(int argc, char **argv)
9296
case 'c': /* chunksize */
9397
sscanf(optarg, "%zu", &chunk_size);
9498
if (chunk_size > BUFSIZE)
95-
{
96-
fprintf(stderr, "chunk size cannot exceed %d\n", BUFSIZE);
97-
exit(1);
98-
}
99+
pg_fatal("chunk size cannot exceed %d", BUFSIZE);
99100
break;
100101
case 's': /* do semantic processing */
101102
testsem = &sem;
@@ -121,13 +122,25 @@ main(int argc, char **argv)
121122
makeJsonLexContextIncremental(&lex, PG_UTF8, need_strings);
122123
initStringInfo(&json);
123124

124-
json_file = fopen(testfile, "r");
125-
fstat(fileno(json_file), &statbuf);
125+
if ((json_file = fopen(testfile, "r")) == NULL)
126+
pg_fatal("error opening input: %m");
127+
128+
if (fstat(fileno(json_file), &statbuf) != 0)
129+
pg_fatal("error statting input: %m");
130+
126131
bytes_left = statbuf.st_size;
127132

128133
for (;;)
129134
{
135+
/* We will break when there's nothing left to read */
136+
137+
if (bytes_left < chunk_size)
138+
chunk_size = bytes_left;
139+
130140
n_read = fread(buff, 1, chunk_size, json_file);
141+
if (n_read < chunk_size)
142+
pg_fatal("error reading input file: %d", ferror(json_file));
143+
131144
appendBinaryStringInfo(&json, buff, n_read);
132145

133146
/*

‎src/test/modules/test_json_parser/test_json_parser_perf.c

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
#include "postgres_fe.h"
2525
#include "common/jsonapi.h"
26+
#include "common/logging.h"
2627
#include "lib/stringinfo.h"
2728
#include "mb/pg_wchar.h"
2829
#include <stdio.h>
@@ -52,7 +53,9 @@ main(int argc, char **argv)
5253

5354
sscanf(argv[1], "%d", &iter);
5455

55-
json_file = fopen(argv[2], "r");
56+
if ((json_file = fopen(argv[2], "r")) == NULL)
57+
pg_fatal("Could not open input file '%s': %m", argv[2]);
58+
5659
while ((n_read = fread(buff, 1, 6000, json_file)) > 0)
5760
{
5861
appendBinaryStringInfo(&json, buff, n_read);
@@ -76,12 +79,8 @@ main(int argc, char **argv)
7679
freeJsonLexContext(lex);
7780
}
7881
if (result != JSON_SUCCESS)
79-
{
80-
fprintf(stderr,
81-
"unexpected result %d (expecting %d) on parse\n",
82-
result, JSON_SUCCESS);
83-
exit(1);
84-
}
82+
pg_fatal("unexpected result %d (expecting %d) on parse",
83+
result, JSON_SUCCESS);
8584
}
8685
exit(0);
8786
}

‎src/test/modules/test_json_parser/tiny.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
[{
2+
"teststring": "\u0001\b\f\n\r\t\\blurfl",
23
"updated": "Tue, 08 Sep 2009 23:28:55 +0000",
34
"links": [{"href"
45
: "http://example.com/broadway/",

‎src/test/modules/test_json_parser/tiny.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
[
22
{
3+
"teststring": "\u0001\b\f\n\r\t\\blurfl",
34
"updated": "Tue, 08 Sep 2009 23:28:55 +0000",
45
"links": [
56
{

0 commit comments

Comments
 (0)