1
1
/*-------------------------------------------------------------------------
2
2
*
3
3
* file_fdw.c
4
- * foreign-data wrapper for server-side flat files.
4
+ * foreign-data wrapper for server-side flat files (or programs) .
5
5
*
6
6
* Copyright (c) 2010-2016, PostgreSQL Global Development Group
7
7
*
@@ -57,8 +57,9 @@ struct FileFdwOption
57
57
* fileGetOptions(), which currently doesn't bother to look at user mappings.
58
58
*/
59
59
static const struct FileFdwOption valid_options [] = {
60
- /* File options */
60
+ /* Data source options */
61
61
{"filename" , ForeignTableRelationId },
62
+ {"program" , ForeignTableRelationId },
62
63
63
64
/* Format options */
64
65
/* oids option is not supported */
@@ -85,20 +86,24 @@ static const struct FileFdwOption valid_options[] = {
85
86
*/
86
87
typedef struct FileFdwPlanState
87
88
{
88
- char * filename ; /* file to read */
89
- List * options ; /* merged COPY options, excluding filename */
89
+ char * filename ; /* file or program to read from */
90
+ bool is_program ; /* true if filename represents an OS command */
91
+ List * options ; /* merged COPY options, excluding filename and
92
+ * is_program */
90
93
BlockNumber pages ; /* estimate of file's physical size */
91
- double ntuples ; /* estimate of number of rows in file */
94
+ double ntuples ; /* estimate of number of data rows */
92
95
} FileFdwPlanState ;
93
96
94
97
/*
95
98
* FDW-specific information for ForeignScanState.fdw_state.
96
99
*/
97
100
typedef struct FileFdwExecutionState
98
101
{
99
- char * filename ; /* file to read */
100
- List * options ; /* merged COPY options, excluding filename */
101
- CopyState cstate ; /* state of reading file */
102
+ char * filename ; /* file or program to read from */
103
+ bool is_program ; /* true if filename represents an OS command */
104
+ List * options ; /* merged COPY options, excluding filename and
105
+ * is_program */
106
+ CopyState cstate ; /* COPY execution state */
102
107
} FileFdwExecutionState ;
103
108
104
109
/*
@@ -139,7 +144,9 @@ static bool fileIsForeignScanParallelSafe(PlannerInfo *root, RelOptInfo *rel,
139
144
*/
140
145
static bool is_valid_option (const char * option , Oid context );
141
146
static void fileGetOptions (Oid foreigntableid ,
142
- char * * filename , List * * other_options );
147
+ char * * filename ,
148
+ bool * is_program ,
149
+ List * * other_options );
143
150
static List * get_file_fdw_attribute_options (Oid relid );
144
151
static bool check_selective_binary_conversion (RelOptInfo * baserel ,
145
152
Oid foreigntableid ,
@@ -196,16 +203,16 @@ file_fdw_validator(PG_FUNCTION_ARGS)
196
203
197
204
/*
198
205
* Only superusers are allowed to set options of a file_fdw foreign table.
199
- * This is because the filename is one of those options, and we don't want
200
- * non-superusers to be able to determine which file gets read .
206
+ * This is because we don't want non-superusers to be able to control
207
+ * which file gets read or which program gets executed .
201
208
*
202
209
* Putting this sort of permissions check in a validator is a bit of a
203
210
* crock, but there doesn't seem to be any other place that can enforce
204
211
* the check more cleanly.
205
212
*
206
- * Note that the valid_options[] array disallows setting filename at any
207
- * options level other than foreign table --- otherwise there'd still be a
208
- * security hole.
213
+ * Note that the valid_options[] array disallows setting filename and
214
+ * program at any options level other than foreign table --- otherwise
215
+ * there'd still be a security hole.
209
216
*/
210
217
if (catalog == ForeignTableRelationId && !superuser ())
211
218
ereport (ERROR ,
@@ -247,11 +254,11 @@ file_fdw_validator(PG_FUNCTION_ARGS)
247
254
}
248
255
249
256
/*
250
- * Separate out filename and column-specific options, since
257
+ * Separate out filename, program, and column-specific options, since
251
258
* ProcessCopyOptions won't accept them.
252
259
*/
253
-
254
- if ( strcmp (def -> defname , "filename " ) == 0 )
260
+ if ( strcmp ( def -> defname , "filename" ) == 0 ||
261
+ strcmp (def -> defname , "program " ) == 0 )
255
262
{
256
263
if (filename )
257
264
ereport (ERROR ,
@@ -296,12 +303,13 @@ file_fdw_validator(PG_FUNCTION_ARGS)
296
303
ProcessCopyOptions (NULL , NULL , true, other_options );
297
304
298
305
/*
299
- * Filename option is required for file_fdw foreign tables.
306
+ * Either filename or program option is required for file_fdw foreign
307
+ * tables.
300
308
*/
301
309
if (catalog == ForeignTableRelationId && filename == NULL )
302
310
ereport (ERROR ,
303
311
(errcode (ERRCODE_FDW_DYNAMIC_PARAMETER_VALUE_NEEDED ),
304
- errmsg ("filename is required for file_fdw foreign tables" )));
312
+ errmsg ("either filename or program is required for file_fdw foreign tables" )));
305
313
306
314
PG_RETURN_VOID ();
307
315
}
@@ -326,12 +334,12 @@ is_valid_option(const char *option, Oid context)
326
334
/*
327
335
* Fetch the options for a file_fdw foreign table.
328
336
*
329
- * We have to separate out " filename" from the other options because
330
- * it must not appear in the options list passed to the core COPY code.
337
+ * We have to separate out filename/program from the other options because
338
+ * those must not appear in the options list passed to the core COPY code.
331
339
*/
332
340
static void
333
341
fileGetOptions (Oid foreigntableid ,
334
- char * * filename , List * * other_options )
342
+ char * * filename , bool * is_program , List * * other_options )
335
343
{
336
344
ForeignTable * table ;
337
345
ForeignServer * server ;
@@ -359,9 +367,11 @@ fileGetOptions(Oid foreigntableid,
359
367
options = list_concat (options , get_file_fdw_attribute_options (foreigntableid ));
360
368
361
369
/*
362
- * Separate out the filename.
370
+ * Separate out the filename or program option (we assume there is only
371
+ * one).
363
372
*/
364
373
* filename = NULL ;
374
+ * is_program = false;
365
375
prev = NULL ;
366
376
foreach (lc , options )
367
377
{
@@ -373,15 +383,22 @@ fileGetOptions(Oid foreigntableid,
373
383
options = list_delete_cell (options , lc , prev );
374
384
break ;
375
385
}
386
+ else if (strcmp (def -> defname , "program" ) == 0 )
387
+ {
388
+ * filename = defGetString (def );
389
+ * is_program = true;
390
+ options = list_delete_cell (options , lc , prev );
391
+ break ;
392
+ }
376
393
prev = lc ;
377
394
}
378
395
379
396
/*
380
- * The validator should have checked that a filename was included in the
381
- * options, but check again, just in case.
397
+ * The validator should have checked that filename or program was included
398
+ * in the options, but check again, just in case.
382
399
*/
383
400
if (* filename == NULL )
384
- elog (ERROR , "filename is required for file_fdw foreign tables" );
401
+ elog (ERROR , "either filename or program is required for file_fdw foreign tables" );
385
402
386
403
* other_options = options ;
387
404
}
@@ -475,12 +492,15 @@ fileGetForeignRelSize(PlannerInfo *root,
475
492
FileFdwPlanState * fdw_private ;
476
493
477
494
/*
478
- * Fetch options. We only need filename at this point, but we might as
479
- * well get everything and not need to re-fetch it later in planning.
495
+ * Fetch options. We only need filename (or program) at this point, but
496
+ * we might as well get everything and not need to re-fetch it later in
497
+ * planning.
480
498
*/
481
499
fdw_private = (FileFdwPlanState * ) palloc (sizeof (FileFdwPlanState ));
482
500
fileGetOptions (foreigntableid ,
483
- & fdw_private -> filename , & fdw_private -> options );
501
+ & fdw_private -> filename ,
502
+ & fdw_private -> is_program ,
503
+ & fdw_private -> options );
484
504
baserel -> fdw_private = (void * ) fdw_private ;
485
505
486
506
/* Estimate relation size */
@@ -583,20 +603,25 @@ static void
583
603
fileExplainForeignScan (ForeignScanState * node , ExplainState * es )
584
604
{
585
605
char * filename ;
606
+ bool is_program ;
586
607
List * options ;
587
608
588
- /* Fetch options --- we only need filename at this point */
609
+ /* Fetch options --- we only need filename and is_program at this point */
589
610
fileGetOptions (RelationGetRelid (node -> ss .ss_currentRelation ),
590
- & filename , & options );
611
+ & filename , & is_program , & options );
591
612
592
- ExplainPropertyText ("Foreign File" , filename , es );
613
+ if (is_program )
614
+ ExplainPropertyText ("Foreign Program" , filename , es );
615
+ else
616
+ ExplainPropertyText ("Foreign File" , filename , es );
593
617
594
618
/* Suppress file size if we're not showing cost details */
595
619
if (es -> costs )
596
620
{
597
621
struct stat stat_buf ;
598
622
599
- if (stat (filename , & stat_buf ) == 0 )
623
+ if (!is_program &&
624
+ stat (filename , & stat_buf ) == 0 )
600
625
ExplainPropertyLong ("Foreign File Size" , (long ) stat_buf .st_size ,
601
626
es );
602
627
}
@@ -611,6 +636,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags)
611
636
{
612
637
ForeignScan * plan = (ForeignScan * ) node -> ss .ps .plan ;
613
638
char * filename ;
639
+ bool is_program ;
614
640
List * options ;
615
641
CopyState cstate ;
616
642
FileFdwExecutionState * festate ;
@@ -623,7 +649,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags)
623
649
624
650
/* Fetch options of foreign table */
625
651
fileGetOptions (RelationGetRelid (node -> ss .ss_currentRelation ),
626
- & filename , & options );
652
+ & filename , & is_program , & options );
627
653
628
654
/* Add any options from the plan (currently only convert_selectively) */
629
655
options = list_concat (options , plan -> fdw_private );
@@ -635,7 +661,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags)
635
661
cstate = BeginCopyFrom (NULL ,
636
662
node -> ss .ss_currentRelation ,
637
663
filename ,
638
- false ,
664
+ is_program ,
639
665
NIL ,
640
666
options );
641
667
@@ -645,6 +671,7 @@ fileBeginForeignScan(ForeignScanState *node, int eflags)
645
671
*/
646
672
festate = (FileFdwExecutionState * ) palloc (sizeof (FileFdwExecutionState ));
647
673
festate -> filename = filename ;
674
+ festate -> is_program = is_program ;
648
675
festate -> options = options ;
649
676
festate -> cstate = cstate ;
650
677
@@ -709,7 +736,7 @@ fileReScanForeignScan(ForeignScanState *node)
709
736
festate -> cstate = BeginCopyFrom (NULL ,
710
737
node -> ss .ss_currentRelation ,
711
738
festate -> filename ,
712
- false ,
739
+ festate -> is_program ,
713
740
NIL ,
714
741
festate -> options );
715
742
}
@@ -738,11 +765,22 @@ fileAnalyzeForeignTable(Relation relation,
738
765
BlockNumber * totalpages )
739
766
{
740
767
char * filename ;
768
+ bool is_program ;
741
769
List * options ;
742
770
struct stat stat_buf ;
743
771
744
772
/* Fetch options of foreign table */
745
- fileGetOptions (RelationGetRelid (relation ), & filename , & options );
773
+ fileGetOptions (RelationGetRelid (relation ), & filename , & is_program , & options );
774
+
775
+ /*
776
+ * If this is a program instead of a file, just return false to skip
777
+ * analyzing the table. We could run the program and collect stats on
778
+ * whatever it currently returns, but it seems likely that in such cases
779
+ * the output would be too volatile for the stats to be useful. Maybe
780
+ * there should be an option to enable doing this?
781
+ */
782
+ if (is_program )
783
+ return false;
746
784
747
785
/*
748
786
* Get size of the file. (XXX if we fail here, would it be better to just
@@ -769,8 +807,8 @@ fileAnalyzeForeignTable(Relation relation,
769
807
770
808
/*
771
809
* fileIsForeignScanParallelSafe
772
- * Reading a file in a parallel worker should work just the same as
773
- * reading it in the leader, so mark scans safe.
810
+ * Reading a file, or external program, in a parallel worker should work
811
+ * just the same as reading it in the leader, so mark scans safe.
774
812
*/
775
813
static bool
776
814
fileIsForeignScanParallelSafe (PlannerInfo * root , RelOptInfo * rel ,
@@ -916,9 +954,10 @@ estimate_size(PlannerInfo *root, RelOptInfo *baserel,
916
954
917
955
/*
918
956
* Get size of the file. It might not be there at plan time, though, in
919
- * which case we have to use a default estimate.
957
+ * which case we have to use a default estimate. We also have to fall
958
+ * back to the default if using a program as the input.
920
959
*/
921
- if (stat (fdw_private -> filename , & stat_buf ) < 0 )
960
+ if (fdw_private -> is_program || stat (fdw_private -> filename , & stat_buf ) < 0 )
922
961
stat_buf .st_size = 10 * BLCKSZ ;
923
962
924
963
/*
@@ -1000,6 +1039,11 @@ estimate_costs(PlannerInfo *root, RelOptInfo *baserel,
1000
1039
* that I/O costs are equivalent to a regular table file of the same size.
1001
1040
* However, we take per-tuple CPU costs as 10x of a seqscan, to account
1002
1041
* for the cost of parsing records.
1042
+ *
1043
+ * In the case of a program source, this calculation is even more divorced
1044
+ * from reality, but we have no good alternative; and it's not clear that
1045
+ * the numbers we produce here matter much anyway, since there's only one
1046
+ * access path for the rel.
1003
1047
*/
1004
1048
run_cost += seq_page_cost * pages ;
1005
1049
@@ -1036,6 +1080,7 @@ file_acquire_sample_rows(Relation onerel, int elevel,
1036
1080
bool * nulls ;
1037
1081
bool found ;
1038
1082
char * filename ;
1083
+ bool is_program ;
1039
1084
List * options ;
1040
1085
CopyState cstate ;
1041
1086
ErrorContextCallback errcallback ;
@@ -1050,12 +1095,12 @@ file_acquire_sample_rows(Relation onerel, int elevel,
1050
1095
nulls = (bool * ) palloc (tupDesc -> natts * sizeof (bool ));
1051
1096
1052
1097
/* Fetch options of foreign table */
1053
- fileGetOptions (RelationGetRelid (onerel ), & filename , & options );
1098
+ fileGetOptions (RelationGetRelid (onerel ), & filename , & is_program , & options );
1054
1099
1055
1100
/*
1056
1101
* Create CopyState from FDW options.
1057
1102
*/
1058
- cstate = BeginCopyFrom (NULL , onerel , filename , false , NIL , options );
1103
+ cstate = BeginCopyFrom (NULL , onerel , filename , is_program , NIL , options );
1059
1104
1060
1105
/*
1061
1106
* Use per-tuple memory context to prevent leak of memory used to read
0 commit comments