Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitaf1a614

Browse files
committed
Allow the low level COPY routines to read arbitrary numbers of fields.
This doesn't involve any user-visible change in behavior, but will beuseful when the COPY routines are exposed to allow their use by ForeignData Wrapper routines, which will be able to use these routines to readirregular CSV files, for example.
1 parent95e42a2 commitaf1a614

File tree

1 file changed

+57
-36
lines changed

1 file changed

+57
-36
lines changed

‎src/backend/commands/copy.c‎

Lines changed: 57 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,11 @@ typedef struct CopyStateData
141141
*/
142142
StringInfoDataattribute_buf;
143143

144+
/* field raw data pointers found by COPY FROM */
145+
146+
intmax_fields;
147+
char**raw_fields;
148+
144149
/*
145150
* Similarly, line_buf holds the whole input line being processed. The
146151
* input cycle is first to read the whole line into line_buf, convert it
@@ -250,10 +255,8 @@ static void CopyOneRowTo(CopyState cstate, Oid tupleOid,
250255
staticvoidCopyFrom(CopyStatecstate);
251256
staticboolCopyReadLine(CopyStatecstate);
252257
staticboolCopyReadLineText(CopyStatecstate);
253-
staticintCopyReadAttributesText(CopyStatecstate,intmaxfields,
254-
char**fieldvals);
255-
staticintCopyReadAttributesCSV(CopyStatecstate,intmaxfields,
256-
char**fieldvals);
258+
staticintCopyReadAttributesText(CopyStatecstate);
259+
staticintCopyReadAttributesCSV(CopyStatecstate);
257260
staticDatumCopyReadBinaryAttribute(CopyStatecstate,
258261
intcolumn_no,FmgrInfo*flinfo,
259262
Oidtypioparam,int32typmod,
@@ -1921,7 +1924,11 @@ CopyFrom(CopyState cstate)
19211924

19221925
/* create workspace for CopyReadAttributes results */
19231926
nfields=file_has_oids ? (attr_count+1) :attr_count;
1924-
field_strings= (char**)palloc(nfields*sizeof(char*));
1927+
if (!cstate->binary)
1928+
{
1929+
cstate->max_fields=nfields;
1930+
cstate->raw_fields= (char**)palloc(nfields*sizeof(char*));
1931+
}
19251932

19261933
/* Initialize state variables */
19271934
cstate->fe_eof= false;
@@ -1985,10 +1992,18 @@ CopyFrom(CopyState cstate)
19851992

19861993
/* Parse the line into de-escaped field values */
19871994
if (cstate->csv_mode)
1988-
fldct=CopyReadAttributesCSV(cstate,nfields,field_strings);
1995+
fldct=CopyReadAttributesCSV(cstate);
19891996
else
1990-
fldct=CopyReadAttributesText(cstate,nfields,field_strings);
1997+
fldct=CopyReadAttributesText(cstate);
1998+
1999+
/* check for overflowing fields */
2000+
if (nfields>0&&fldct>nfields)
2001+
ereport(ERROR,
2002+
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2003+
errmsg("extra data after last expected column")));
2004+
19912005
fieldno=0;
2006+
field_strings=cstate->raw_fields;
19922007

19932008
/* Read the OID field if present */
19942009
if (file_has_oids)
@@ -2218,7 +2233,8 @@ CopyFrom(CopyState cstate)
22182233

22192234
pfree(values);
22202235
pfree(nulls);
2221-
pfree(field_strings);
2236+
if (!cstate->binary)
2237+
pfree(cstate->raw_fields);
22222238

22232239
pfree(in_functions);
22242240
pfree(typioparams);
@@ -2717,21 +2733,22 @@ GetDecimalFromHex(char hex)
27172733
* performing de-escaping as needed.
27182734
*
27192735
* The input is in line_buf. We use attribute_buf to hold the result
2720-
* strings. fieldvals[k] is set to point to the k'th attribute string,
2721-
* or NULL when the input matches the null marker string. (Note that the
2722-
* caller cannot check for nulls since the returned string would be the
2723-
* post-de-escaping equivalent, which may look the same as some valid data
2724-
* string.)
2736+
* strings. cstate->raw_fields[k] is set to point to the k'th attribute
2737+
* string, or NULL when the input matches the null marker string.
2738+
* This array is expanded as necessary.
2739+
*
2740+
* (Note that the caller cannot check for nulls since the returned
2741+
* string would be the post-de-escaping equivalent, which may look
2742+
* the same as some valid data string.)
27252743
*
27262744
* delim is the column delimiter string (must be just one byte for now).
27272745
* null_print is the null marker string. Note that this is compared to
27282746
* the pre-de-escaped input string.
27292747
*
2730-
* The return value is the number of fields actually read.(We error out
2731-
* if this would exceed maxfields, which is the length of fieldvals[].)
2748+
* The return value is the number of fields actually read.
27322749
*/
27332750
staticint
2734-
CopyReadAttributesText(CopyStatecstate,intmaxfields,char**fieldvals)
2751+
CopyReadAttributesText(CopyStatecstate)
27352752
{
27362753
chardelimc=cstate->delim[0];
27372754
intfieldno;
@@ -2743,7 +2760,7 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
27432760
* We need a special case for zero-column tables: check that the input
27442761
* line is empty, and return.
27452762
*/
2746-
if (maxfields <=0)
2763+
if (cstate->max_fields <=0)
27472764
{
27482765
if (cstate->line_buf.len!=0)
27492766
ereport(ERROR,
@@ -2759,7 +2776,7 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
27592776
* data line, so we can just force attribute_buf to be large enough and
27602777
* then transfer data without any checks for enough space.We need to do
27612778
* it this way because enlarging attribute_buf mid-stream would invalidate
2762-
* pointers already stored intofieldvals[].
2779+
* pointers already stored intocstate->raw_fields[].
27632780
*/
27642781
if (cstate->attribute_buf.maxlen <=cstate->line_buf.len)
27652782
enlargeStringInfo(&cstate->attribute_buf,cstate->line_buf.len);
@@ -2779,15 +2796,17 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
27792796
intinput_len;
27802797
boolsaw_non_ascii= false;
27812798

2782-
/* Make sure space remains in fieldvals[] */
2783-
if (fieldno >=maxfields)
2784-
ereport(ERROR,
2785-
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
2786-
errmsg("extra data after last expected column")));
2799+
/* Make sure there is enough space for the next value */
2800+
if (fieldno >=cstate->max_fields)
2801+
{
2802+
cstate->max_fields *=2;
2803+
cstate->raw_fields=
2804+
repalloc(cstate->raw_fields,cstate->max_fields*sizeof(char*));
2805+
}
27872806

27882807
/* Remember start of field on both input and output sides */
27892808
start_ptr=cur_ptr;
2790-
fieldvals[fieldno]=output_ptr;
2809+
cstate->raw_fields[fieldno]=output_ptr;
27912810

27922811
/* Scan data for field */
27932812
for (;;)
@@ -2912,7 +2931,7 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
29122931
*/
29132932
if (saw_non_ascii)
29142933
{
2915-
char*fld=fieldvals[fieldno];
2934+
char*fld=cstate->raw_fields[fieldno];
29162935

29172936
pg_verifymbstr(fld,output_ptr- (fld+1), false);
29182937
}
@@ -2921,7 +2940,7 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
29212940
input_len=end_ptr-start_ptr;
29222941
if (input_len==cstate->null_print_len&&
29232942
strncmp(start_ptr,cstate->null_print,input_len)==0)
2924-
fieldvals[fieldno]=NULL;
2943+
cstate->raw_fields[fieldno]=NULL;
29252944

29262945
fieldno++;
29272946
/* Done if we hit EOL instead of a delim */
@@ -2944,7 +2963,7 @@ CopyReadAttributesText(CopyState cstate, int maxfields, char **fieldvals)
29442963
* "standard" (i.e. common) CSV usage.
29452964
*/
29462965
staticint
2947-
CopyReadAttributesCSV(CopyStatecstate,intmaxfields,char**fieldvals)
2966+
CopyReadAttributesCSV(CopyStatecstate)
29482967
{
29492968
chardelimc=cstate->delim[0];
29502969
charquotec=cstate->quote[0];
@@ -2958,7 +2977,7 @@ CopyReadAttributesCSV(CopyState cstate, int maxfields, char **fieldvals)
29582977
* We need a special case for zero-column tables: check that the input
29592978
* line is empty, and return.
29602979
*/
2961-
if (maxfields <=0)
2980+
if (cstate->max_fields <=0)
29622981
{
29632982
if (cstate->line_buf.len!=0)
29642983
ereport(ERROR,
@@ -2974,7 +2993,7 @@ CopyReadAttributesCSV(CopyState cstate, int maxfields, char **fieldvals)
29742993
* data line, so we can just force attribute_buf to be large enough and
29752994
* then transfer data without any checks for enough space.We need to do
29762995
* it this way because enlarging attribute_buf mid-stream would invalidate
2977-
* pointers already stored intofieldvals[].
2996+
* pointers already stored intocstate->raw_fields[].
29782997
*/
29792998
if (cstate->attribute_buf.maxlen <=cstate->line_buf.len)
29802999
enlargeStringInfo(&cstate->attribute_buf,cstate->line_buf.len);
@@ -2994,15 +3013,17 @@ CopyReadAttributesCSV(CopyState cstate, int maxfields, char **fieldvals)
29943013
char*end_ptr;
29953014
intinput_len;
29963015

2997-
/* Make sure space remains in fieldvals[] */
2998-
if (fieldno >=maxfields)
2999-
ereport(ERROR,
3000-
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
3001-
errmsg("extra data after last expected column")));
3016+
/* Make sure there is enough space for the next value */
3017+
if (fieldno >=cstate->max_fields)
3018+
{
3019+
cstate->max_fields *=2;
3020+
cstate->raw_fields=
3021+
repalloc(cstate->raw_fields,cstate->max_fields*sizeof(char*));
3022+
}
30023023

30033024
/* Remember start of field on both input and output sides */
30043025
start_ptr=cur_ptr;
3005-
fieldvals[fieldno]=output_ptr;
3026+
cstate->raw_fields[fieldno]=output_ptr;
30063027

30073028
/*
30083029
* Scan data for field,
@@ -3090,7 +3111,7 @@ CopyReadAttributesCSV(CopyState cstate, int maxfields, char **fieldvals)
30903111
input_len=end_ptr-start_ptr;
30913112
if (!saw_quote&&input_len==cstate->null_print_len&&
30923113
strncmp(start_ptr,cstate->null_print,input_len)==0)
3093-
fieldvals[fieldno]=NULL;
3114+
cstate->raw_fields[fieldno]=NULL;
30943115

30953116
fieldno++;
30963117
/* Done if we hit EOL instead of a delim */

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp