Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commite4a8fb8

Browse files
peterejeltz
andcommitted
replication parser: pure parser and reentrant scanner
Use the flex %option reentrant and the bison option %pure-parser tomake the generated scanner and parser pure, reentrant, andthread-safe.Make the generated scanner use palloc() etc. instead of malloc() etc.Previously, we only used palloc() for the buffer, but flex would stilluse malloc() for its internal structures. As a result, there could besome small memory leaks in case of uncaught errors. Now, all thememory is under palloc() control, so there are no more such issues.Simplify flex scan buffer management: Instead of constructing thebuffer from pieces and then using yy_scan_buffer(), we can just useyy_scan_string(), which does the same thing internally.The previous code was necessary because we allocated the buffer withpalloc() and the rest of the state was handled by malloc(). But thisis no longer the case; everything is under palloc() now.Use flex yyextra to handle context information, instead of globalvariables. This complements the other changes to make the scannerreentrant.Reviewed-by: Heikki Linnakangas <hlinnaka@iki.fi>Co-authored-by: Andreas Karlsson <andreas@proxel.se>Reviewed-by: Andreas Karlsson <andreas@proxel.se>Discussion:https://www.postgresql.org/message-id/flat/eb6faeac-2a8a-4b69-9189-c33c520e5b7b@eisentraut.org
1 parent5af6990 commite4a8fb8

File tree

5 files changed

+116
-80
lines changed

5 files changed

+116
-80
lines changed

‎src/backend/nls.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ GETTEXT_TRIGGERS = $(BACKEND_COMMON_GETTEXT_TRIGGERS) \
99
yyerror\
1010
jsonpath_yyerror:3\
1111
parser_yyerror\
12-
replication_yyerror\
12+
replication_yyerror:2\
1313
scanner_yyerror\
1414
syncrep_yyerror\
1515
report_invalid_record:2\

‎src/backend/replication/repl_gram.y

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,6 @@
2424

2525
#include"repl_gram.h"
2626

27-
/* silence -Wmissing-variable-declarations*/
28-
externint replication_yychar;
29-
externint replication_yynerrs;
30-
3127

3228
/* Result of the parsing is returned here*/
3329
Node *replication_parse_result;
@@ -43,6 +39,9 @@ Node *replication_parse_result;
4339

4440
%}
4541

42+
%parse-param {yyscan_t yyscanner}
43+
%lex-param {yyscan_t yyscanner}
44+
%pure-parser
4645
%expect0
4746
%name-prefix="replication_yy"
4847

@@ -106,6 +105,8 @@ Node *replication_parse_result;
106105
firstcmd:commandopt_semicolon
107106
{
108107
replication_parse_result =$1;
108+
109+
(void)yynerrs;/* suppress compiler warning*/
109110
}
110111
;
111112

‎src/backend/replication/repl_scanner.l

Lines changed: 93 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -38,30 +38,36 @@ fprintf_to_ereport(const char *fmt, const char *msg)
3838
ereport(ERROR, (errmsg_internal("%s", msg)));
3939
}
4040

41-
/* Handle to the buffer that the lexer uses internally*/
42-
static YY_BUFFER_STATE scanbufhandle;
43-
44-
/* Pushed-back token (we only handle one)*/
45-
staticintrepl_pushed_back_token;
41+
structreplication_yy_extra_type
42+
{
43+
/* Pushed-back token (we only handle one) */
44+
intrepl_pushed_back_token;
4645

47-
/* Work area for collecting literals*/
48-
static StringInfoData litbuf;
46+
/* Work area for collecting literals */
47+
StringInfoData litbuf;
48+
};
49+
#defineYY_EXTRA_TYPEstructreplication_yy_extra_type *
4950

50-
staticvoidstartlit(void);
51-
staticchar *litbufdup(void);
52-
staticvoidaddlit(char *ytext,int yleng);
53-
staticvoidaddlitchar(unsignedchar ychar);
51+
staticvoidstartlit(yyscan_t yyscanner);
52+
staticchar *litbufdup(yyscan_t yyscanner);
53+
staticvoidaddlit(char *ytext,int yleng,yyscan_t yyscanner);
54+
staticvoidaddlitchar(unsignedchar ychar,yyscan_t yyscanner);
5455

5556
/* LCOV_EXCL_START */
5657

5758
%}
5859

60+
%option reentrant
61+
%option bison-bridge
5962
%option 8bit
6063
%option never-interactive
6164
%option nodefault
6265
%option noinput
6366
%option nounput
6467
%option noyywrap
68+
%option noyyalloc
69+
%option noyyrealloc
70+
%option noyyfree
6571
%option warn
6672
%option prefix="replication_yy"
6773

@@ -108,11 +114,11 @@ identifier{ident_start}{ident_cont}*
108114
/* This code is inserted at the start of replication_yylex() */
109115

110116
/* If we have a pushed-back token, return that. */
111-
if (repl_pushed_back_token)
117+
if (yyextra->repl_pushed_back_token)
112118
{
113-
intresult = repl_pushed_back_token;
119+
intresult =yyextra->repl_pushed_back_token;
114120

115-
repl_pushed_back_token =0;
121+
yyextra->repl_pushed_back_token =0;
116122
return result;
117123
}
118124
%}
@@ -142,63 +148,63 @@ UPLOAD_MANIFEST{ return K_UPLOAD_MANIFEST; }
142148
{space}+{/* do nothing */ }
143149

144150
{digit}+{
145-
replication_yylval.uintval =strtoul(yytext,NULL,10);
151+
yylval->uintval =strtoul(yytext,NULL,10);
146152
return UCONST;
147153
}
148154

149155
{hexdigit}+\/{hexdigit}+{
150156
uint32hi,
151157
lo;
152158
if (sscanf(yytext,"%X/%X", &hi, &lo) !=2)
153-
replication_yyerror("invalid streaming start location");
154-
replication_yylval.recptr = ((uint64) hi) <<32 | lo;
159+
replication_yyerror(yyscanner,"invalid streaming start location");
160+
yylval->recptr = ((uint64) hi) <<32 | lo;
155161
return RECPTR;
156162
}
157163

158164
{xqstart}{
159165
BEGIN(xq);
160-
startlit();
166+
startlit(yyscanner);
161167
}
162168

163169
<xq>{quotestop}{
164170
yyless(1);
165171
BEGIN(INITIAL);
166-
replication_yylval.str =litbufdup();
172+
yylval->str =litbufdup(yyscanner);
167173
return SCONST;
168174
}
169175

170176
<xq>{xqdouble}{
171-
addlitchar('\'');
177+
addlitchar('\'', yyscanner);
172178
}
173179

174180
<xq>{xqinside} {
175-
addlit(yytext, yyleng);
181+
addlit(yytext, yyleng, yyscanner);
176182
}
177183

178184
{xdstart}{
179185
BEGIN(xd);
180-
startlit();
186+
startlit(yyscanner);
181187
}
182188

183189
<xd>{xdstop}{
184190
intlen;
185191

186192
yyless(1);
187193
BEGIN(INITIAL);
188-
replication_yylval.str =litbufdup();
189-
len =strlen(replication_yylval.str);
190-
truncate_identifier(replication_yylval.str, len,true);
194+
yylval->str =litbufdup(yyscanner);
195+
len =strlen(yylval->str);
196+
truncate_identifier(yylval->str, len,true);
191197
return IDENT;
192198
}
193199

194200
<xd>{xdinside} {
195-
addlit(yytext, yyleng);
201+
addlit(yytext, yyleng, yyscanner);
196202
}
197203

198204
{identifier}{
199205
intlen =strlen(yytext);
200206

201-
replication_yylval.str =downcase_truncate_identifier(yytext, len,true);
207+
yylval->str =downcase_truncate_identifier(yytext, len,true);
202208
return IDENT;
203209
}
204210

@@ -207,7 +213,7 @@ UPLOAD_MANIFEST{ return K_UPLOAD_MANIFEST; }
207213
return yytext[0];
208214
}
209215

210-
<xq,xd><<EOF>>{replication_yyerror("unterminated quoted string"); }
216+
<xq,xd><<EOF>>{replication_yyerror(yyscanner,"unterminated quoted string"); }
211217

212218

213219
<<EOF>>{
@@ -218,68 +224,63 @@ UPLOAD_MANIFEST{ return K_UPLOAD_MANIFEST; }
218224

219225
/* LCOV_EXCL_STOP*/
220226

227+
/* see scan.l*/
228+
#undef yyextra
229+
#defineyyextra (((structyyguts_t *) yyscanner)->yyextra_r)
230+
221231
staticvoid
222-
startlit(void)
232+
startlit(yyscan_t yyscanner)
223233
{
224-
initStringInfo(&litbuf);
234+
initStringInfo(&yyextra->litbuf);
225235
}
226236

227237
staticchar *
228-
litbufdup(void)
238+
litbufdup(yyscan_t yyscanner)
229239
{
230-
return litbuf.data;
240+
returnyyextra->litbuf.data;
231241
}
232242

233243
staticvoid
234-
addlit(char *ytext,int yleng)
244+
addlit(char *ytext,int yleng,yyscan_t yyscanner)
235245
{
236-
appendBinaryStringInfo(&litbuf, ytext, yleng);
246+
appendBinaryStringInfo(&yyextra->litbuf, ytext, yleng);
237247
}
238248

239249
staticvoid
240-
addlitchar(unsignedchar ychar)
250+
addlitchar(unsignedchar ychar,yyscan_t yyscanner)
241251
{
242-
appendStringInfoChar(&litbuf, ychar);
252+
appendStringInfoChar(&yyextra->litbuf, ychar);
243253
}
244254

245255
void
246-
replication_yyerror(constchar *message)
256+
replication_yyerror(yyscan_t yyscanner,constchar *message)
247257
{
248258
ereport(ERROR,
249259
(errcode(ERRCODE_SYNTAX_ERROR),
250260
errmsg_internal("%s", message)));
251261
}
252262

253263
void
254-
replication_scanner_init(constchar *str)
264+
replication_scanner_init(constchar *str,yyscan_t *yyscannerp)
255265
{
256-
Sizeslen =strlen(str);
257-
char *scanbuf;
258-
259-
/*
260-
* Might be left over after ereport()
261-
*/
262-
if (YY_CURRENT_BUFFER)
263-
yy_delete_buffer(YY_CURRENT_BUFFER);
264-
265-
/*
266-
* Make a scan buffer with special termination needed by flex.
267-
*/
268-
scanbuf = (char *)palloc(slen +2);
269-
memcpy(scanbuf, str, slen);
270-
scanbuf[slen] = scanbuf[slen +1] = YY_END_OF_BUFFER_CHAR;
271-
scanbufhandle =yy_scan_buffer(scanbuf, slen +2);
272-
273-
/* Make sure we start in proper state */
274-
BEGIN(INITIAL);
275-
repl_pushed_back_token =0;
266+
yyscan_tyyscanner;
267+
structreplication_yy_extra_type *yyext =palloc0_object(structreplication_yy_extra_type);
268+
269+
if (yylex_init(yyscannerp) !=0)
270+
elog(ERROR,"yylex_init() failed: %m");
271+
272+
yyscanner = *yyscannerp;
273+
274+
yyset_extra(yyext, yyscanner);
275+
276+
yy_scan_string(str, yyscanner);
276277
}
277278

278279
void
279-
replication_scanner_finish(void)
280+
replication_scanner_finish(yyscan_t yyscanner)
280281
{
281-
yy_delete_buffer(scanbufhandle);
282-
scanbufhandle =NULL;
282+
pfree(yyextra);
283+
yylex_destroy(yyscanner);
283284
}
284285

285286
/*
@@ -291,9 +292,10 @@ replication_scanner_finish(void)
291292
* IDENT token here, although some other cases are possible.
292293
*/
293294
bool
294-
replication_scanner_is_replication_command(void)
295+
replication_scanner_is_replication_command(yyscan_t yyscanner)
295296
{
296-
intfirst_token =replication_yylex();
297+
YYSTYPEdummy;
298+
intfirst_token =replication_yylex(&dummy, yyscanner);
297299

298300
switch (first_token)
299301
{
@@ -308,10 +310,37 @@ replication_scanner_is_replication_command(void)
308310
case K_UPLOAD_MANIFEST:
309311
case K_SHOW:
310312
/* Yes; push back the first token so we can parse later. */
311-
repl_pushed_back_token = first_token;
313+
yyextra->repl_pushed_back_token = first_token;
312314
returntrue;
313315
default:
314316
/* Nope; we don't bother to push back the token. */
315317
returnfalse;
316318
}
317319
}
320+
321+
/*
322+
* Interface functions to make flex use palloc() instead of malloc().
323+
* It'd be better to make these static, but flex insists otherwise.
324+
*/
325+
326+
void *
327+
yyalloc(yy_size_t size,yyscan_t yyscanner)
328+
{
329+
returnpalloc(size);
330+
}
331+
332+
void *
333+
yyrealloc(void *ptr,yy_size_t size,yyscan_t yyscanner)
334+
{
335+
if (ptr)
336+
returnrepalloc(ptr, size);
337+
else
338+
returnpalloc(size);
339+
}
340+
341+
void
342+
yyfree(void *ptr,yyscan_t yyscanner)
343+
{
344+
if (ptr)
345+
pfree(ptr);
346+
}

‎src/backend/replication/walsender.c

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1951,6 +1951,7 @@ WalSndWaitForWal(XLogRecPtr loc)
19511951
bool
19521952
exec_replication_command(constchar*cmd_string)
19531953
{
1954+
yyscan_tscanner;
19541955
intparse_rc;
19551956
Node*cmd_node;
19561957
constchar*cmdtag;
@@ -1990,15 +1991,15 @@ exec_replication_command(const char *cmd_string)
19901991
ALLOCSET_DEFAULT_SIZES);
19911992
old_context=MemoryContextSwitchTo(cmd_context);
19921993

1993-
replication_scanner_init(cmd_string);
1994+
replication_scanner_init(cmd_string,&scanner);
19941995

19951996
/*
19961997
* Is it a WalSender command?
19971998
*/
1998-
if (!replication_scanner_is_replication_command())
1999+
if (!replication_scanner_is_replication_command(scanner))
19992000
{
20002001
/* Nope; clean up and get out. */
2001-
replication_scanner_finish();
2002+
replication_scanner_finish(scanner);
20022003

20032004
MemoryContextSwitchTo(old_context);
20042005
MemoryContextDelete(cmd_context);
@@ -2016,13 +2017,13 @@ exec_replication_command(const char *cmd_string)
20162017
/*
20172018
* Looks like a WalSender command, so parse it.
20182019
*/
2019-
parse_rc=replication_yyparse();
2020+
parse_rc=replication_yyparse(scanner);
20202021
if (parse_rc!=0)
20212022
ereport(ERROR,
20222023
(errcode(ERRCODE_SYNTAX_ERROR),
20232024
errmsg_internal("replication command parser returned %d",
20242025
parse_rc)));
2025-
replication_scanner_finish();
2026+
replication_scanner_finish(scanner);
20262027

20272028
cmd_node=replication_parse_result;
20282029

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp