Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitea7b4e9

Browse files
committed
Add support for incrementally parsing backup manifests
This adds the infrastructure for using the new non-recursive JSON parserin processing manifests. It's important that callers make sure that thelast piece of json handed to the incremental manifest parser containsthe entire last few lines of the manifest, including the checksum.Author: Andrew DunstanReviewed-By: Jacob ChampionDiscussion:https://postgr.es/m/7b0a51d6-0d9d-7366-3a1a-f74397a02f55@dunslane.net
1 parent3311ea8 commitea7b4e9

File tree

2 files changed

+118
-8
lines changed

2 files changed

+118
-8
lines changed

‎src/common/parse_manifest.c

Lines changed: 113 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,13 @@ typedef struct
9191
char*manifest_checksum;
9292
}JsonManifestParseState;
9393

94+
typedefstructJsonManifestParseIncrementalState
95+
{
96+
JsonLexContextlex;
97+
JsonSemActionsem;
98+
pg_cryptohash_ctx*manifest_ctx;
99+
}JsonManifestParseIncrementalState;
100+
94101
staticJsonParseErrorTypejson_manifest_object_start(void*state);
95102
staticJsonParseErrorTypejson_manifest_object_end(void*state);
96103
staticJsonParseErrorTypejson_manifest_array_start(void*state);
@@ -104,14 +111,99 @@ static void json_manifest_finalize_system_identifier(JsonManifestParseState *par
104111
staticvoidjson_manifest_finalize_file(JsonManifestParseState*parse);
105112
staticvoidjson_manifest_finalize_wal_range(JsonManifestParseState*parse);
106113
staticvoidverify_manifest_checksum(JsonManifestParseState*parse,
107-
char*buffer,size_tsize);
114+
char*buffer,size_tsize,
115+
pg_cryptohash_ctx*incr_ctx);
108116
staticvoidjson_manifest_parse_failure(JsonManifestParseContext*context,
109117
char*msg);
110118

111119
staticinthexdecode_char(charc);
112120
staticboolhexdecode_string(uint8*result,char*input,intnbytes);
113121
staticboolparse_xlogrecptr(XLogRecPtr*result,char*input);
114122

123+
/*
124+
* Set up for incremental parsing of the manifest.
125+
*
126+
*/
127+
128+
JsonManifestParseIncrementalState*
129+
json_parse_manifest_incremental_init(JsonManifestParseContext*context)
130+
{
131+
JsonManifestParseIncrementalState*incstate;
132+
JsonManifestParseState*parse;
133+
pg_cryptohash_ctx*manifest_ctx;
134+
135+
incstate=palloc(sizeof(JsonManifestParseIncrementalState));
136+
parse=palloc(sizeof(JsonManifestParseState));
137+
138+
parse->context=context;
139+
parse->state=JM_EXPECT_TOPLEVEL_START;
140+
parse->saw_version_field= false;
141+
142+
makeJsonLexContextIncremental(&(incstate->lex),PG_UTF8, true);
143+
144+
incstate->sem.semstate=parse;
145+
incstate->sem.object_start=json_manifest_object_start;
146+
incstate->sem.object_end=json_manifest_object_end;
147+
incstate->sem.array_start=json_manifest_array_start;
148+
incstate->sem.array_end=json_manifest_array_end;
149+
incstate->sem.object_field_start=json_manifest_object_field_start;
150+
incstate->sem.object_field_end=NULL;
151+
incstate->sem.array_element_start=NULL;
152+
incstate->sem.array_element_end=NULL;
153+
incstate->sem.scalar=json_manifest_scalar;
154+
155+
manifest_ctx=pg_cryptohash_create(PG_SHA256);
156+
if (manifest_ctx==NULL)
157+
context->error_cb(context,"out of memory");
158+
if (pg_cryptohash_init(manifest_ctx)<0)
159+
context->error_cb(context,"could not initialize checksum of manifest");
160+
incstate->manifest_ctx=manifest_ctx;
161+
162+
returnincstate;
163+
}
164+
165+
/*
166+
* parse the manifest in pieces.
167+
*
168+
* The caller must ensure that the final piece contains the final lines
169+
* with the complete checksum.
170+
*/
171+
172+
void
173+
json_parse_manifest_incremental_chunk(
174+
JsonManifestParseIncrementalState*incstate,char*chunk,intsize,
175+
boolis_last)
176+
{
177+
JsonParseErrorTyperes,
178+
expected;
179+
JsonManifestParseState*parse=incstate->sem.semstate;
180+
JsonManifestParseContext*context=parse->context;
181+
182+
res=pg_parse_json_incremental(&(incstate->lex),&(incstate->sem),
183+
chunk,size,is_last);
184+
185+
expected=is_last ?JSON_SUCCESS :JSON_INCOMPLETE;
186+
187+
if (res!=expected)
188+
json_manifest_parse_failure(context,
189+
json_errdetail(res,&(incstate->lex)));
190+
191+
if (is_last&&parse->state!=JM_EXPECT_EOF)
192+
json_manifest_parse_failure(context,"manifest ended unexpectedly");
193+
194+
if (!is_last)
195+
{
196+
if (pg_cryptohash_update(incstate->manifest_ctx,
197+
(uint8*)chunk,size)<0)
198+
context->error_cb(context,"could not update checksum of manifest");
199+
}
200+
else
201+
{
202+
verify_manifest_checksum(parse,chunk,size,incstate->manifest_ctx);
203+
}
204+
}
205+
206+
115207
/*
116208
* Main entrypoint to parse a JSON-format backup manifest.
117209
*
@@ -157,7 +249,7 @@ json_parse_manifest(JsonManifestParseContext *context, char *buffer,
157249
json_manifest_parse_failure(context,"manifest ended unexpectedly");
158250

159251
/* Verify the manifest checksum. */
160-
verify_manifest_checksum(&parse,buffer,size);
252+
verify_manifest_checksum(&parse,buffer,size,NULL);
161253

162254
freeJsonLexContext(lex);
163255
}
@@ -390,6 +482,8 @@ json_manifest_object_field_start(void *state, char *fname, bool isnull)
390482
break;
391483
}
392484

485+
pfree(fname);
486+
393487
returnJSON_SUCCESS;
394488
}
395489

@@ -698,10 +792,14 @@ json_manifest_finalize_wal_range(JsonManifestParseState *parse)
698792
* The last line of the manifest file is excluded from the manifest checksum,
699793
* because the last line is expected to contain the checksum that covers
700794
* the rest of the file.
795+
*
796+
* For an incremental parse, this will just be called on the last chunk of the
797+
* manifest, and the cryptohash context paswed in. For a non-incremental
798+
* parse incr_ctx will be NULL.
701799
*/
702800
staticvoid
703801
verify_manifest_checksum(JsonManifestParseState*parse,char*buffer,
704-
size_tsize)
802+
size_tsize,pg_cryptohash_ctx*incr_ctx)
705803
{
706804
JsonManifestParseContext*context=parse->context;
707805
size_ti;
@@ -736,11 +834,18 @@ verify_manifest_checksum(JsonManifestParseState *parse, char *buffer,
736834
"last line not newline-terminated");
737835

738836
/* Checksum the rest. */
739-
manifest_ctx=pg_cryptohash_create(PG_SHA256);
740-
if (manifest_ctx==NULL)
741-
context->error_cb(context,"out of memory");
742-
if (pg_cryptohash_init(manifest_ctx)<0)
743-
context->error_cb(context,"could not initialize checksum of manifest");
837+
if (incr_ctx==NULL)
838+
{
839+
manifest_ctx=pg_cryptohash_create(PG_SHA256);
840+
if (manifest_ctx==NULL)
841+
context->error_cb(context,"out of memory");
842+
if (pg_cryptohash_init(manifest_ctx)<0)
843+
context->error_cb(context,"could not initialize checksum of manifest");
844+
}
845+
else
846+
{
847+
manifest_ctx=incr_ctx;
848+
}
744849
if (pg_cryptohash_update(manifest_ctx, (uint8*)buffer,penultimate_newline+1)<0)
745850
context->error_cb(context,"could not update checksum of manifest");
746851
if (pg_cryptohash_final(manifest_ctx,manifest_checksum_actual,

‎src/include/common/parse_manifest.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
structJsonManifestParseContext;
2222
typedefstructJsonManifestParseContextJsonManifestParseContext;
23+
typedefstructJsonManifestParseIncrementalStateJsonManifestParseIncrementalState;
2324

2425
typedefvoid (*json_manifest_version_callback) (JsonManifestParseContext*,
2526
intmanifest_version);
@@ -48,5 +49,9 @@ struct JsonManifestParseContext
4849

4950
externvoidjson_parse_manifest(JsonManifestParseContext*context,
5051
char*buffer,size_tsize);
52+
externJsonManifestParseIncrementalState*json_parse_manifest_incremental_init(JsonManifestParseContext*context);
53+
externvoidjson_parse_manifest_incremental_chunk(
54+
JsonManifestParseIncrementalState*incstate,char*chunk,intsize,
55+
boolis_last);
5156

5257
#endif

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp