Commit511a4a2

authored

refactor: stream batched items to db during backfill (#229)

Streams and upserts in chunks as items arrive, instead of collecting all first.- Chunks of 250 items- Upsert runs while fetch continues- Logging: removes mid-sync count log (total unknown until done)

1 parent8fe9847 commit511a4a2Copy full SHA for 511a4a2

File tree

1 file changed

+14

-12

lines changed

packages/sync-engine/src
- stripeSync.ts

1 file changed

+14

-12

lines changed

`‎packages/sync-engine/src/stripeSync.ts‎`

Lines changed: 14 additions & 12 deletions

Original file line number	Diff line number	Diff line change
`@@ -1009,25 +1009,27 @@ export class StripeSync {`
`1009`	`1009`	`fetch:()=>Stripe.ApiListPromise<T>,`
`1010`	`1010`	`upsert:(items:T[])=>Promise<T[]>`
`1011`	`1011`	`):Promise<Sync>{`
`1012`		`-constitems:T[]=[]`
	`1012`	`+constchunkSize=250`
	`1013`	`+letchunk:T[]=[]`
	`1014`	`+letsynced=0`
`1013`	`1015`
`1014`	`1016`	`this.config.logger?.info('Fetching items to sync from Stripe')`
`1015`	`1017`	`forawait(constitemoffetch()){`
`1016`		`-items.push(item)`
`1017`		`-}`
	`1018`	`+chunk.push(item)`
	`1019`	`+synced++`
	`1020`	+if(synced%1000===0)this.config.logger?.info(`Synced${synced} items`)
`1018`	`1021`
`1019`		`-if(!items.length)return{synced:0}`
	`1022`	`+if(chunk.length>=chunkSize){`
	`1023`	`+awaitupsert(chunk)`
	`1024`	`+chunk=[]`
	`1025`	`+}`
	`1026`	`+}`
`1020`	`1027`
`1021`		-this.config.logger?.info(`Upserting${items.length} items`)
`1022`		`-constchunkSize=250`
`1023`		`-for(leti=0;i<items.length;i+=chunkSize){`
`1024`		`-constchunk=items.slice(i,i+chunkSize)`
	`1028`	`+if(chunk.length>0)awaitupsert(chunk)`
`1025`	`1029`
`1026`		`-awaitupsert(chunk)`
`1027`		`-}`
`1028`		`-this.config.logger?.info('Upserted items')`
	`1030`	+this.config.logger?.info(`Upserted${synced} items`)
`1029`	`1031`
`1030`		`-return{synced:items.length}`
	`1032`	`+return{ synced}`
`1031`	`1033`	`}`
`1032`	`1034`
`1033`	`1035`	`privateasyncupsertCharges(`

0 commit comments

Comments

(0)

Movatterモバイル変換

Navigation Menu

Search code, repositories, users, issues, pull requests...

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Uh oh!

Commit511a4a2

File tree

1 file changed

1 file changed

`‎packages/sync-engine/src/stripeSync.ts‎`

0 commit comments