Commit4c52bc9

authored

SDK - Patch parallel uploads (#1468)

1 parente79f1ca commit4c52bc9Copy full SHA for 4c52bc9

File tree

+21

-20

lines changed

+21

-20

lines changed

Lines changed: 14 additions & 16 deletions

Original file line number	Diff line number	Diff line change
`@@ -533,23 +533,21 @@ impl Collection {`
`533`	`533`
`534`	`534`	`letmut set =JoinSet::new();`
`535`	`535`	`for batchin documents.chunks(batch_sizeasusize){`
`536`		`-if set.len() < parallel_batches{`
`537`		`-let local_self =self.clone();`
`538`		`-let local_batch = batch.to_owned();`
`539`		`-let local_args = args.clone();`
`540`		`-let local_pipelines = pipelines.clone();`
`541`		`-let local_pool = pool.clone();`
`542`		`- set.spawn(asyncmove{`
`543`		`- local_self`
`544`		`-._upsert_documents(local_batch, local_args, local_pipelines, local_pool)`
`545`		`-.await`
`546`		`-});`
`547`		`-}else{`
`548`		`-ifletSome(res) = set.join_next().await{`
`549`		`- res??;`
`550`		`- progress_bar.inc(batch_size);`
`551`		`-}`
	`536`	`+if set.len() >= parallel_batches{`
	`537`	`+ set.join_next().await.unwrap()??;`
	`538`	`+ progress_bar.inc(batch_size);`
`552`	`539`	`}`
	`540`	`+`
	`541`	`+let local_self =self.clone();`
	`542`	`+let local_batch = batch.to_owned();`
	`543`	`+let local_args = args.clone();`
	`544`	`+let local_pipelines = pipelines.clone();`
	`545`	`+let local_pool = pool.clone();`
	`546`	`+ set.spawn(asyncmove{`
	`547`	`+ local_self`
	`548`	`+._upsert_documents(local_batch, local_args, local_pipelines, local_pool)`
	`549`	`+.await`
	`550`	`+});`
`553`	`551`	`}`
`554`	`552`
`555`	`553`	`whileletSome(res) = set.join_next().await{`

Lines changed: 7 additions & 4 deletions

Original file line number	Diff line number	Diff line change
`@@ -442,7 +442,10 @@ mod tests {`
`442`	`442`	`json!({`
`443`	`443`	`"title":{`
`444`	`444`	`"semantic_search":{`
`445`		`-"model":"intfloat/e5-small"`
	`445`	`+"model":"intfloat/e5-small-v2",`
	`446`	`+"parameters":{`
	`447`	`+"prompt":"passage: "`
	`448`	`+}`
`446`	`449`	`}`
`447`	`450`	`},`
`448`	`451`	`"body":{`
`@@ -454,9 +457,9 @@ mod tests {`
`454`	`457`	`}`
`455`	`458`	`},`
`456`	`459`	`"semantic_search":{`
`457`		`-"model":"hkunlp/instructor-base",`
	`460`	`+"model":"intfloat/e5-small-v2",`
`458`	`461`	`"parameters":{`
`459`		`-"instruction":"Represent the Wikipedia document for retrieval"`
	`462`	`+"prompt":"passage:"`
`460`	`463`	`}`
`461`	`464`	`},`
`462`	`465`	`"full_text_search":{`
`@@ -475,7 +478,7 @@ mod tests {`
`475`	`478`	`documents.clone(),`
`476`	`479`	`Some(`
`477`	`480`	`json!({`
`478`		`-"batch_size":4,`
	`481`	`+"batch_size":2,`
`479`	`482`	`"parallel_batches":5`
`480`	`483`	`})`
`481`	`484`	`.into(),`

Comments

(0)