|
4 | 4 |
|
5 | 5 | from .documentimportDocument |
6 | 6 | from .loggerimportlogger |
7 | | - |
| 7 | +fromcollections.abcimportIterable |
8 | 8 |
|
9 | 9 | classDocuments(object): |
10 | 10 | RESOURCE_PATH='documents' |
@@ -52,23 +52,37 @@ def import_jsonl(self, documents_jsonl): |
52 | 52 |
|
53 | 53 | # `documents` can be either a list of document objects (or) |
54 | 54 | # JSONL-formatted string containing multiple documents |
55 | | -defimport_(self,documents,params=None): |
56 | | -ifisinstance(documents,list): |
57 | | -document_strs= [] |
58 | | -fordocumentindocuments: |
59 | | -document_strs.append(json.dumps(document)) |
60 | | - |
61 | | -docs_import='\n'.join(document_strs) |
62 | | -api_response=self.api_call.post(self._endpoint_path('import'),docs_import,params,as_json=False) |
63 | | -res_obj_strs=api_response.split('\n') |
64 | | - |
65 | | -response_objs= [] |
66 | | -forres_obj_strinres_obj_strs: |
67 | | -try: |
68 | | -res_obj_json=json.loads(res_obj_str) |
69 | | -exceptjson.JSONDecodeErrorase: |
70 | | -raiseTypesenseClientError("Invalid response")frome |
71 | | -response_objs.append(res_obj_json) |
| 55 | +defimport_(self,documents,params=None,batch_size=None): |
| 56 | +ifisinstance(documents,Iterable): |
| 57 | +ifbatch_size: |
| 58 | +response_objs= [] |
| 59 | +batch= [] |
| 60 | +fordocumentindocuments: |
| 61 | +batch.append(document) |
| 62 | +if (len(batch)==batch_size): |
| 63 | +api_response=self.import_(batch,params) |
| 64 | +response_objs.extend(api_response) |
| 65 | +batch= [] |
| 66 | +ifbatch: |
| 67 | +api_response=self.import_(batch,params) |
| 68 | +response_objs.extend(api_response) |
| 69 | + |
| 70 | +else: |
| 71 | +document_strs= [] |
| 72 | +fordocumentindocuments: |
| 73 | +document_strs.append(json.dumps(document)) |
| 74 | + |
| 75 | +docs_import='\n'.join(document_strs) |
| 76 | +api_response=self.api_call.post(self._endpoint_path('import'),docs_import,params,as_json=False) |
| 77 | +res_obj_strs=api_response.split('\n') |
| 78 | + |
| 79 | +response_objs= [] |
| 80 | +forres_obj_strinres_obj_strs: |
| 81 | +try: |
| 82 | +res_obj_json=json.loads(res_obj_str) |
| 83 | +exceptjson.JSONDecodeErrorase: |
| 84 | +raiseTypesenseClientError("Invalid response")frome |
| 85 | +response_objs.append(res_obj_json) |
72 | 86 |
|
73 | 87 | returnresponse_objs |
74 | 88 | else: |
|