Commitac6576e

committed

Added hnsw.search_mem_multiplier option

1 parent67eff41 commitac6576eCopy full SHA for ac6576e

File tree

+15

-6

lines changed

+15

-6

lines changed

Lines changed: 6 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,7 @@ static const struct config_enum_entry hnsw_iterative_search_options[] = {`
`28`	`28`	`inthnsw_ef_search;`
`29`	`29`	`inthnsw_iterative_search;`
`30`	`30`	`inthnsw_max_search_tuples;`
	`31`	`+doublehnsw_search_mem_multiplier;`
`31`	`32`	`inthnsw_lock_tranche_id;`
`32`	`33`	`staticrelopt_kindhnsw_relopt_kind;`
`33`	`34`
`@@ -87,6 +88,11 @@ HnswInit(void)`
`87`	`88`	`NULL,&hnsw_max_search_tuples,`
`88`	`89`	`20000,1,INT_MAX,PGC_USERSET,0,NULL,NULL,NULL);`
`89`	`90`
	`91`	`+/* Same range and default as hash_mem_multiplier */`
	`92`	`+DefineCustomRealVariable("hnsw.search_mem_multiplier","Sets the multiple of work_mem to use for iterative search",`
	`93`	`+NULL,&hnsw_search_mem_multiplier,`
	`94`	`+2,1,1000,PGC_USERSET,0,NULL,NULL,NULL);`
	`95`	`+`
`90`	`96`	`MarkGUCPrefixReserved("hnsw");`
`91`	`97`	`}`
`92`	`98`

Lines changed: 2 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -111,6 +111,7 @@`
`111`	`111`	`externinthnsw_ef_search;`
`112`	`112`	`externinthnsw_iterative_search;`
`113`	`113`	`externinthnsw_max_search_tuples;`
	`114`	`+externdoublehnsw_search_mem_multiplier;`
`114`	`115`	`externinthnsw_lock_tranche_id;`
`115`	`116`
`116`	`117`	`typedefenumHnswIterativeSearchMode`
`@@ -372,6 +373,7 @@ typedef struct HnswScanOpaqueData`
`372`	`373`	`intm;`
`373`	`374`	`int64tuples;`
`374`	`375`	`doublepreviousDistance;`
	`376`	`+SizemaxMemory;`
`375`	`377`	`MemoryContexttmpCtx;`
`376`	`378`
`377`	`379`	`/* Support functions */`

Lines changed: 5 additions & 3 deletions

Original file line number	Diff line number	Diff line change
`@@ -138,6 +138,8 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)`
`138`	`138`	`/* Set support functions */`
`139`	`139`	`HnswInitSupport(&so->support,index);`
`140`	`140`
	`141`	`+so->maxMemory= (Size) (work_mem1024.0hnsw_search_mem_multiplier);`
	`142`	`+`
`141`	`143`	`scan->opaque=so;`
`142`	`144`
`143`	`145`	`returnscan;`
`@@ -244,13 +246,13 @@ hnswgettuple(IndexScanDesc scan, ScanDirection dir)`
`244`	`246`	`so->w=lappend(so->w,HnswGetSearchCandidate(w_node,pairingheap_remove_first(so->discarded)));`
`245`	`247`	`}`
`246`	`248`	`/* Prevent scans from consuming too much memory */`
`247`		`-elseif (MemoryContextMemAllocated(so->tmpCtx, false)> (Size)work_mem*1024L)`
	`249`	`+elseif (MemoryContextMemAllocated(so->tmpCtx, false) >=so->maxMemory)`
`248`	`250`	`{`
`249`	`251`	`if (pairingheap_is_empty(so->discarded))`
`250`	`252`	`{`
`251`	`253`	`ereport(DEBUG1,`
`252`		`-(errmsg("hnsw index scanexceeded work_mem after "INT64_FORMAT" tuples",so->tuples),`
`253`		`-errhint("Increasework_mem to scan more tuples.")));`
	`254`	`+(errmsg("hnsw index scanreached memory limit after "INT64_FORMAT" tuples",so->tuples),`
	`255`	`+errhint("Increasehnsw.search_mem_multiplier to scan more tuples.")));`
`254`	`256`
`255`	`257`	`break;`
`256`	`258`	`}`

Lines changed: 2 additions & 3 deletions

Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,6 @@`
`28`	`28`	`SET enable_seqscan = off;`
`29`	`29`	`SET hnsw.iterative_search = relaxed_order;`
`30`	`30`	`SET hnsw.max_search_tuples = 100000;`
`31`		`-SET work_mem = '8MB';`
`32`	`31`	`SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;`
`33`	`32`	`));`
`34`	`33`	`is($count, 10);`
`@@ -45,7 +44,6 @@`
`45`	`44`	`SET enable_seqscan = off;`
`46`	`45`	`SET hnsw.iterative_search = relaxed_order;`
`47`	`46`	`SET hnsw.max_search_tuples =$max_tuples;`
`48`		`-SET work_mem = '8MB';`
`49`	`47`	`SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst WHERE i =$i) LIMIT 11) t;`
`50`	`48`	`));`
`51`	`49`	`$sum +=$count;`
`@@ -61,8 +59,9 @@`
`61`	`59`	`SET hnsw.iterative_search = relaxed_order;`
`62`	`60`	`SET client_min_messages = debug1;`
`63`	`61`	`SET work_mem = '1MB';`
	`62`	`+SET hnsw.search_mem_multiplier = 1;`
`64`	`63`	`SELECT COUNT(*) FROM (SELECT v FROM tst WHERE i % 10000 = 0 ORDER BY v <-> (SELECT v FROM tst LIMIT 1) LIMIT 11) t;`
`65`	`64`	`));`
`66`		`-like($stderr,qr/hnsw index scanexceeded work_mem after\d+ tuples/);`
	`65`	`+like($stderr,qr/hnsw index scanreached memory limit after\d+ tuples/);`
`67`	`66`
`68`	`67`	`done_testing();`

Comments

(0)