Commit2abe945

authored

fix: optimize the handling of CLIP embedding weight (leejet#840)

1 parentf3140ea commit2abe945Copy full SHA for 2abe945

File tree

-4

lines changed

-4

lines changed

Lines changed: 5 additions & 4 deletions

Original file line number	Diff line number	Diff line change
`@@ -553,12 +553,13 @@ class CLIPEmbeddings : public GGMLBlock {`
`553`	`553`	`voidinit_params(structggml_context* ctx,const String2GGMLType& tensor_types = {},const std::string prefix ="") {`
`554`	`554`	`enum ggml_type token_wtype = GGML_TYPE_F32;`
`555`	`555`	`if (!force_clip_f32) {`
`556`		`-auto tensor_type = tensor_types.find(prefix +"token_embedding.weight");`
`557`		`-if (tensor_type != tensor_types.end())`
	`556`	`+auto tensor_type = tensor_types.find(prefix +"token_embedding.weight");`
	`557`	`+ std::set<ggml_type> allow_types = {GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0};`
	`558`	`+if (tensor_type != tensor_types.end() && allow_types.find(tensor_type->second) != allow_types.end()) {`
`558`	`559`	`token_wtype = tensor_type->second;`
	`560`	`+ }`
`559`	`561`	`}`
`560`		`-enum ggml_type position_wtype = GGML_TYPE_F32;`
`561`		`-`
	`562`	`+enum ggml_type position_wtype = GGML_TYPE_F32;`
`562`	`563`	`params["token_embedding.weight"] =ggml_new_tensor_2d(ctx, token_wtype, embed_dim, vocab_size);`
`563`	`564`	`params["position_embedding.weight"] =ggml_new_tensor_2d(ctx, position_wtype, embed_dim, num_positions);`
`564`	`565`	`}`

Lines changed: 2 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -2422,6 +2422,8 @@ bool ModelLoader::tensor_should_be_converted(const TensorStorage& tensor_storage`
`2422`	`2422`	`// Pass, do not convert. For MMDiT`
`2423`	`2423`	`}elseif (contains(name,"time_embed.") \|\|contains(name,"label_emb.")) {`
`2424`	`2424`	`// Pass, do not convert. For Unet`
	`2425`	`+ }elseif (contains(name,"embedding")) {`
	`2426`	`+// Pass, do not convert embedding`
`2425`	`2427`	`}else {`
`2426`	`2428`	`returntrue;`
`2427`	`2429`	`}`

Comments

(0)