Commit1323a84

and

authored

For max tokens using max new tokens instead of max_length (#1220)

Co-authored-by: Santi Adavani <santis@gmail.com>

1 parent343febe commit1323a84Copy full SHA for 1323a84

File tree

-2

lines changed

-2

lines changed

Lines changed: 2 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -236,7 +236,7 @@ mistralai/Mistral-7B-v0.1`
`236`	`236`	`let md5_digest = md5::compute(to_hash.as_bytes());`
`237`	`237`	`let fingerprint = uuid::Uuid::from_slice(&md5_digest.0)?;`
`238`	`238`
`239`		`-letmut args = serde_json::json!({"max_length": max_tokens,"temperature": temperature,"do_sample":true,"num_return_sequences": n});`
	`239`	`+letmut args = serde_json::json!({"max_new_tokens": max_tokens,"temperature": temperature,"do_sample":true,"num_return_sequences": n});`
`240`	`240`	`ifletSome(t) = chat_template`
`241`	`241`	`.or_else(\|\|try_get_model_chat_template(&model_name).map(\|s\| s.to_string()))`
`242`	`242`	`{`
`@@ -322,7 +322,7 @@ mistralai/Mistral-7B-v0.1`
`322`	`322`	`let md5_digest = md5::compute(to_hash.as_bytes());`
`323`	`323`	`let fingerprint = uuid::Uuid::from_slice(&md5_digest.0)?;`
`324`	`324`
`325`		`-letmut args = serde_json::json!({"max_length": max_tokens,"temperature": temperature,"do_sample":true,"num_return_sequences": n});`
	`325`	`+letmut args = serde_json::json!({"max_new_tokens": max_tokens,"temperature": temperature,"do_sample":true,"num_return_sequences": n});`
`326`	`326`	`ifletSome(t) = chat_template`
`327`	`327`	`.or_else(\|\|try_get_model_chat_template(&model_name).map(\|s\| s.to_string()))`
`328`	`328`	`{`

Comments

(0)