👉🏻 CosyVoice 👈🏻

Highlight🔥

Multilingual

Ultra-Low Latency

High Accuracy

Strong Stability

Natural Experience

Roadmap

Install

Clone and install

Model download

# SDK模型下载frommodelscopeimportsnapshot_downloadsnapshot_download('iic/CosyVoice2-0.5B',local_dir='pretrained_models/CosyVoice2-0.5B')snapshot_download('iic/CosyVoice-300M',local_dir='pretrained_models/CosyVoice-300M')snapshot_download('iic/CosyVoice-300M-SFT',local_dir='pretrained_models/CosyVoice-300M-SFT')snapshot_download('iic/CosyVoice-300M-Instruct',local_dir='pretrained_models/CosyVoice-300M-Instruct')snapshot_download('iic/CosyVoice-ttsfrd',local_dir='pretrained_models/CosyVoice-ttsfrd')

# git模型下载，请确保已安装git lfsmkdir -p pretrained_modelsgit clone https://www.modelscope.cn/iic/CosyVoice2-0.5B.git pretrained_models/CosyVoice2-0.5Bgit clone https://www.modelscope.cn/iic/CosyVoice-300M.git pretrained_models/CosyVoice-300Mgit clone https://www.modelscope.cn/iic/CosyVoice-300M-SFT.git pretrained_models/CosyVoice-300M-SFTgit clone https://www.modelscope.cn/iic/CosyVoice-300M-Instruct.git pretrained_models/CosyVoice-300M-Instructgit clone https://www.modelscope.cn/iic/CosyVoice-ttsfrd.git pretrained_models/CosyVoice-ttsfrd

cd pretrained_models/CosyVoice-ttsfrd/unzip resource.zip -d.pip install ttsfrd_dependency-0.1-py3-none-any.whlpip install ttsfrd-0.4.2-cp310-cp310-linux_x86_64.whl

Basic Usage

importsyssys.path.append('third_party/Matcha-TTS')fromcosyvoice.cli.cosyvoiceimportCosyVoice,CosyVoice2fromcosyvoice.utils.file_utilsimportload_wavimporttorchaudio

CosyVoice2 Usage

cosyvoice=CosyVoice2('pretrained_models/CosyVoice2-0.5B',load_jit=False,load_trt=False,load_vllm=False,fp16=False)# NOTE if you want to reproduce the results on https://funaudiollm.github.io/cosyvoice2, please add text_frontend=False during inference# zero_shot usageprompt_speech_16k=load_wav('./asset/zero_shot_prompt.wav',16000)fori,jinenumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。','希望你以后能够做的比我还好呦。',prompt_speech_16k,stream=False)):torchaudio.save('zero_shot_{}.wav'.format(i),j['tts_speech'],cosyvoice.sample_rate)# save zero_shot spk for future usageassertcosyvoice.add_zero_shot_spk('希望你以后能够做的比我还好呦。',prompt_speech_16k,'my_zero_shot_spk')isTruefori,jinenumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。','','',zero_shot_spk_id='my_zero_shot_spk',stream=False)):torchaudio.save('zero_shot_{}.wav'.format(i),j['tts_speech'],cosyvoice.sample_rate)cosyvoice.save_spkinfo()# fine grained control, for supported control, check cosyvoice/tokenizer/tokenizer.py#L248fori,jinenumerate(cosyvoice.inference_cross_lingual('在他讲述那个荒诞故事的过程中，他突然[laughter]停下来，因为他自己也被逗笑了[laughter]。',prompt_speech_16k,stream=False)):torchaudio.save('fine_grained_control_{}.wav'.format(i),j['tts_speech'],cosyvoice.sample_rate)# instruct usagefori,jinenumerate(cosyvoice.inference_instruct2('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。','用四川话说这句话',prompt_speech_16k,stream=False)):torchaudio.save('instruct_{}.wav'.format(i),j['tts_speech'],cosyvoice.sample_rate)# bistream usage, you can use generator as input, this is useful when using text llm model as input# NOTE you should still have some basic sentence split logic because llm can not handle arbitrary sentence lengthdeftext_generator():yield'收到好友从远方寄来的生日礼物，'yield'那份意外的惊喜与深深的祝福'yield'让我心中充满了甜蜜的快乐，'yield'笑容如花儿般绽放。'fori,jinenumerate(cosyvoice.inference_zero_shot(text_generator(),'希望你以后能够做的比我还好呦。',prompt_speech_16k,stream=False)):torchaudio.save('zero_shot_{}.wav'.format(i),j['tts_speech'],cosyvoice.sample_rate)

CosyVoice2 vllm Usage

conda create -n cosyvoice_vllm --clone cosyvoiceconda activate cosyvoice_vllmpip install vllm==v0.9.0 -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host=mirrors.aliyun.compython vllm_example.py

CosyVoice Usage

cosyvoice=CosyVoice('pretrained_models/CosyVoice-300M-SFT',load_jit=False,load_trt=False,fp16=False)# sft usageprint(cosyvoice.list_available_spks())# change stream=True for chunk stream inferencefori,jinenumerate(cosyvoice.inference_sft('你好，我是通义生成式语音大模型，请问有什么可以帮您的吗？','中文女',stream=False)):torchaudio.save('sft_{}.wav'.format(i),j['tts_speech'],cosyvoice.sample_rate)cosyvoice=CosyVoice('pretrained_models/CosyVoice-300M')# zero_shot usage, <|zh|><|en|><|jp|><|yue|><|ko|> for Chinese/English/Japanese/Cantonese/Koreanprompt_speech_16k=load_wav('./asset/zero_shot_prompt.wav',16000)fori,jinenumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。','希望你以后能够做的比我还好呦。',prompt_speech_16k,stream=False)):torchaudio.save('zero_shot_{}.wav'.format(i),j['tts_speech'],cosyvoice.sample_rate)# cross_lingual usageprompt_speech_16k=load_wav('./asset/cross_lingual_prompt.wav',16000)fori,jinenumerate(cosyvoice.inference_cross_lingual('<|en|>And then later on, fully acquiring that company. So keeping management in line, interest in line with the asset that\'s coming into the family is a reason why sometimes we don\'t buy the whole thing.',prompt_speech_16k,stream=False)):torchaudio.save('cross_lingual_{}.wav'.format(i),j['tts_speech'],cosyvoice.sample_rate)# vc usageprompt_speech_16k=load_wav('./asset/zero_shot_prompt.wav',16000)source_speech_16k=load_wav('./asset/cross_lingual_prompt.wav',16000)fori,jinenumerate(cosyvoice.inference_vc(source_speech_16k,prompt_speech_16k,stream=False)):torchaudio.save('vc_{}.wav'.format(i),j['tts_speech'],cosyvoice.sample_rate)cosyvoice=CosyVoice('pretrained_models/CosyVoice-300M-Instruct')# instruct usage, support <laughter></laughter><strong></strong>[laughter][breath]fori,jinenumerate(cosyvoice.inference_instruct('在面对挑战时，他展现了非凡的<strong>勇气</strong>与<strong>智慧</strong>。','中文男','Theo\'Crimson\', is a fiery, passionate rebel leader. Fights with fervor for justice, but struggles with impulsiveness.',stream=False)):torchaudio.save('instruct_{}.wav'.format(i),j['tts_speech'],cosyvoice.sample_rate)

Start web demo

# change iic/CosyVoice-300M-SFT for sft inference, or iic/CosyVoice-300M-Instruct for instruct inferencepython3webui.py--port50000--model_dirpretrained_models/CosyVoice-300M

Advanced Usage

Build for deployment

cd runtime/pythondocker build -t cosyvoice:v1.0.# change iic/CosyVoice-300M to iic/CosyVoice-300M-Instruct if you want to use instruct inference# for grpc usagedocker run -d --runtime=nvidia -p 50000:50000 cosyvoice:v1.0 /bin/bash -c"cd /opt/CosyVoice/CosyVoice/runtime/python/grpc && python3 server.py --port 50000 --max_conc 4 --model_dir iic/CosyVoice-300M && sleep infinity"cd grpc&& python3 client.py --port 50000 --mode<sft|zero_shot|cross_lingual|instruct># for fastapi usagedocker run -d --runtime=nvidia -p 50000:50000 cosyvoice:v1.0 /bin/bash -c"cd /opt/CosyVoice/CosyVoice/runtime/python/fastapi && python3 server.py --port 50000 --model_dir iic/CosyVoice-300M && sleep infinity"cd fastapi&& python3 client.py --port 50000 --mode<sft|zero_shot|cross_lingual|instruct>

Discussion & Communication

Acknowledge

Citations

@article{du2024cosyvoice,title={Cosyvoice: A scalable multilingual zero-shot text-to-speech synthesizer based on supervised semantic tokens},author={Du, Zhihao and Chen, Qian and Zhang, Shiliang and Hu, Kai and Lu, Heng and Yang, Yexin and Hu, Hangrui and Zheng, Siqi and Gu, Yue and Ma, Ziyang and others},journal={arXiv preprint arXiv:2407.05407},year={2024}}@article{du2024cosyvoice,title={Cosyvoice 2: Scalable streaming speech synthesis with large language models},author={Du, Zhihao and Wang, Yuxuan and Chen, Qian and Shi, Xian and Lv, Xiang and Zhao, Tianyu and Gao, Zhifu and Yang, Yexin and Gao, Changfeng and Wang, Hui and others},journal={arXiv preprint arXiv:2412.10117},year={2024}}@article{du2025cosyvoice,title={CosyVoice 3: Towards In-the-wild Speech Generation via Scaling-up and Post-training},author={Du, Zhihao and Gao, Changfeng and Wang, Yuxuan and Yu, Fan and Zhao, Tianyu and Wang, Hao and Lv, Xiang and Wang, Hui and Shi, Xian and An, Keyu and others},journal={arXiv preprint arXiv:2505.17589},year={2025}}@inproceedings{lyu2025build,title={Build LLM-Based Zero-Shot Streaming TTS System with Cosyvoice},author={Lyu, Xiang and Wang, Yuxuan and Zhao, Tianyu and Wang, Hao and Liu, Huadai and Du, Zhihao},booktitle={ICASSP 2025-2025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},pages={1--2},year={2025},organization={IEEE}}

Disclaimer