@Article{DRLTVR2022,  author  = {Qiang Wang and Yanhao Zhang and Yun Zheng and Pan Pan and Xian-Sheng Hua},  journal = {arXiv:2203.07111},  title   = {Disentangled Representation Learning for Text-Video Retrieval},  year    = {2022},}

Catalog

Setup

Setup code environment

git clone https://github.com/foolwood/DRL.gitcd DRLconda create -n drl python=3.9conda activate drlpip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple/pip install torch==1.8.1+cu102 torchvision==0.9.1+cu102 -f https://download.pytorch.org/whl/torch_stable.html

Download CLIP Model (as pretraining)

cd tvr/modelswget https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt# wget https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt# wget https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt

Download Datasets

cd data/MSR-VTTwget https://www.robots.ox.ac.uk/~maxbain/frozen-in-time/data/MSRVTT.zip; unzip MSRVTT.zipmv MSRVTT/videos/all ./videos; mv MSRVTT/annotation/MSR_VTT.json ./anns/MSRVTT_data.json

Fine-tuning code

CUDA_VISIBLE_DEVICES=0,1,2,3 python -m torch.distributed.launch --nproc_per_node=4 \main.py --do_train 1 --workers 8 --n_display 50 \--epochs 5 --lr 1e-4 --coef_lr 1e-3 --batch_size 128 --batch_size_val 128 \--anno_path data/MSR-VTT/anns --video_path data/MSR-VTT/videos --datatype msrvtt \--max_words 32 --max_frames 12 --video_framerate 1 \--base_encoder ViT-B/32 --agg_module seqTransf \--interaction wti --wti_arch 2 --cdcr 3 --cdcr_alpha1 0.11 --cdcr_alpha2 0.0 --cdcr_lambda 0.001 \--output_dir ckpts/ckpt_msrvtt_wti_cdcr

Visualization demo

License

Acknowledgments