News

July 2022

Pretrained models

Usage

$ pip install git+https://github.com/rinnakk/japanese-clip.git

fromPILimportImageimporttorchimportjapanese_clipasja_clipdevice="cuda"iftorch.cuda.is_available()else"cpu"# ja_clip.available_models()# ['rinna/japanese-clip-vit-b-16', 'rinna/japanese-cloob-vit-b-16']# If you want v0.1.0 models, set `revision='v0.1.0'`model,preprocess=ja_clip.load("rinna/japanese-clip-vit-b-16",cache_dir="/tmp/japanese_clip",device=device)tokenizer=ja_clip.load_tokenizer()image=preprocess(Image.open("./data/dog.jpeg")).unsqueeze(0).to(device)encodings=ja_clip.tokenize(texts=["犬","猫","象"],max_seq_len=77,device=device,tokenizer=tokenizer,# this is optional. if you don't pass, load tokenizer each time)withtorch.no_grad():image_features=model.get_image_features(image)text_features=model.get_text_features(**encodings)text_probs= (100.0*image_features @text_features.T).softmax(dim=-1)print("Label probs:",text_probs)# prints: [[1.0, 0.0, 0.0]]

Citation

@inproceedings{japanese-clip,  author = {シーン 誠, 趙 天雨, 沢田 慶},  title = {日本語における言語画像事前学習モデルの構築と公開},  booktitle= {The 25th Meeting on Image Recognition and Understanding},  year = 2022,  month = July,}

License