DALL-E 2 - Pytorch

Status

Pre-Trained Models

Appreciation

Install

$ pip install dalle2-pytorch

Usage

importtorchfromdalle2_pytorchimportCLIPclip=CLIP(dim_text=512,dim_image=512,dim_latent=512,num_text_tokens=49408,text_enc_depth=1,text_seq_len=256,text_heads=8,visual_enc_depth=1,visual_image_size=256,visual_patch_size=32,visual_heads=8,use_all_token_embeds=True,# whether to use fine-grained contrastive learning (FILIP)decoupled_contrastive_learning=True,# use decoupled contrastive learning (DCL) objective function, removing positive pairs from the denominator of the InfoNCE loss (CLOOB + DCL)extra_latent_projection=True,# whether to use separate projections for text-to-image vs image-to-text comparisons (CLOOB)use_visual_ssl=True,# whether to do self supervised learning on imagesvisual_ssl_type='simclr',# can be either 'simclr' or 'simsiam', depending on using DeCLIP or SLIPuse_mlm=False,# use masked language learning (MLM) on text (DeCLIP)text_ssl_loss_weight=0.05,# weight for text MLM lossimage_ssl_loss_weight=0.05# weight for image self-supervised learning loss).cuda()# mock datatext=torch.randint(0,49408, (4,256)).cuda()images=torch.randn(4,3,256,256).cuda()# trainloss=clip(text,images,return_loss=True# needs to be set to True to return contrastive loss)loss.backward()# do the above with as many texts and images as possible in a loop

importtorchfromdalle2_pytorchimportUnet,Decoder,CLIP# trained clip from step 1clip=CLIP(dim_text=512,dim_image=512,dim_latent=512,num_text_tokens=49408,text_enc_depth=1,text_seq_len=256,text_heads=8,visual_enc_depth=1,visual_image_size=256,visual_patch_size=32,visual_heads=8).cuda()# unet for the decoderunet=Unet(dim=128,image_embed_dim=512,cond_dim=128,channels=3,dim_mults=(1,2,4,8)).cuda()# decoder, which contains the unet and clipdecoder=Decoder(unet=unet,clip=clip,timesteps=100,image_cond_drop_prob=0.1,text_cond_drop_prob=0.5).cuda()# mock images (get a lot of this)images=torch.randn(4,3,256,256).cuda()# feed images into decoderloss=decoder(images)loss.backward()# do the above for many many many many steps# then it will learn to generate images based on the CLIP image embeddings

importtorchfromdalle2_pytorchimportDiffusionPriorNetwork,DiffusionPrior,CLIP# get trained CLIP from step oneclip=CLIP(dim_text=512,dim_image=512,dim_latent=512,num_text_tokens=49408,text_enc_depth=6,text_seq_len=256,text_heads=8,visual_enc_depth=6,visual_image_size=256,visual_patch_size=32,visual_heads=8,).cuda()# setup prior network, which contains an autoregressive transformerprior_network=DiffusionPriorNetwork(dim=512,depth=6,dim_head=64,heads=8).cuda()# diffusion prior network, which contains the CLIP and network (with transformer) abovediffusion_prior=DiffusionPrior(net=prior_network,clip=clip,timesteps=100,cond_drop_prob=0.2).cuda()# mock datatext=torch.randint(0,49408, (4,256)).cuda()images=torch.randn(4,3,256,256).cuda()# feed text and images into diffusion prior networkloss=diffusion_prior(text,images)loss.backward()# do the above for many many many steps# now the diffusion prior can generate image embeddings from the text embeddings

importtorchfromdalle2_pytorchimportUnet,Decoder,CLIP# trained clip from step 1clip=CLIP(dim_text=512,dim_image=512,dim_latent=512,num_text_tokens=49408,text_enc_depth=6,text_seq_len=256,text_heads=8,visual_enc_depth=6,visual_image_size=256,visual_patch_size=32,visual_heads=8).cuda()# 2 unets for the decoder (a la cascading DDPM)unet1=Unet(dim=32,image_embed_dim=512,cond_dim=128,channels=3,dim_mults= (1,2,4,8)).cuda()unet2=Unet(dim=32,image_embed_dim=512,cond_dim=128,channels=3,dim_mults= (1,2,4,8,16)).cuda()# decoder, which contains the unet(s) and clipdecoder=Decoder(clip=clip,unet= (unet1,unet2),# insert both unets in order of low resolution to highest resolution (you can have as many stages as you want here)image_sizes= (256,512),# resolutions, 256 for first unet, 512 for second. these must be unique and in ascending order (matches with the unets passed in)timesteps=1000,image_cond_drop_prob=0.1,text_cond_drop_prob=0.5).cuda()# mock images (get a lot of this)images=torch.randn(4,3,512,512).cuda()# feed images into decoder, specifying which unet you want to train# each unet can be trained separately, which is one of the benefits of the cascading DDPM schemeloss=decoder(images,unet_number=1)loss.backward()loss=decoder(images,unet_number=2)loss.backward()# do the above for many steps for both unets

fromdalle2_pytorchimportDALLE2dalle2=DALLE2(prior=diffusion_prior,decoder=decoder)# send the text as a string if you want to use the simple tokenizer from DALLE v1# or you can do it as token ids, if you have your own tokenizertexts= ['glistening morning dew on a flower petal']images=dalle2(texts)# (1, 3, 256, 256)

importtorchfromdalle2_pytorchimportDALLE2,DiffusionPriorNetwork,DiffusionPrior,Unet,Decoder,CLIPclip=CLIP(dim_text=512,dim_image=512,dim_latent=512,num_text_tokens=49408,text_enc_depth=6,text_seq_len=256,text_heads=8,visual_enc_depth=6,visual_image_size=256,visual_patch_size=32,visual_heads=8).cuda()# mock datatext=torch.randint(0,49408, (4,256)).cuda()images=torch.randn(4,3,256,256).cuda()# trainloss=clip(text,images,return_loss=True)loss.backward()# do above for many steps ...# prior networks (with transformer)prior_network=DiffusionPriorNetwork(dim=512,depth=6,dim_head=64,heads=8).cuda()diffusion_prior=DiffusionPrior(net=prior_network,clip=clip,timesteps=1000,sample_timesteps=64,cond_drop_prob=0.2).cuda()loss=diffusion_prior(text,images)loss.backward()# do above for many steps ...# decoder (with unet)unet1=Unet(dim=128,image_embed_dim=512,text_embed_dim=512,cond_dim=128,channels=3,dim_mults=(1,2,4,8),cond_on_text_encodings=True# set to True for any unets that need to be conditioned on text encodings).cuda()unet2=Unet(dim=16,image_embed_dim=512,cond_dim=128,channels=3,dim_mults= (1,2,4,8,16)).cuda()decoder=Decoder(unet= (unet1,unet2),image_sizes= (128,256),clip=clip,timesteps=100,image_cond_drop_prob=0.1,text_cond_drop_prob=0.5).cuda()forunet_numberin (1,2):loss=decoder(images,text=text,unet_number=unet_number)# this can optionally be decoder(images, text) if you wish to condition on the text encodings as well, though it was hinted in the paper it didn't do muchloss.backward()# do above for many stepsdalle2=DALLE2(prior=diffusion_prior,decoder=decoder)images=dalle2(    ['cute puppy chasing after a squirrel'],cond_scale=2.# classifier free guidance strength (> 1 would strengthen the condition))# save your image (in this example, of size 256x256)

Training on Preprocessed CLIP Embeddings

importtorchfromdalle2_pytorchimportDiffusionPriorNetwork,DiffusionPrior,CLIP# get trained CLIP from step oneclip=CLIP(dim_text=512,dim_image=512,dim_latent=512,num_text_tokens=49408,text_enc_depth=6,text_seq_len=256,text_heads=8,visual_enc_depth=6,visual_image_size=256,visual_patch_size=32,visual_heads=8,).cuda()# setup prior network, which contains an autoregressive transformerprior_network=DiffusionPriorNetwork(dim=512,depth=6,dim_head=64,heads=8).cuda()# diffusion prior network, which contains the CLIP and network (with transformer) abovediffusion_prior=DiffusionPrior(net=prior_network,clip=clip,timesteps=100,cond_drop_prob=0.2,condition_on_text_encodings=False# this probably should be true, but just to get Laion started).cuda()# mock datatext=torch.randint(0,49408, (4,256)).cuda()images=torch.randn(4,3,256,256).cuda()# precompute the text and image embeddings# here using the diffusion prior class, but could be done with CLIP aloneclip_image_embeds=diffusion_prior.clip.embed_image(images).image_embedclip_text_embeds=diffusion_prior.clip.embed_text(text).text_embed# feed text and images into diffusion prior networkloss=diffusion_prior(text_embed=clip_text_embeds,image_embed=clip_image_embeds)loss.backward()# do the above for many many many steps# now the diffusion prior can generate image embeddings from the text embeddings

importtorchfromdalle2_pytorchimportDiffusionPriorNetwork,DiffusionPrior# setup prior network, which contains an autoregressive transformerprior_network=DiffusionPriorNetwork(dim=512,depth=6,dim_head=64,heads=8).cuda()# diffusion prior network, which contains the CLIP and network (with transformer) abovediffusion_prior=DiffusionPrior(net=prior_network,image_embed_dim=512,# this needs to be settimesteps=100,cond_drop_prob=0.2,condition_on_text_encodings=False# this probably should be true, but just to get Laion started).cuda()# mock datatext=torch.randint(0,49408, (4,256)).cuda()images=torch.randn(4,3,256,256).cuda()# precompute the text and image embeddings# here using the diffusion prior class, but could be done with CLIP aloneclip_image_embeds=torch.randn(4,512).cuda()clip_text_embeds=torch.randn(4,512).cuda()# feed text and images into diffusion prior networkloss=diffusion_prior(text_embed=clip_text_embeds,image_embed=clip_image_embeds)loss.backward()# do the above for many many many steps# now the diffusion prior can generate image embeddings from the text embeddings

OpenAI CLIP

importtorchfromdalle2_pytorchimportDALLE2,DiffusionPriorNetwork,DiffusionPrior,Unet,Decoder,OpenAIClipAdapter# openai pretrained clip - defaults to ViT-B/32clip=OpenAIClipAdapter()# mock datatext=torch.randint(0,49408, (4,256)).cuda()images=torch.randn(4,3,256,256).cuda()# prior networks (with transformer)prior_network=DiffusionPriorNetwork(dim=512,depth=6,dim_head=64,heads=8).cuda()diffusion_prior=DiffusionPrior(net=prior_network,clip=clip,timesteps=100,cond_drop_prob=0.2).cuda()loss=diffusion_prior(text,images)loss.backward()# do above for many steps ...# decoder (with unet)unet1=Unet(dim=128,image_embed_dim=512,cond_dim=128,channels=3,dim_mults=(1,2,4,8),text_embed_dim=512,cond_on_text_encodings=True# set to True for any unets that need to be conditioned on text encodings (ex. first unet in cascade)).cuda()unet2=Unet(dim=16,image_embed_dim=512,cond_dim=128,channels=3,dim_mults= (1,2,4,8,16)).cuda()decoder=Decoder(unet= (unet1,unet2),image_sizes= (128,256),clip=clip,timesteps=1000,sample_timesteps= (250,27),image_cond_drop_prob=0.1,text_cond_drop_prob=0.5).cuda()forunet_numberin (1,2):loss=decoder(images,text=text,unet_number=unet_number)# this can optionally be decoder(images, text) if you wish to condition on the text encodings as well, though it was hinted in the paper it didn't do muchloss.backward()# do above for many stepsdalle2=DALLE2(prior=diffusion_prior,decoder=decoder)images=dalle2(    ['a butterfly trying to escape a tornado'],cond_scale=2.# classifier free guidance strength (> 1 would strengthen the condition))# save your image (in this example, of size 256x256)

$ pip install open-clip-torch

fromdalle2_pytorchimportOpenClipAdapterclip=OpenClipAdapter('ViT-H/14')

Inpainting

importtorchfromdalle2_pytorchimportUnet,Decoder,CLIP# trained clip from step 1clip=CLIP(dim_text=512,dim_image=512,dim_latent=512,num_text_tokens=49408,text_enc_depth=6,text_seq_len=256,text_heads=8,visual_enc_depth=6,visual_image_size=256,visual_patch_size=32,visual_heads=8).cuda()# 2 unets for the decoder (a la cascading DDPM)unet=Unet(dim=16,image_embed_dim=512,cond_dim=128,channels=3,dim_mults= (1,1,1,1)).cuda()# decoder, which contains the unet(s) and clipdecoder=Decoder(clip=clip,unet= (unet,),# insert both unets in order of low resolution to highest resolution (you can have as many stages as you want here)image_sizes= (256,),# resolutions, 256 for first unet, 512 for second. these must be unique and in ascending order (matches with the unets passed in)timesteps=1000,image_cond_drop_prob=0.1,text_cond_drop_prob=0.5).cuda()# mock images (get a lot of this)images=torch.randn(4,3,256,256).cuda()# feed images into decoder, specifying which unet you want to train# each unet can be trained separately, which is one of the benefits of the cascading DDPM schemeloss=decoder(images,unet_number=1)loss.backward()# do the above for many steps for both unetsmock_image_embed=torch.randn(1,512).cuda()# then to do inpaintinginpaint_image=torch.randn(1,3,256,256).cuda()# (batch, channels, height, width)inpaint_mask=torch.ones(1,256,256).bool().cuda()# (batch, height, width)inpainted_images=decoder.sample(image_embed=mock_image_embed,inpaint_image=inpaint_image,# just pass in the inpaint imageinpaint_mask=inpaint_mask# and the mask)inpainted_images.shape# (1, 3, 256, 256)

Experimental

DALL-E2 with Latent Diffusion

importtorchfromdalle2_pytorchimportUnet,Decoder,CLIP,VQGanVAE# trained clip from step 1clip=CLIP(dim_text=512,dim_image=512,dim_latent=512,num_text_tokens=49408,text_enc_depth=1,text_seq_len=256,text_heads=8,visual_enc_depth=1,visual_image_size=256,visual_patch_size=32,visual_heads=8)# 3 unets for the decoder (a la cascading DDPM)# first two unets are doing latent diffusion# vqgan-vae must be trained beforehandvae1=VQGanVAE(dim=32,image_size=256,layers=3,layer_mults= (1,2,4))vae2=VQGanVAE(dim=32,image_size=512,layers=3,layer_mults= (1,2,4))unet1=Unet(dim=32,image_embed_dim=512,cond_dim=128,channels=3,sparse_attn=True,sparse_attn_window=2,dim_mults= (1,2,4,8))unet2=Unet(dim=32,image_embed_dim=512,channels=3,dim_mults= (1,2,4,8,16),cond_on_image_embeds=True,cond_on_text_encodings=False)unet3=Unet(dim=32,image_embed_dim=512,channels=3,dim_mults= (1,2,4,8,16),cond_on_image_embeds=True,cond_on_text_encodings=False,attend_at_middle=False)# decoder, which contains the unet(s) and clipdecoder=Decoder(clip=clip,vae= (vae1,vae2),# latent diffusion for unet1 (vae1) and unet2 (vae2), but not for the last unet3unet= (unet1,unet2,unet3),# insert unets in order of low resolution to highest resolution (you can have as many stages as you want here)image_sizes= (256,512,1024),# resolutions, 256 for first unet, 512 for second, 1024 for thirdtimesteps=100,image_cond_drop_prob=0.1,text_cond_drop_prob=0.5).cuda()# mock images (get a lot of this)images=torch.randn(1,3,1024,1024).cuda()# feed images into decoder, specifying which unet you want to train# each unet can be trained separately, which is one of the benefits of the cascading DDPM schemewithdecoder.one_unet_in_gpu(1):loss=decoder(images,unet_number=1)loss.backward()withdecoder.one_unet_in_gpu(2):loss=decoder(images,unet_number=2)loss.backward()withdecoder.one_unet_in_gpu(3):loss=decoder(images,unet_number=3)loss.backward()# do the above for many steps for both unets# then it will learn to generate images based on the CLIP image embeddings# chaining the unets from lowest resolution to highest resolution (thus cascading)mock_image_embed=torch.randn(1,512).cuda()images=decoder.sample(mock_image_embed)# (1, 3, 1024, 1024)

Training wrapper

Decoder Training

importtorchfromdalle2_pytorchimportDALLE2,Unet,Decoder,CLIP,DecoderTrainerclip=CLIP(dim_text=512,dim_image=512,dim_latent=512,num_text_tokens=49408,text_enc_depth=6,text_seq_len=256,text_heads=8,visual_enc_depth=6,visual_image_size=256,visual_patch_size=32,visual_heads=8).cuda()# mock datatext=torch.randint(0,49408, (32,256)).cuda()images=torch.randn(32,3,256,256).cuda()# decoder (with unet)unet1=Unet(dim=128,image_embed_dim=512,text_embed_dim=512,cond_dim=128,channels=3,dim_mults=(1,2,4,8),cond_on_text_encodings=True,).cuda()unet2=Unet(dim=16,image_embed_dim=512,cond_dim=128,channels=3,dim_mults= (1,2,4,8,16),).cuda()decoder=Decoder(unet= (unet1,unet2),image_sizes= (128,256),clip=clip,timesteps=1000).cuda()decoder_trainer=DecoderTrainer(decoder,lr=3e-4,wd=1e-2,ema_beta=0.99,ema_update_after_step=1000,ema_update_every=10,)forunet_numberin (1,2):loss=decoder_trainer(images,text=text,unet_number=unet_number,# which unet to train onmax_batch_size=4# gradient accumulation - this sets the maximum batch size in which to do forward and backwards pass - for this example 32 / 4 == 8 times    )decoder_trainer.update(unet_number)# update the specific unet as well as its exponential moving average# after much training# you can sample from the exponentially moving averaged unets as somock_image_embed=torch.randn(32,512).cuda()images=decoder_trainer.sample(image_embed=mock_image_embed,text=text)# (4, 3, 256, 256)

Diffusion Prior Training

importtorchfromdalle2_pytorchimportDALLE2,DiffusionPriorNetwork,DiffusionPrior,DiffusionPriorTrainer,Unet,Decoder,CLIPclip=CLIP(dim_text=512,dim_image=512,dim_latent=512,num_text_tokens=49408,text_enc_depth=6,text_seq_len=256,text_heads=8,visual_enc_depth=6,visual_image_size=256,visual_patch_size=32,visual_heads=8).cuda()# mock datatext=torch.randint(0,49408, (512,256)).cuda()images=torch.randn(512,3,256,256).cuda()# prior networks (with transformer)prior_network=DiffusionPriorNetwork(dim=512,depth=6,dim_head=64,heads=8).cuda()diffusion_prior=DiffusionPrior(net=prior_network,clip=clip,timesteps=100,cond_drop_prob=0.2).cuda()diffusion_prior_trainer=DiffusionPriorTrainer(diffusion_prior,lr=3e-4,wd=1e-2,ema_beta=0.99,ema_update_after_step=1000,ema_update_every=10,)loss=diffusion_prior_trainer(text,images,max_batch_size=4)diffusion_prior_trainer.update()# this will update the optimizer as well as the exponential moving averaged diffusion prior# after much of the above three lines in a loop# you can sample from the exponential moving average of the diffusion prior identically to how you do so for DiffusionPriorimage_embeds=diffusion_prior_trainer.sample(text,max_batch_size=4)# (512, 512) - exponential moving averaged image embeddings

Bonus

Unconditional Training

importtorchfromdalle2_pytorchimportUnet,Decoder,DecoderTrainer# unet for the cascading ddpmunet1=Unet(dim=128,dim_mults=(1,2,4,8)).cuda()unet2=Unet(dim=32,dim_mults= (1,2,4,8,16)).cuda()# decoder, which contains the unetsdecoder=Decoder(unet= (unet1,unet2),image_sizes= (256,512),# first unet up to 256px, then second to 512pxtimesteps=1000,unconditional=True).cuda()# decoder trainerdecoder_trainer=DecoderTrainer(decoder)# images (get a lot of this)images=torch.randn(1,3,512,512).cuda()# feed images into decoderforiin (1,2):loss=decoder_trainer(images,unet_number=i)decoder_trainer.update(unet_number=i)# do the above for many many many many images# then it will learn to generate imagesimages=decoder_trainer.sample(batch_size=36,max_batch_size=4)# (36, 3, 512, 512)

Dataloaders

Decoder Dataloaders

Decoder: Image Embedding Dataset

fromdalle2_pytorch.dataloadersimportImageEmbeddingDataset,create_image_embedding_dataloader# Create a dataloader directly.dataloader=create_image_embedding_dataloader(tar_url="/path/or/url/to/webdataset/{0000..9999}.tar",# Uses bracket expanding notation. This specifies to read all tars from 0000.tar to 9999.tarembeddings_url="path/or/url/to/embeddings/folder",# Included if .npy files are not in webdataset. Left out or set to None otherwisenum_workers=4,batch_size=32,shard_width=4,# If a file in the webdataset shard 3 is named 0003039.jpg, we know the shard width is 4 and the last three digits are the indexshuffle_num=200,# Does a shuffle of the data with a buffer size of 200shuffle_shards=True,# Shuffle the order the shards are read inresample_shards=False,# Sample shards with replacement. If true, an epoch will be infinite unless stopped manually)forimg,embindataloader:print(img.shape)# torch.Size([32, 3, 256, 256])print(emb["img"].shape)# torch.Size([32, 512])# Train decoder only as shown above# Or create a dataset without a loader so you can configure it manuallydataset=ImageEmbeddingDataset(urls="/path/or/url/to/webdataset/{0000..9999}.tar",embedding_folder_url="path/or/url/to/embeddings/folder",shard_width=4,shuffle_shards=True,resample=False)

Scripts

train_diffusion_prior.py

Todo

Citations

@misc{ramesh2022,title   ={Hierarchical Text-Conditional Image Generation with CLIP Latents},author  ={Aditya Ramesh et al},year    ={2022}}

@misc{crowson2022,author  ={Katherine Crowson},url     ={https://twitter.com/rivershavewings}}

@misc{rombach2021highresolution,title   ={High-Resolution Image Synthesis with Latent Diffusion Models},author  ={Robin Rombach and Andreas Blattmann and Dominik Lorenz and Patrick Esser and Björn Ommer},year    ={2021},eprint  ={2112.10752},archivePrefix ={arXiv},primaryClass ={cs.CV}}

@article{shen2019efficient,author  ={Zhuoran Shen and Mingyuan Zhang and Haiyu Zhao and Shuai Yi and Hongsheng Li},title   ={Efficient Attention: Attention with Linear Complexities},journal ={CoRR},year    ={2018},url     ={http://arxiv.org/abs/1812.01243},}

@article{Yu2021VectorquantizedIM,title   ={Vector-quantized Image Modeling with Improved VQGAN},author  ={Jiahui Yu and Xin Li and Jing Yu Koh and Han Zhang and Ruoming Pang and James Qin and Alexander Ku and Yuanzhong Xu and Jason Baldridge and Yonghui Wu},journal ={ArXiv},year    ={2021},volume  ={abs/2110.04627}}

@article{Shleifer2021NormFormerIT,title   ={NormFormer: Improved Transformer Pretraining with Extra Normalization},author  ={Sam Shleifer and Jason Weston and Myle Ott},journal ={ArXiv},year    ={2021},volume  ={abs/2110.09456}}

@article{Yu2022CoCaCC,title   ={CoCa: Contrastive Captioners are Image-Text Foundation Models},author  ={Jiahui Yu and Zirui Wang and Vijay Vasudevan and Legg Yeung and Mojtaba Seyedhosseini and Yonghui Wu},journal ={ArXiv},year    ={2022},volume  ={abs/2205.01917}}

@misc{wang2021crossformer,title   ={CrossFormer: A Versatile Vision Transformer Hinging on Cross-scale Attention},author  ={Wenxiao Wang and Lu Yao and Long Chen and Binbin Lin and Deng Cai and Xiaofei He and Wei Liu},year    ={2021},eprint  ={2108.00154},archivePrefix ={arXiv},primaryClass ={cs.CV}}

@article{ho2021cascaded,title   ={Cascaded Diffusion Models for High Fidelity Image Generation},author  ={Ho, Jonathan and Saharia, Chitwan and Chan, William and Fleet, David J and Norouzi, Mohammad and Salimans, Tim},journal ={arXiv preprint arXiv:2106.15282},year    ={2021}}

@misc{Saharia2022,title   ={Imagen: unprecedented photorealism × deep level of language understanding},author  ={Chitwan Saharia*, William Chan*, Saurabh Saxena†, Lala Li†, Jay Whang†, Emily Denton, Seyed Kamyar Seyed Ghasemipour, Burcu Karagol Ayan, S. Sara Mahdavi, Rapha Gontijo Lopes, Tim Salimans, Jonathan Ho†, David Fleet†, Mohammad Norouzi*},year    ={2022}}

@article{Choi2022PerceptionPT,title   ={Perception Prioritized Training of Diffusion Models},author  ={Jooyoung Choi and Jungbeom Lee and Chaehun Shin and Sungwon Kim and Hyunwoo J. Kim and Sung-Hoon Yoon},journal ={ArXiv},year    ={2022},volume  ={abs/2204.00227}}

@article{Saharia2021PaletteID,title   ={Palette: Image-to-Image Diffusion Models},author  ={Chitwan Saharia and William Chan and Huiwen Chang and Chris A. Lee and Jonathan Ho and Tim Salimans and David J. Fleet and Mohammad Norouzi},journal ={ArXiv},year    ={2021},volume  ={abs/2111.05826}}

@article{Lugmayr2022RePaintIU,title   ={RePaint: Inpainting using Denoising Diffusion Probabilistic Models},author  ={Andreas Lugmayr and Martin Danelljan and Andr{\'e}s Romero and Fisher Yu and Radu Timofte and Luc Van Gool},journal ={ArXiv},year    ={2022},volume  ={abs/2201.09865}}

@misc{chen2022analog,title   ={Analog Bits: Generating Discrete Data using Diffusion Models with Self-Conditioning},author  ={Ting Chen and Ruixiang Zhang and Geoffrey Hinton},year    ={2022},eprint  ={2208.04202},archivePrefix ={arXiv},primaryClass ={cs.CV}}

@article{Qiao2019WeightS,title   ={Weight Standardization},author  ={Siyuan Qiao and Huiyu Wang and Chenxi Liu and Wei Shen and Alan Loddon Yuille},journal ={ArXiv},year    ={2019},volume  ={abs/1903.10520}}

@inproceedings{rogozhnikov2022einops,title   ={Einops: Clear and Reliable Tensor Manipulations with Einstein-like Notation},author  ={Alex Rogozhnikov},booktitle ={International Conference on Learning Representations},year    ={2022},url     ={https://openreview.net/forum?id=oapKSVM2bcj}}

@article{Sunkara2022NoMS,title   ={No More Strided Convolutions or Pooling: A New CNN Building Block for Low-Resolution Images and Small Objects},author  ={Raja Sunkara and Tie Luo},journal ={ArXiv},year    ={2022},volume  ={abs/2208.03641}}

@article{Salimans2022ProgressiveDF,title   ={Progressive Distillation for Fast Sampling of Diffusion Models},author  ={Tim Salimans and Jonathan Ho},journal ={ArXiv},year    ={2022},volume  ={abs/2202.00512}}