[submodule "ggml"]

option(SD_BUILD_EXAMPLES"sd: build examples"${SD_STANDALONE})

option(SD_CUBLAS"sd: cuda backend"OFF)

option(SD_METAL"sd: metal backend"OFF)

option(SD_FLASH_ATTN"sd: use flash attention for x4 less memory usage"OFF)

option(SD_FAST_SOFTMAX"sd: x1.5 faster softmax, indeterministic (sometimes, same seed don't generate same image), cuda only"OFF)

option(BUILD_SHARED_LIBS"sd: build shared libs"OFF)

if(SD_CUBLAS)

message("Use CUBLAS as backend stable-diffusion")

set(GGML_CUBLASON)

add_definitions(-DSD_USE_CUBLAS)

if(SD_FAST_SOFTMAX)

set(GGML_CUDA_FAST_SOFTMAXON)

endif()

endif()

if(SD_METAL)

message("Use Metal as backend stable-diffusion")

set(GGML_METALON)

add_definitions(-DSD_USE_METAL)

endif()

if(SD_FLASH_ATTN)

- Plain C/C++ implementation based on[ggml](https://github.com/ggerganov/ggml), working in the same way as[llama.cpp](https://github.com/ggerganov/llama.cpp)

- Super lightweight and without external dependencies

- SD1.x andSD2.x support

-[SD-Turbo](https://huggingface.co/stabilityai/sd-turbo) support

- SD1.x,SD2.x and SDXL support

-[SD-Turbo](https://huggingface.co/stabilityai/sd-turbo)and[SDXL-Turbo](https://huggingface.co/stabilityai/sdxl-turbo)support

- 16-bit, 32-bit float support

- 4-bit, 5-bit and 8-bit integer quantization support

- Accelerated memory-efficient CPU inference

- Only requires~2.3GB when using txt2img with fp16 precision to generate a 512x512 image, enabling Flash Attention just requires~1.8GB.

- AVX, AVX2 and AVX512 support for x86 architectures

- Full CUDA backend for GPU acceleration.

- Full CUDAand Metalbackend for GPU acceleration.

- Can load ckpt, safetensors and diffusers models/checkpoints. Standalone VAEs models

- No need to convert to`.ggml` or`.gguf` anymore!

- Flash Attention for memory usage optimization (only cpu for now)

- LoRA support, same as[stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora)

- Latent Consistency Models support (LCM/LCM-LoRA)

- Faster and memory efficient latent decoding with[TAESD](https://github.com/madebyollin/taesd)

- Upscale images generated with[ESRGAN](https://github.com/xinntao/Real-ESRGAN)

- VAE tiling processing for reduce memory usage

- Sampling method

- The current implementation of ggml_conv_2d is slow and has high memory usage

- Implement Winograd Convolution 2D for 3x3 kernel filtering

-[ ] Continuing to reduce memory usage (quantizing the weights of ggml_conv_2d)

-[ ] Implement[Real-ESRGAN](https://github.com/xinntao/Real-ESRGAN/tree/master) upscaler

-[ ] Implement Textual Inversion (embeddings)

-[ ] Implement Inpainting support

-[ ] k-quants support

cmake --build . --config Release

cmake .. -DSD_METAL=ON

cmake --build . --config Release

usage: sd[arguments]

usage:./bin/sd[arguments]

arguments:

-h, --help show this help message and exit

-m, --model[MODEL] path to model

--vae[VAE] path to vae

--taesd[TAESD_PATH] path to taesd. Using Tiny AutoEncoder for fast decoding (low quality)

--upscale-model[ESRGAN_PATH] path to esrgan model. Upscale images after generate, just RealESRGAN_x4plus_anime_6B supported by now.

--type[TYPE] weight type (f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0)

If not specified, the default is the type of the weight file.

--lora-model-dir[DIR] lora model directory

-s SEED, --seed SEED RNG seed (default: 42, use random seed for < 0)

-b, --batch-count COUNT number of images to generate.

--schedule {discrete, karras} Denoiser sigma schedule (default: discrete)

--clip-skip N number of layers to skip of clip model (default: 0)

--vae-tiling process vae in tiles to reduce memory usage

-v, --verbose print extra info

sd -m ../models/v1-5-pruned-emaonly.safetensors -p"a lovely cat" --taesd ../models/diffusion_pytorch_model.safetensors

You can use ESRGAN to upscale the generated images. At the moment, only the[RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth) model is supported. Support for more models of this architecture will be added soon.

- Specify the model path using the`--upscale-model PATH` parameter. example:

sd -m ../models/v1-5-pruned-emaonly.safetensors -p"a lovely cat" --upscale-model ../models/RealESRGAN_x4plus_anime_6B.pth

-[stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui)

-[k-diffusion](https://github.com/crowsonkb/k-diffusion)

-[latent-consistency-model](https://github.com/luosiallen/latent-consistency-model)

-[generative-models](https://github.com/Stability-AI/generative-models/)

std::string model_path;

std::string vae_path;

std::string taesd_path;

std::string esrgan_path;

ggml_type wtype = GGML_TYPE_COUNT;

std::string lora_model_dir;

std::string output_path ="output.png";

std::string prompt;

std::string negative_prompt;

float cfg_scale =7.0f;

int clip_skip = -1;// <= 0 represents unspecified

int width =512;

int height =512;

int batch_count =1;

RNGType rng_type = CUDA_RNG;

int64_t seed =42;

bool verbose =false;

bool vae_tiling =false;

};

voidprint_params(SDParams params) {

printf(" wtype: %s\n", params.wtype < GGML_TYPE_COUNT ?ggml_type_name(params.wtype) :"unspecified");

printf(" vae_path: %s\n", params.vae_path.c_str());

printf(" taesd_path: %s\n", params.taesd_path.c_str());

printf(" esrgan_path: %s\n", params.esrgan_path.c_str());

printf(" output_path: %s\n", params.output_path.c_str());

printf(" init_img: %s\n", params.input_path.c_str());

printf(" prompt: %s\n", params.prompt.c_str());

printf(" negative_prompt: %s\n", params.negative_prompt.c_str());

printf(" cfg_scale: %.2f\n", params.cfg_scale);

printf(" clip_skip: %d\n", params.clip_skip);

printf(" width: %d\n", params.width);

printf(" height: %d\n", params.height);

printf(" sample_method: %s\n", sample_method_str[params.sample_method]);

printf(" rng: %s\n", rng_type_to_str[params.rng_type]);

printf(" seed: %ld\n", params.seed);

printf(" batch_count: %d\n", params.batch_count);

printf(" vae_tiling: %s\n", params.vae_tiling ?"true" :"false");

}

voidprint_usage(int argc,constchar* argv[]) {

printf(" -m, --model [MODEL] path to model\n");

printf(" --vae [VAE] path to vae\n");

printf(" --taesd [TAESD_PATH] path to taesd. Using Tiny AutoEncoder for fast decoding (low quality)\n");

printf(" --upscale-model [ESRGAN_PATH] path to esrgan model. Upscale images after generate, just RealESRGAN_x4plus_anime_6B supported by now.\n");

printf(" --type [TYPE] weight type (f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0)\n");

printf(" If not specified, the default is the type of the weight file.\n");

printf(" --lora-model-dir [DIR] lora model directory\n");

printf(" -s SEED, --seed SEED RNG seed (default: 42, use random seed for < 0)\n");

printf(" -b, --batch-count COUNT number of images to generate.\n");

printf(" --schedule {discrete, karras} Denoiser sigma schedule (default: discrete)\n");

printf(" --clip-skip N ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1)\n");

printf(" <= 0 represents unspecified, will be 1 for SD1.x, 2 for SD2.x\n");

printf(" --vae-tiling process vae in tiles to reduce memory usage\n");

printf(" -v, --verbose print extra info\n");

}

break;

}

params.taesd_path = argv[i];

}elseif (arg =="--upscale-model") {

if (++i >= argc) {

invalid_arg =true;

break;

}

params.esrgan_path = argv[i];

}elseif (arg =="--type") {

if (++i >= argc) {

invalid_arg =true;

break;

}

params.sample_steps =std::stoi(argv[i]);

}elseif (arg =="--clip-skip") {

if (++i >= argc) {

invalid_arg =true;

break;

}

params.clip_skip =std::stoi(argv[i]);

}elseif (arg =="--vae-tiling") {

params.vae_tiling =true;

}elseif (arg =="-b" || arg =="--batch-count") {

if (++i >= argc) {

invalid_arg =true;

}

}

StableDiffusionsd(params.n_threads, vae_decode_only, params.taesd_path,true, params.lora_model_dir, params.rng_type);

StableDiffusionsd(params.n_threads, vae_decode_only, params.taesd_path,params.esrgan_path,true, params.vae_tiling, params.lora_model_dir, params.rng_type);

if (!sd.load_from_file(params.model_path, params.vae_path, params.wtype, params.schedule)) {

if (!sd.load_from_file(params.model_path, params.vae_path, params.wtype, params.schedule, params.clip_skip)) {

return1;

}

params.seed);

}

if (params.esrgan_path.size() >0) {

params.width *=4;

params.height *=4;

}

if (results.size() ==0 || results.size() != params.batch_count) {

LOG_ERROR("generate failed");

return1;