Skip to content

Add support for APG (adaptive projected guidance) + unconditionnal SLG #593

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Add support for APG (adaptive projected guidance) + unconditionnal SLG #593

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a5dbce5
apg: first implementation
stduhpf Feb 11, 2025
02114c2
refactor guidance params in lib
stduhpf Feb 12, 2025
98e056b
main: add apg support
stduhpf Feb 12, 2025
e64b3b8
add apg settings to image params
stduhpf Feb 12, 2025
98064d0
Fix cfg 1 crash
stduhpf Feb 12, 2025
6baa3a6
Fix CI build
stduhpf Feb 12, 2025
fb44a88
apg: add experimental threshold smoothing parameter
stduhpf Feb 12, 2025
8408ee1
add uncond slg variant
stduhpf Mar 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 80 additions & 17 deletions examples/cli/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,15 @@ struct SDParams {
int upscale_repeats = 1;

std::vector<int> skip_layers = {7, 8, 9};
float slg_scale = 0.f;
float slg_scale = 0.0f;
float skip_layer_start = 0.01f;
float skip_layer_end = 0.2f;
bool slg_uncond = false;

float apg_eta = 1.0f;
float apg_momentum = 0.0f;
float apg_norm_threshold = 0.0f;
float apg_norm_smoothing = 0.0f;
};

void print_params(SDParams params) {
Expand Down Expand Up @@ -213,13 +219,21 @@ void print_usage(int argc, const char* argv[]) {
printf(" -n, --negative-prompt PROMPT the negative prompt (default: \"\")\n");
printf(" --cfg-scale SCALE unconditional guidance scale: (default: 7.0)\n");
printf(" --guidance SCALE guidance scale for img2img (default: 3.5)\n");
printf(" --apg-eta VALUE parallel projected guidance scale for APG (default: 1.0, recommended: between 0 and 1)\n");
printf(" --apg-momentum VALUE CFG update direction momentum for APG (default: 0, recommended: around -0.5)\n");
printf(" --apg-nt, --apg-rescale VALUE CFG update direction norm threshold for APG (default: 0 = disabled, recommended: 4-15)\n");
printf(" --apg-nt-smoothing VALUE EXPERIMENTAL! Norm threshold smoothing for APG (default: 0 = disabled)\n");
printf(" (replaces saturation with a smooth approximation)\n");
printf(" --slg-scale SCALE skip layer guidance (SLG) scale, only for DiT models: (default: 0)\n");
printf(" 0 means disabled, a value of 2.5 is nice for sd3.5 medium\n");
printf(" --eta SCALE eta in DDIM, only for DDIM and TCD: (default: 0)\n");
printf(" --slg-uncond Use CFG's forward pass for SLG instead of a separate pass, only for DiT models\n");
printf(" To use this, it's recommended to keep slg-scale to 0, both for performance and quality reasons\n");
printf(" This should be slightly faster than normal cfg when cfg_scale != 1.\n");
printf(" --skip-layers LAYERS Layers to skip for SLG steps: (default: [7,8,9])\n");
printf(" --skip-layer-start START SLG enabling point: (default: 0.01)\n");
printf(" --skip-layer-end END SLG disabling point: (default: 0.2)\n");
printf(" SLG will be enabled at step int([STEPS]*[START]) and disabled at int([STEPS]*[END])\n");
printf(" --eta SCALE eta in DDIM, only for DDIM and TCD: (default: 0)\n");
printf(" --strength STRENGTH strength for noising/unnoising (default: 0.75)\n");
printf(" --style-ratio STYLE-RATIO strength for keeping input identity (default: 20%%)\n");
printf(" --control-strength STRENGTH strength to apply Control Net (default: 0.9)\n");
Expand Down Expand Up @@ -580,6 +594,8 @@ void parse_args(int argc, const char** argv, SDParams& params) {
break;
}
params.slg_scale = std::stof(argv[i]);
} else if (arg == "--slg-uncond") {
params.slg_uncond = true;
} else if (arg == "--skip-layers") {
if (++i >= argc) {
invalid_arg = true;
Expand Down Expand Up @@ -629,6 +645,30 @@ void parse_args(int argc, const char** argv, SDParams& params) {
break;
}
params.skip_layer_end = std::stof(argv[i]);
} else if (arg == "--apg-eta") {
if (++i >= argc) {
invalid_arg = true;
break;
}
params.apg_eta = std::stof(argv[i]);
} else if (arg == "--apg-momentum") {
if (++i >= argc) {
invalid_arg = true;
break;
}
params.apg_momentum = std::stof(argv[i]);
} else if (arg == "--apg-nt" || arg == "--apg-rescale") {
if (++i >= argc) {
invalid_arg = true;
break;
}
params.apg_norm_threshold = std::stof(argv[i]);
} else if (arg == "--apg-nt-smoothing") {
if (++i >= argc) {
invalid_arg = true;
break;
}
params.apg_norm_smoothing = std::stof(argv[i]);
} else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
print_usage(argc, argv);
Expand Down Expand Up @@ -719,7 +759,20 @@ std::string get_image_params(SDParams params, int64_t seed) {
}
parameter_string += "Steps: " + std::to_string(params.sample_steps) + ", ";
parameter_string += "CFG scale: " + std::to_string(params.cfg_scale) + ", ";
if (params.apg_eta != 1) {
parameter_string += "APG eta: " + std::to_string(params.apg_eta) + ", ";
}
if (params.apg_momentum != 0) {
parameter_string += "CFG momentum: " + std::to_string(params.apg_momentum) + ", ";
}
if (params.apg_norm_threshold != 0) {
parameter_string += "CFG normalization threshold: " + std::to_string(params.apg_norm_threshold) + ", ";
if (params.apg_norm_smoothing != 0) {
parameter_string += "CFG normalization threshold: " + std::to_string(params.apg_norm_smoothing) + ", ";
}
}
if (params.slg_scale != 0 && params.skip_layers.size() != 0) {
parameter_string += "Unconditional SLG: " + std::string(params.slg_uncond ? "True" : "False") + ", ";
parameter_string += "SLG scale: " + std::to_string(params.cfg_scale) + ", ";
parameter_string += "Skip layers: [";
for (const auto& layer : params.skip_layers) {
Expand Down Expand Up @@ -963,11 +1016,16 @@ int main(int argc, const char* argv[]) {
params.style_ratio,
params.normalize_input,
params.input_id_images_path.c_str(),
params.skip_layers.data(),
params.skip_layers.size(),
params.slg_scale,
params.skip_layer_start,
params.skip_layer_end);
sd_slg_params_t{params.skip_layers.data(),
params.skip_layers.size(),
params.slg_scale,
params.skip_layer_start,
params.skip_layer_end,
params.slg_uncond},
sd_apg_params_t{params.apg_eta,
params.apg_momentum,
params.apg_norm_threshold,
params.apg_norm_smoothing});
} else {
sd_image_t input_image = {(uint32_t)params.width,
(uint32_t)params.height,
Expand Down Expand Up @@ -1032,11 +1090,16 @@ int main(int argc, const char* argv[]) {
params.style_ratio,
params.normalize_input,
params.input_id_images_path.c_str(),
params.skip_layers.data(),
params.skip_layers.size(),
params.slg_scale,
params.skip_layer_start,
params.skip_layer_end);
sd_slg_params_t{params.skip_layers.data(),
params.skip_layers.size(),
params.slg_scale,
params.skip_layer_start,
params.skip_layer_end,
params.slg_uncond},
sd_apg_params_t{params.apg_eta,
params.apg_momentum,
params.apg_norm_threshold,
params.apg_norm_smoothing});
}
}

Expand Down Expand Up @@ -1075,19 +1138,19 @@ int main(int argc, const char* argv[]) {

std::string dummy_name, ext, lc_ext;
bool is_jpg;
size_t last = params.output_path.find_last_of(".");
size_t last = params.output_path.find_last_of(".");
size_t last_path = std::min(params.output_path.find_last_of("/"),
params.output_path.find_last_of("\\"));
if (last != std::string::npos // filename has extension
&& (last_path == std::string::npos || last > last_path)) {
if (last != std::string::npos // filename has extension
&& (last_path == std::string::npos || last > last_path)) {
dummy_name = params.output_path.substr(0, last);
ext = lc_ext = params.output_path.substr(last);
std::transform(ext.begin(), ext.end(), lc_ext.begin(), ::tolower);
is_jpg = lc_ext == ".jpg" || lc_ext == ".jpeg" || lc_ext == ".jpe";
} else {
dummy_name = params.output_path;
ext = lc_ext = "";
is_jpg = false;
is_jpg = false;
}
// appending ".png" to absent or unknown extension
if (!is_jpg && lc_ext != ".png") {
Expand All @@ -1099,7 +1162,7 @@ int main(int argc, const char* argv[]) {
continue;
}
std::string final_image_path = i > 0 ? dummy_name + "_" + std::to_string(i + 1) + ext : dummy_name + ext;
if(is_jpg) {
if (is_jpg) {
stbi_write_jpg(final_image_path.c_str(), results[i].width, results[i].height, results[i].channel,
results[i].data, 90, get_image_params(params, params.seed + i).c_str());
printf("save result JPEG image to '%s'\n", final_image_path.c_str());
Expand Down
Loading
Loading