feat: denoise strength as starting noise level

src/runtime/denoiser.hpp

@@ -1013,6 +1013,7 @@ struct Denoiser {
                                             const sd::Tensor<float>& latent)         = 0;
     virtual sd::Tensor<float> inverse_noise_scaling(float sigma,
                                                     const sd::Tensor<float>& latent) = 0;
+    virtual float noise_level_to_sigma(float noise_level)                            = 0;
 
     virtual std::vector<float> get_sigmas(uint32_t n, int image_seq_len, scheduler_t scheduler_type, SDVersion version, const char* extra_sample_args = nullptr) {
         auto bound_t_to_sigma = std::bind(&Denoiser::t_to_sigma, this, std::placeholders::_1);
@@ -1160,6 +1161,10 @@ struct CompVisDenoiser : public Denoiser {
         SD_UNUSED(sigma);
         return latent;
     }
+
+    float noise_level_to_sigma(float noise_level) {
+        return noise_level / (1.0f - noise_level);
+    }
 };
 
 struct CompVisVDenoiser : public CompVisDenoiser {
@@ -1247,6 +1252,10 @@ struct DiscreteFlowDenoiser : public Denoiser {
     sd::Tensor<float> inverse_noise_scaling(float sigma, const sd::Tensor<float>& latent) override {
         return latent * (1.0f / (1.0f - sigma));
     }
+
+    float noise_level_to_sigma(float noise_level) {
+        return noise_level;
+    }
 };
 
 struct FluxFlowDenoiser : public DiscreteFlowDenoiser {
@@ -1384,6 +1393,11 @@ struct MiniT2IFlowDenoiser : public Denoiser {
         return latent;
     }
 
+    float noise_level_to_sigma(float noise_level) {
+        SD_UNUSED(noise_level);
+        return 1.0f;
+    }
+
     std::vector<float> get_sigmas(uint32_t n, int image_seq_len, scheduler_t scheduler_type, SDVersion version, const char* extra_sample_args = nullptr) override {
         SD_UNUSED(image_seq_len);
         SD_UNUSED(scheduler_type);

src/stable-diffusion.cpp

@@ -4050,13 +4050,56 @@ static std::optional<ImageGenerationLatents> prepare_image_generation_latents(sd
         LOG_INFO("IMG2IMG");
 
         if (request->strength < 1.f) {
-            size_t t_enc = static_cast<size_t>(plan->sample_steps * request->strength);
-            if (t_enc == static_cast<size_t>(plan->sample_steps)) {
-                t_enc--;
+            bool strength_as_noise_level = false;
+            bool force_first_sigma       = false;
+            for (const auto& [key, value] : parse_key_value_args(sd_img_gen_params->sample_params.extra_sample_args, "img2img arg")) {
+                if (key == "strength_as_noise_level") {
+                    if (!parse_strict_bool(value, strength_as_noise_level)) {
+                        LOG_WARN("ignoring invalid img2img sample arg '%s=%s'", key.c_str(), value.c_str());
+                    }
+                } else if (key == "force_first_sigma") {
+                    if (!parse_strict_bool(value, force_first_sigma)) {
+                        LOG_WARN("ignoring invalid img2img sample arg '%s=%s'", key.c_str(), value.c_str());
+                    }
+                }
+            }
+
+            size_t t_enc;
+            float target_sigma = -1;
+            if (!strength_as_noise_level) {
+                t_enc = static_cast<size_t>(plan->sample_steps * request->strength);
+                if (t_enc == static_cast<size_t>(plan->sample_steps)) {
+                    t_enc--;
+                }
+            } else {
+                LOG_DEBUG("Interpreting denoise strength as relative noise level");
+                // assume x_noised = K * (x * (1-noise_level) + noise * noise_level) = K * lerp(x, noise, noise_level)
+                // K = 1, noise_level = sigma for flow models
+                // K = 1+sigma, noise_level=sigma/(1+sigma) for diffusion models
+                float target_noise_level = request->strength;
+                target_sigma             = sd_ctx->sd->denoiser->noise_level_to_sigma(target_noise_level);
+                size_t start_index       = 0;
+                for (size_t i = 0; i < plan->sigmas.size(); ++i) {
+                    if (plan->sigmas[i] <= target_sigma) {
+                        start_index = i;
+                        break;
+                    }
+                }
+
+                if (start_index >= plan->sigmas.size() - 1) {
+                    start_index = plan->sigmas.size() - 2;  // Leave at least 1 step
+                }
+                t_enc = plan->sample_steps - start_index - 1;
             }
             LOG_INFO("target t_enc is %zu steps", t_enc);
             std::vector<float> sigma_sched;
             sigma_sched.assign(plan->sigmas.begin() + plan->sample_steps - t_enc - 1, plan->sigmas.end());
+
+            if (target_sigma > 0 && force_first_sigma && strength_as_noise_level) {
+                LOG_DEBUG("force_first_sigma to %.4f (from %.4f)", target_sigma, sigma_sched[0]);
+                sigma_sched[0] = target_sigma;
+            }
+
             plan->sigmas       = std::move(sigma_sched);
             plan->sample_steps = static_cast<int>(plan->sigmas.size() - 1);
         }