From 9c7561ec7e830e22ff259687b1e75136692364e7 Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Wed, 1 Jan 2025 20:13:15 +0100 Subject: [PATCH 1/7] Updated build-pipeline for new version --- .github/workflows/backends.yml | 98 +++++++++++-------- .../StableDiffusion.NET.Backend.Rocm.nuspec | 2 +- StableDiffusion.NET/Backends/RocmBackend.cs | 6 +- 3 files changed, 60 insertions(+), 46 deletions(-) diff --git a/.github/workflows/backends.yml b/.github/workflows/backends.yml index cfbe27b..3f9be4f 100644 --- a/.github/workflows/backends.yml +++ b/.github/workflows/backends.yml @@ -23,17 +23,15 @@ jobs: matrix: include: - build: 'noavx' - defines: '-DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON' + defines: 'DGGML_NATIVE=OFF -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx' - defines: '-DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON' + defines: 'DGGML_NATIVE=OFF -DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx2' - defines: '-DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: 'DGGML_NATIVE=OFF -DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx512' - defines: '-DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: 'DGGML_NATIVE=OFF -DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON' - build: 'cuda12' - defines: '-DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON' - - build: 'rocm5' - defines: '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS="gfx1100;gfx1102;gfx1030" -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON' - build: 'vulkan' defines: "-DSD_VULKAN=ON -DSD_BUILD_SHARED_LIBS=ON" @@ -49,27 +47,11 @@ jobs: - name: Install cuda-toolkit id: cuda-toolkit-12 if: ${{ matrix.build == 'cuda12' }} - uses: Jimver/cuda-toolkit@v0.2.16 + uses: Jimver/cuda-toolkit@v0.2.19 with: - cuda: '12.5.0' + cuda: '12.6.2' method: network sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]' - use-github-cache: false - use-local-cache: false - - - name: Install rocm-toolkit - id: rocm-toolkit - if: ${{ matrix.build == 'rocm5' }} - uses: Cyberhan123/rocm-toolkit@v0.1.0 - with: - rocm: '5.5.0' - - - name: Install Ninja - id: install-ninja - if: ${{ matrix.build == 'rocm5' }} - uses: urkle/action-get-ninja@v1 - with: - version: 1.11.1 - name: Install Vulkan SDK id: get_vulkan @@ -90,20 +72,11 @@ jobs: - name: Upload artifact id: upload_artifact - if: ${{ matrix.build != 'rocm5' }} uses: actions/upload-artifact@v4.3.1 with: name: windows-${{ matrix.build }} path: .\build\bin\Release\stable-diffusion.dll - - name: Upload artifact Rocm - id: upload_artifact_rocm - if: ${{ matrix.build == 'rocm5' }} - uses: actions/upload-artifact@v4.3.1 - with: - name: windows-${{ matrix.build }} - path: .\build\bin\stable-diffusion.dll - windows-2019: runs-on: windows-2019 @@ -111,7 +84,7 @@ jobs: matrix: include: - build: 'cuda11' - defines: '-DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler"' + defines: '-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler"' steps: - name: Checkout @@ -157,8 +130,9 @@ jobs: shell: bash env: - WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/7dff44ba-e3af-4448-841c-0d616c8da6e7/w_BaseKit_p_2024.1.0.595_offline.exe - WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel + WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b380d914-366b-4b77-a74a-05e3c38b3514/intel-oneapi-base-toolkit-2025.0.0.882_offline.exe + WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel + ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI" steps: - name: Checkout @@ -194,6 +168,49 @@ jobs: name: windows-sycl path: .\build\bin\stable-diffusion.dll + windows-hip: + runs-on: windows-latest + + steps: + - name: Install + id: depends + run: | + $ErrorActionPreference = "Stop" + write-host "Downloading AMD HIP SDK Installer" + Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q4-WinSvr2022-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe" + write-host "Installing AMD HIP SDK" + Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -Wait + write-host "Completed AMD HIP SDK installation" + - name: Verify ROCm + id: verify + run: | + & 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' --version + + - name: Checkout + id: checkout + uses: actions/checkout@v4.1.2 + with: + repository: 'leejet/stable-diffusion.cpp' + ref: '${{ github.event.inputs.commit }}' + submodules: recursive + + - name: Build + id: cmake_build + run: | + $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path) + $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}" + mkdir build + cd build + cmake -G "Unix Makefiles" .. -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" -DGGML_HIP=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS="gfx1100;gfx1101;gfx1030" -DGGML_RPC=ON -DSD_BUILD_SHARED_LIBS=ON + cmake --build . --config Release + + - name: Upload artifact + id: upload_artifact + uses: actions/upload-artifact@v4.3.1 + with: + name: windows-rocm6 + path: .\build\bin\stable-diffusion.dll + linux: runs-on: ubuntu-latest strategy: @@ -208,9 +225,9 @@ jobs: - build: 'avx512' defines: '-DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON' - build: 'cuda11' - defines: '-DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON' - build: 'cuda12' - defines: '-DSD_CUBLAS=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON' - build: 'sycl' defines: '-DSD_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DSD_BUILD_SHARED_LIBS=ON' @@ -307,7 +324,7 @@ jobs: run: | mkdir build cd build - cmake .. -DCMAKE_C_COMPILER=hipcc -DCMAKE_CXX_COMPILER=hipcc -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS="gfx1100;gfx1102;gfx1030" -DSD_BUILD_SHARED_LIBS=ON + cmake .. -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" -DGGML_HIP=ON -DSD_BUILD_SHARED_LIBS=ON cmake --build . --config Release - name: Upload artifact Rocm @@ -361,6 +378,7 @@ jobs: needs: - windows + - windows-hip - windows-sycl - linux - linux-hip diff --git a/Backends/StableDiffusion.NET.Backend.Rocm.nuspec b/Backends/StableDiffusion.NET.Backend.Rocm.nuspec index ad46784..bce4c19 100644 --- a/Backends/StableDiffusion.NET.Backend.Rocm.nuspec +++ b/Backends/StableDiffusion.NET.Backend.Rocm.nuspec @@ -18,7 +18,7 @@ - + diff --git a/StableDiffusion.NET/Backends/RocmBackend.cs b/StableDiffusion.NET/Backends/RocmBackend.cs index cfbcf42..d72c941 100644 --- a/StableDiffusion.NET/Backends/RocmBackend.cs +++ b/StableDiffusion.NET/Backends/RocmBackend.cs @@ -14,11 +14,7 @@ public partial class RocmBackend : IBackend public int Priority { get; set; } = 10; - public bool IsAvailable => ((RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - && RocmVersion is 5) - || (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) - && RocmVersion is 6)) - && (RuntimeInformation.OSArchitecture == Architecture.X64); + public bool IsAvailable => (RocmVersion == 6) && (RuntimeInformation.OSArchitecture == Architecture.X64); public string PathPart => RocmVersion switch { From 9190b76ecec61c3470ea2e1fc78de81fe98775f8 Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Thu, 2 Jan 2025 18:12:06 +0100 Subject: [PATCH 2/7] Fixed cuda build --- .github/workflows/backends.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/backends.yml b/.github/workflows/backends.yml index 3f9be4f..4fc2134 100644 --- a/.github/workflows/backends.yml +++ b/.github/workflows/backends.yml @@ -23,15 +23,15 @@ jobs: matrix: include: - build: 'noavx' - defines: 'DGGML_NATIVE=OFF -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx' - defines: 'DGGML_NATIVE=OFF -DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx2' - defines: 'DGGML_NATIVE=OFF -DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx512' - defines: 'DGGML_NATIVE=OFF -DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON' - build: 'cuda12' - defines: '-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DSD_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="61;89" -DSD_BUILD_SHARED_LIBS=ON' - build: 'vulkan' defines: "-DSD_VULKAN=ON -DSD_BUILD_SHARED_LIBS=ON" @@ -84,7 +84,7 @@ jobs: matrix: include: - build: 'cuda11' - defines: '-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler"' + defines: '-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_ARCHITECTURES="61" -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler"' steps: - name: Checkout @@ -217,17 +217,17 @@ jobs: matrix: include: - build: 'noavx' - defines: '-DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx' - defines: '-DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx2' - defines: '-DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx512' - defines: '-DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON' - build: 'cuda11' - defines: '-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DSD_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="61" -DSD_BUILD_SHARED_LIBS=ON' - build: 'cuda12' - defines: '-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DSD_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="61;89" -DSD_BUILD_SHARED_LIBS=ON' - build: 'sycl' defines: '-DSD_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DSD_BUILD_SHARED_LIBS=ON' @@ -341,13 +341,13 @@ jobs: matrix: include: - build: 'noavx' - defines: '-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx' - defines: '-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX2=OFF -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx2' - defines: '-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX2=ON -DSD_BUILD_SHARED_LIBS=ON' - build: 'avx512' - defines: '-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON' + defines: '-DGGML_NATIVE=OFF -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -DGGML_AVX512=ON -DSD_BUILD_SHARED_LIBS=ON' steps: - name: Checkout From f79b74efccf2c1b7f671177760313a7108f0cb2a Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Fri, 3 Jan 2025 14:39:50 +0100 Subject: [PATCH 3/7] Added flash attention and slg layer skip parameters; added comments for parameters --- StableDiffusion.NET/Enums/Quantization.cs | 2 + .../DiffusionModelBuilderExtension.cs | 8 ++ StableDiffusion.NET/Models/DiffusionModel.cs | 45 +++++++++-- .../Models/Parameter/ControlNetParameter.cs | 32 ++++++++ .../Parameter/DiffusionModelParameter.cs | 80 ++++++++++++++++++- .../Models/Parameter/DiffusionParameter.cs | 60 ++++++++++++++ .../Extensions/DiffusionParameterExtension.cs | 28 +++++++ .../Interfaces/IDiffusionModelParameter.cs | 1 + .../Models/Parameter/PhotoMakerParameter.cs | 11 +++ .../Models/Parameter/UpscaleModelParameter.cs | 11 +++ StableDiffusion.NET/Native/Native.cs | 17 +++- 11 files changed, 284 insertions(+), 11 deletions(-) diff --git a/StableDiffusion.NET/Enums/Quantization.cs b/StableDiffusion.NET/Enums/Quantization.cs index 39e886a..2658c63 100644 --- a/StableDiffusion.NET/Enums/Quantization.cs +++ b/StableDiffusion.NET/Enums/Quantization.cs @@ -37,6 +37,8 @@ public enum Quantization Q4_0_4_4 = 31, Q4_0_4_8 = 32, Q4_0_8_8 = 33, + TQ1_0 = 34, + TQ2_0 = 35, Unspecified } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs b/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs index e156e30..53ae229 100644 --- a/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs +++ b/StableDiffusion.NET/Models/Builder/Extensions/DiffusionModelBuilderExtension.cs @@ -119,4 +119,12 @@ public static T WithSchedule(this T builder, Schedule schedule) return builder; } + + public static T WithFlashAttention(this T builder, bool flashAttention = true) + where T : IDiffusionModelBuilder + { + builder.Parameter.FlashAttention = flashAttention; + + return builder; + } } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/DiffusionModel.cs b/StableDiffusion.NET/Models/DiffusionModel.cs index cbb2e7d..e53b698 100644 --- a/StableDiffusion.NET/Models/DiffusionModel.cs +++ b/StableDiffusion.NET/Models/DiffusionModel.cs @@ -59,7 +59,8 @@ private void Initialize() ModelParameter.Schedule, ModelParameter.KeepClipOnCPU, ModelParameter.KeepControlNetOnCPU, - ModelParameter.KeepVaeOnCPU); + ModelParameter.KeepVaeOnCPU, + ModelParameter.FlashAttention); if (_ctx == null) throw new NullReferenceException("Failed to initialize diffusion-model."); } @@ -122,7 +123,12 @@ public IImage TextToImage(string prompt, DiffusionParameter? parameter parameter.ControlNet.Strength, parameter.PhotoMaker.StyleRatio, parameter.PhotoMaker.NormalizeInput, - parameter.PhotoMaker.InputIdImageDirectory); + parameter.PhotoMaker.InputIdImageDirectory, + parameter.SkipLayers, + parameter.SkipLayers.Length, + parameter.SlgScale, + parameter.SkipLayerStart, + parameter.SkipLayerEnd); Marshal.FreeHGlobal((nint)nativeControlNetImage.data); } @@ -152,7 +158,12 @@ public IImage TextToImage(string prompt, DiffusionParameter? parameter parameter.ControlNet.Strength, parameter.PhotoMaker.StyleRatio, parameter.PhotoMaker.NormalizeInput, - parameter.PhotoMaker.InputIdImageDirectory); + parameter.PhotoMaker.InputIdImageDirectory, + parameter.SkipLayers, + parameter.SkipLayers.Length, + parameter.SlgScale, + parameter.SkipLayerStart, + parameter.SkipLayerEnd); } } } @@ -174,7 +185,12 @@ public IImage TextToImage(string prompt, DiffusionParameter? parameter 0, parameter.PhotoMaker.StyleRatio, parameter.PhotoMaker.NormalizeInput, - parameter.PhotoMaker.InputIdImageDirectory); + parameter.PhotoMaker.InputIdImageDirectory, + parameter.SkipLayers, + parameter.SkipLayers.Length, + parameter.SlgScale, + parameter.SkipLayerStart, + parameter.SkipLayerEnd); } return ImageHelper.ToImage(result); @@ -246,7 +262,12 @@ private IImage ImageToImage(string prompt, Native.sd_image_t image, Di parameter.ControlNet.Strength, parameter.PhotoMaker.StyleRatio, parameter.PhotoMaker.NormalizeInput, - parameter.PhotoMaker.InputIdImageDirectory); + parameter.PhotoMaker.InputIdImageDirectory, + parameter.SkipLayers, + parameter.SkipLayers.Length, + parameter.SlgScale, + parameter.SkipLayerStart, + parameter.SkipLayerEnd); Marshal.FreeHGlobal((nint)nativeControlNetImage.data); } @@ -278,7 +299,12 @@ private IImage ImageToImage(string prompt, Native.sd_image_t image, Di parameter.ControlNet.Strength, parameter.PhotoMaker.StyleRatio, parameter.PhotoMaker.NormalizeInput, - parameter.PhotoMaker.InputIdImageDirectory); + parameter.PhotoMaker.InputIdImageDirectory, + parameter.SkipLayers, + parameter.SkipLayers.Length, + parameter.SlgScale, + parameter.SkipLayerStart, + parameter.SkipLayerEnd); } } } @@ -302,7 +328,12 @@ private IImage ImageToImage(string prompt, Native.sd_image_t image, Di 0, parameter.PhotoMaker.StyleRatio, parameter.PhotoMaker.NormalizeInput, - parameter.PhotoMaker.InputIdImageDirectory); + parameter.PhotoMaker.InputIdImageDirectory, + parameter.SkipLayers, + parameter.SkipLayers.Length, + parameter.SlgScale, + parameter.SkipLayerStart, + parameter.SkipLayerEnd); } return ImageHelper.ToImage(result); diff --git a/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs b/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs index 9c89dd2..406435c 100644 --- a/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/ControlNetParameter.cs @@ -8,12 +8,44 @@ public sealed class ControlNetParameter { public bool IsEnabled => Image != null; + /// + /// image condition, control net + /// public IImage? Image { get; set; } = null; + + /// + /// strength to apply Control Net (default: 0.9) + /// 1.0 corresponds to full destruction of information in init image + /// public float Strength { get; set; } = 0.9f; + + /// + /// apply canny preprocessor (edge detection) + /// public bool CannyPreprocess { get; set; } = false; + + /// + /// + /// public float CannyHighThreshold { get; set; } = 0.08f; + + /// + /// + /// public float CannyLowThreshold { get; set; } = 0.08f; + + /// + /// + /// public float CannyWeak { get; set; } = 0.8f; + + /// + /// + /// public float CannyStrong { get; set; } = 1.0f; + + /// + /// + /// public bool CannyInverse { get; set; } = false; } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs index 8a54a79..da59ebc 100644 --- a/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/DiffusionModelParameter.cs @@ -4,36 +4,114 @@ public sealed class DiffusionModelParameter : IDiffusionModelParameter, IQuantiz { public DiffusionModelType DiffusionModelType { get; set; } = DiffusionModelType.None; + /// + /// path to vae + /// public string VaePath { get; set; } = string.Empty; + + /// + /// path to taesd. Using Tiny AutoEncoder for fast decoding (low quality) + /// public string TaesdPath { get; set; } = string.Empty; + /// + /// lora model directory + /// public string LoraModelDirectory { get; set; } = string.Empty; + + /// + /// path to embeddings + /// public string EmbeddingsDirectory { get; set; } = string.Empty; + + /// + /// path to control net model + /// public string ControlNetPath { get; set; } = string.Empty; + /// + /// number of threads to use during computation (default: -1) + /// If threads = -1, then threads will be set to the number of CPU physical cores + /// public int ThreadCount { get; set; } = 1; + /// + /// + /// public bool VaeDecodeOnly { get; set; } = false; + + /// + /// process vae in tiles to reduce memory usage + /// public bool VaeTiling { get; set; } = false; + + /// + /// keep controlnet in cpu + /// public bool KeepControlNetOnCPU { get; set; } = false; + + /// + /// keep clip in cpu (for low vram) + /// public bool KeepClipOnCPU { get; set; } = false; + + /// + /// keep vae in cpu (for low vram) + /// public bool KeepVaeOnCPU { get; set; } = false; + /// + /// use flash attention in the diffusion model (for low vram) + /// Might lower quality, since it implies converting k and v to f16. + /// This might crash if it is not supported by the backend. + /// + public bool FlashAttention { get; set; } = false; + + /// + /// RNG (default: Standard) + /// public RngType RngType { get; set; } = RngType.Standard; + + /// + /// Denoiser sigma schedule (default: Default) + /// public Schedule Schedule { get; set; } = Schedule.Default; + /// + /// + /// public Quantization Quantization { get; set; } = Quantization.Unspecified; // SD <= 3 only + /// + /// path to full model + /// public string ModelPath { get; set; } = string.Empty; + + /// + /// path to PHOTOMAKER stacked id embeddings + /// public string StackedIdEmbeddingsDirectory { get; set; } = string.Empty; // Flux & SD3.5 only + /// + /// path to the standalone diffusion model + /// public string DiffusionModelPath { get; set; } = string.Empty; + + /// + /// path to the clip-l text encoder + /// public string ClipLPath { get; set; } = string.Empty; - public string T5xxlPath { get; set; } = string.Empty; + /// + /// path to the the t5xxl text encoder + /// + public string T5xxlPath { get; set; } = string.Empty; // SD3.5 only + /// + /// path to the clip-g text encoder + /// public string ClipGPath { get; set; } = string.Empty; } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs b/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs index c6809e5..03d819b 100644 --- a/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/DiffusionParameter.cs @@ -12,22 +12,82 @@ public sealed class DiffusionParameter public static DiffusionParameter SD3_5Default => new() { Width = 1024, Height = 1024, CfgScale = 4.5f, Guidance = 1f, SampleSteps = 20, SampleMethod = Sampler.Euler }; public static DiffusionParameter FluxDefault => new() { Width = 1024, Height = 1024, CfgScale = 1, Guidance = 3.5f, SampleSteps = 20, SampleMethod = Sampler.Euler }; + /// + /// the negative prompt (default: ""); + /// public string NegativePrompt { get; set; } = string.Empty; + + /// + /// image width, in pixel space (default: 512) + /// public int Width { get; set; } = 512; + + /// + /// image height, in pixel space (default: 512) + /// public int Height { get; set; } = 512; + + /// + /// sampling method (default: Euler_A) + /// public Sampler SampleMethod { get; set; } = Sampler.Euler_A; + + /// + /// number of sample steps (default: 25) + /// public int SampleSteps { get; set; } = 25; + + /// + /// RNG seed. use -1 for a random seed (default: -1) + /// public long Seed { get; set; } = -1; + + /// + /// strength for noising/unnoising (default: 0.7) + /// public float Strength { get; set; } = 0.7f; + + /// + /// ignore last layers of CLIP network; 1 ignores none, 2 ignores one layer (default: -1) + /// -1 represents unspecified, will be 1 for SD1.x, 2 for SD2.x + /// public int ClipSkip { get; set; } = -1; + /// + /// skip layer guidance (SLG) scale, only for DiT models: (default: 0) + /// 0 means disabled, a value of 2.5 is nice for sd3.5 medium + /// + public float SlgScale { get; set; } = 0f; + + /// + /// Layers to skip for SLG steps: (default: [7,8,9]) + /// + public int[] SkipLayers { get; set; } = [7, 8, 9]; + + /// + /// SLG enabling point: (default: 0.01) + /// + public float SkipLayerStart { get; set; } = 0.01f; + + /// + /// SLG disabling point: (default: 0.2) + /// + public float SkipLayerEnd { get; set; } = 0.2f; + public ControlNetParameter ControlNet { get; } = new(); // Stable Diffusion only + /// + /// unconditional guidance scale: (default: 7.5) + /// public float CfgScale { get; set; } = 7.5f; + public PhotoMakerParameter PhotoMaker { get; } = new(); // Flux only + /// + /// + /// public float Guidance { get; set; } = 3.5f; #endregion diff --git a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs index 52d4bef..53f4b3e 100644 --- a/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs +++ b/StableDiffusion.NET/Models/Parameter/Extensions/DiffusionParameterExtension.cs @@ -64,6 +64,34 @@ public static DiffusionParameter WithNegativePrompt(this DiffusionParameter para return parameter; } + public static DiffusionParameter WithSlgScale(this DiffusionParameter parameter, float slgScale) + { + parameter.SlgScale = slgScale; + + return parameter; + } + + public static DiffusionParameter WithSkipLayers(this DiffusionParameter parameter, int[] layers) + { + parameter.SkipLayers = layers; + + return parameter; + } + + public static DiffusionParameter WithSkipLayerStart(this DiffusionParameter parameter, float skipLayerStart) + { + parameter.SkipLayerStart = skipLayerStart; + + return parameter; + } + + public static DiffusionParameter WithSkipLayerEnd(this DiffusionParameter parameter, float skipLayerEnd) + { + parameter.SkipLayerEnd = skipLayerEnd; + + return parameter; + } + public static DiffusionParameter WithControlNet(this DiffusionParameter parameter, IImage image, float? strength = null) { parameter.ControlNet.Image = image; diff --git a/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs b/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs index e18ed24..12569ed 100644 --- a/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/Interfaces/IDiffusionModelParameter.cs @@ -16,6 +16,7 @@ public interface IDiffusionModelParameter bool KeepControlNetOnCPU { get; set; } bool KeepClipOnCPU { get; set; } bool KeepVaeOnCPU { get; set; } + bool FlashAttention { get; set; } RngType RngType { get; set; } Schedule Schedule { get; set; } diff --git a/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs b/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs index cab62fd..57466e1 100644 --- a/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/PhotoMakerParameter.cs @@ -5,7 +5,18 @@ namespace StableDiffusion.NET; [PublicAPI] public sealed class PhotoMakerParameter { + /// + /// path to PHOTOMAKER input id images dir + /// public string InputIdImageDirectory { get; set; } = string.Empty; + + /// + /// strength for keeping input identity (default: 20) + /// public float StyleRatio { get; set; } = 20f; + + /// + /// normalize PHOTOMAKER input id images + /// public bool NormalizeInput { get; set; } = false; } \ No newline at end of file diff --git a/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs index 023a817..17844de 100644 --- a/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs +++ b/StableDiffusion.NET/Models/Parameter/UpscaleModelParameter.cs @@ -5,8 +5,19 @@ namespace StableDiffusion.NET; [PublicAPI] public sealed class UpscaleModelParameter : IQuantizedModelParameter { + /// + /// path to esrgan model. Upscale images after generate, just RealESRGAN_x4plus_anime_6B supported by now + /// public string ModelPath { get; set; } = string.Empty; + + /// + /// number of threads to use during computation (default: -1) + /// If threads = -1, then threads will be set to the number of CPU physical cores + /// public int ThreadCount { get; set; } = 1; + /// + /// + /// public Quantization Quantization { get; set; } = Quantization.F16; } \ No newline at end of file diff --git a/StableDiffusion.NET/Native/Native.cs b/StableDiffusion.NET/Native/Native.cs index 66f55ab..a532b48 100644 --- a/StableDiffusion.NET/Native/Native.cs +++ b/StableDiffusion.NET/Native/Native.cs @@ -69,7 +69,8 @@ internal struct sd_image_t schedule_t s, [MarshalAs(UnmanagedType.I1)] bool keep_clip_on_cpu, [MarshalAs(UnmanagedType.I1)] bool keep_control_net_cpu, - [MarshalAs(UnmanagedType.I1)] bool keep_vae_on_cpu); + [MarshalAs(UnmanagedType.I1)] bool keep_vae_on_cpu, + [MarshalAs(UnmanagedType.I1)] bool diffusion_flash_attn); [LibraryImport(LIB_NAME, EntryPoint = "free_sd_ctx")] internal static partial void free_sd_ctx(sd_ctx_t* sd_ctx); @@ -91,7 +92,12 @@ internal struct sd_image_t float control_strength, float style_strength, [MarshalAs(UnmanagedType.I1)] bool normalize_input, - [MarshalAs(UnmanagedType.LPStr)] string input_id_images_path); + [MarshalAs(UnmanagedType.LPStr)] string input_id_images_path, + int[] skip_layers, + int skip_layers_count, + float slg_scale, + float skip_layer_start, + float skip_layer_end); [LibraryImport(LIB_NAME, EntryPoint = "img2img")] internal static partial sd_image_t* img2img(sd_ctx_t* sd_ctx, @@ -112,7 +118,12 @@ internal struct sd_image_t float control_strength, float style_strength, [MarshalAs(UnmanagedType.I1)] bool normalize_input, - [MarshalAs(UnmanagedType.LPStr)] string input_id_images_path); + [MarshalAs(UnmanagedType.LPStr)] string input_id_images_path, + int[] skip_layers, + int skip_layers_count, + float slg_scale, + float skip_layer_start, + float skip_layer_end); [LibraryImport(LIB_NAME, EntryPoint = "img2vid")] internal static partial sd_image_t* img2vid(sd_ctx_t* sd_ctx, From b174c2aeb60b7f2f3543d572a5eb09448cde2830 Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Fri, 3 Jan 2025 14:56:19 +0100 Subject: [PATCH 4/7] Updated backend in example --- Examples/ImageCreationUI/ImageCreationUI.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Examples/ImageCreationUI/ImageCreationUI.csproj b/Examples/ImageCreationUI/ImageCreationUI.csproj index 1a68fce..1f97149 100644 --- a/Examples/ImageCreationUI/ImageCreationUI.csproj +++ b/Examples/ImageCreationUI/ImageCreationUI.csproj @@ -11,9 +11,9 @@ - - - + + + From b7e5ee7232ea5e3277e939becbc626f426f138fe Mon Sep 17 00:00:00 2001 From: DarthAffe Date: Fri, 3 Jan 2025 14:56:27 +0100 Subject: [PATCH 5/7] Added flash attention parameter to example --- Examples/ImageCreationUI/MainWindow.xaml | 5 +++++ Examples/ImageCreationUI/MainWindowViewModel.cs | 11 +++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Examples/ImageCreationUI/MainWindow.xaml b/Examples/ImageCreationUI/MainWindow.xaml index 13aa12f..a549ca1 100644 --- a/Examples/ImageCreationUI/MainWindow.xaml +++ b/Examples/ImageCreationUI/MainWindow.xaml @@ -91,6 +91,11 @@