fix: set eps of ggml_norm(LayerNorm) to 1e-5

leejet · leejet · commit ed374983f334 · 2023-10-27T00:50:23.000+08:00
diff --git a/stable-diffusion.cpp b/stable-diffusion.cpp
@@ -18,6 +18,8 @@
 #include "rng_philox.h"
 #include "stable-diffusion.h"
 
+#define EPS 1e-05
+
 static SDLogLevel log_level = SDLogLevel::INFO;
 
 #define __FILENAME__ "stable-diffusion.cpp"
@@ -586,7 +588,7 @@ struct ResidualAttentionBlock {
 
         // layer norm 1
         {
-            x = ggml_norm(ctx, x, 1e-6f);
+            x = ggml_norm(ctx, x, EPS);
             x = ggml_add(ctx,
                          ggml_mul(ctx, ggml_repeat(ctx, ln1_w, x), x),
                          ggml_repeat(ctx, ln1_b, x));
@@ -636,7 +638,7 @@ struct ResidualAttentionBlock {
 
         // layer norm 2
         {
-            x = ggml_norm(ctx, x, 1e-6f);
+            x = ggml_norm(ctx, x, EPS);
 
             x = ggml_add(ctx, ggml_mul(ctx, ggml_repeat(ctx, ln2_w, x), x),
                          ggml_repeat(ctx, ln2_b, x));
@@ -766,7 +768,7 @@ struct CLIPTextModel {
 
         // final layer norm
         {
-            x = ggml_norm(ctx, x, 1e-6f);
+            x = ggml_norm(ctx, x, EPS);
 
             x = ggml_add(ctx, ggml_mul(ctx, ggml_repeat(ctx, final_ln_w, x), x),
                          ggml_repeat(ctx, final_ln_b, x));
@@ -1200,7 +1202,7 @@ struct SpatialTransformer {
             // layer norm 1
             {
                 x = ggml_reshape_2d(ctx, x, c, w * h * n);
-                x = ggml_norm(ctx, x, 1e-6f);
+                x = ggml_norm(ctx, x, EPS);
                 x = ggml_add(ctx,
                              ggml_mul(ctx,
                                       ggml_repeat(ctx, transformer.norm1_w, x),
@@ -1248,7 +1250,7 @@ struct SpatialTransformer {
 
             // layer norm 2
             {
-                x = ggml_norm(ctx, x, 1e-6f);
+                x = ggml_norm(ctx, x, EPS);
                 x = ggml_add(ctx,
                              ggml_mul(ctx,
                                       ggml_repeat(ctx, transformer.norm2_w, x), x),
@@ -1299,7 +1301,7 @@ struct SpatialTransformer {
             // layer norm 3
             {
                 x = ggml_reshape_2d(ctx, x, c, h * w * n);  // [N * h * w, in_channels]
-                x = ggml_norm(ctx, x, 1e-6f);
+                x = ggml_norm(ctx, x, EPS);
                 x = ggml_add(ctx,
                              ggml_mul(ctx,
                                       ggml_repeat(ctx, transformer.norm3_w, x), x),
