Skip to content

Commit 50250bf

Browse files
leejetleejet
committed
sync: use new ggml ops
1 parent 193fb62 commit 50250bf

File tree

5 files changed

+35
-15
lines changed

5 files changed

+35
-15
lines changed

‎control.hpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ class ControlNetBlock : public GGMLBlock {
205205
struct ggml_tensor* x,
206206
struct ggml_tensor* hint,
207207
struct ggml_tensor* guided_hint,
208-
std::vector<float> timesteps,
208+
struct ggml_tensor* timesteps,
209209
struct ggml_tensor* context,
210210
struct ggml_tensor* y = NULL) {
211211
// x: [N, in_channels, h, w] or [N, in_channels/2, h, w]
@@ -231,7 +231,7 @@ class ControlNetBlock : public GGMLBlock {
231231

232232
auto middle_block_out = std::dynamic_pointer_cast<Conv2d>(blocks["middle_block_out.0"]);
233233

234-
auto t_emb = new_timestep_embedding(ctx, allocr, timesteps, model_channels); // [N, model_channels]
234+
auto t_emb = ggml_nn_timestep_embedding(ctx, timesteps, model_channels); // [N, model_channels]
235235

236236
auto emb = time_embed_0->forward(ctx, t_emb);
237237
emb = ggml_silu_inplace(ctx, emb);
@@ -386,7 +386,7 @@ struct ControlNet : public GGMLModule {
386386

387387
struct ggml_cgraph* build_graph(struct ggml_tensor* x,
388388
struct ggml_tensor* hint,
389-
std::vector<float> timesteps,
389+
struct ggml_tensor* timesteps,
390390
struct ggml_tensor* context,
391391
struct ggml_tensor* y = NULL) {
392392
struct ggml_cgraph* gf = ggml_new_graph_custom(compute_ctx, CONTROL_NET_GRAPH_SIZE, false);
@@ -395,6 +395,7 @@ struct ControlNet : public GGMLModule {
395395
hint = to_backend(hint);
396396
context = to_backend(context);
397397
y = to_backend(y);
398+
timesteps = to_backend(timesteps);
398399

399400
auto outs = control_net.forward(compute_ctx,
400401
compute_allocr,
@@ -420,7 +421,7 @@ struct ControlNet : public GGMLModule {
420421
void compute(int n_threads,
421422
struct ggml_tensor* x,
422423
struct ggml_tensor* hint,
423-
std::vector<float> timesteps,
424+
struct ggml_tensor* timesteps,
424425
struct ggml_tensor* context,
425426
struct ggml_tensor* y,
426427
struct ggml_tensor** output = NULL,

‎ggml

Submodule ggml updated from 9a5ce30 to 4ce2039

‎ggml_extend.hpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -606,6 +606,13 @@ __STATIC_INLINE__ float ggml_backend_tensor_get_f32(ggml_tensor* tensor) {
606606
return value;
607607
}
608608

609+
__STATIC_INLINE__ struct ggml_tensor* vector_to_ggml_tensor(struct ggml_context* ctx,
610+
const std::vector<float>& vec) {
611+
struct ggml_tensor* t = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, vec.size());
612+
memcpy(t->data, (const void*)vec.data(), ggml_nbytes(t));
613+
return t;
614+
}
615+
609616
__STATIC_INLINE__ std::vector<float> arange(float start, float end, float step = 1.f) {
610617
std::vector<float> result;
611618

@@ -675,6 +682,15 @@ __STATIC_INLINE__ struct ggml_tensor* new_timestep_embedding(struct ggml_context
675682
return embedding;
676683
}
677684

685+
686+
__STATIC_INLINE__ struct ggml_tensor * ggml_nn_timestep_embedding(
687+
struct ggml_context * ctx,
688+
struct ggml_tensor * timesteps,
689+
int dim,
690+
int max_period = 10000) {
691+
return ggml_timestep_embedding(ctx, timesteps, dim, max_period);
692+
}
693+
678694
// struct GGMLComputeGraph {
679695
// virtual void init(struct ggml_context* ctx, ggml_type wtype) = 0;
680696
// virtual std::string get_desc() = 0;

‎stable-diffusion.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,8 @@ class StableDiffusionGGML {
363363
struct ggml_tensor* c = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, 1024, 2, 1, 1);
364364
ggml_set_f32(c, 0.5);
365365

366-
std::vector<float> timesteps = {999.f}; // [N, ]
366+
struct ggml_tensor* timesteps = ggml_new_tensor_1d(work_ctx, GGML_TYPE_F32, 1);
367+
ggml_set_f32(timesteps, 999);
367368
int64_t t0 = ggml_time_ms();
368369
struct ggml_tensor* out = ggml_dup_tensor(work_ctx, x_t);
369370
diffusion_model->compute(n_threads, x_t, timesteps, c, NULL, NULL, -1, {}, 0.f, &out);
@@ -675,7 +676,8 @@ class StableDiffusionGGML {
675676
}
676677

677678
float t = denoiser->schedule->sigma_to_t(sigma);
678-
std::vector<float> timesteps(x->ne[3], t); // [N, ]
679+
std::vector<float> timesteps_vec(x->ne[3], t); // [N, ]
680+
auto timesteps = vector_to_ggml_tensor(work_ctx, timesteps_vec);
679681

680682
copy_ggml_tensor(noised_input, input);
681683
// noised_input = noised_input * c_in

‎unet.hpp

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -112,9 +112,9 @@ class SpatialVideoTransformer : public SpatialTransformer {
112112
x = ggml_cont(ctx, ggml_permute(ctx, x, 1, 2, 0, 3)); // [N, h, w, inner_dim]
113113
x = ggml_reshape_3d(ctx, x, inner_dim, w * h, n); // [N, h * w, inner_dim]
114114

115-
std::vector<float> num_frames = arange(0, timesteps);
115+
auto num_frames = ggml_arange(ctx, 0, timesteps, 1);
116116
// since b is 1, no need to do repeat
117-
auto t_emb = new_timestep_embedding(ctx, allocr, num_frames, in_channels, max_time_embed_period); // [N, in_channels]
117+
auto t_emb = ggml_nn_timestep_embedding(ctx, num_frames, in_channels, max_time_embed_period); // [N, in_channels]
118118

119119
auto emb = time_pos_embed_0->forward(ctx, t_emb);
120120
emb = ggml_silu_inplace(ctx, emb);
@@ -377,7 +377,7 @@ class UnetModelBlock : public GGMLBlock {
377377
struct ggml_tensor* forward(struct ggml_context* ctx,
378378
struct ggml_allocr* allocr,
379379
struct ggml_tensor* x,
380-
std::vector<float> timesteps,
380+
struct ggml_tensor* timesteps,
381381
struct ggml_tensor* context,
382382
struct ggml_tensor* c_concat = NULL,
383383
struct ggml_tensor* y = NULL,
@@ -386,7 +386,6 @@ class UnetModelBlock : public GGMLBlock {
386386
float control_strength = 0.f) {
387387
// x: [N, in_channels, h, w] or [N, in_channels/2, h, w]
388388
// timesteps: [N,]
389-
// t_emb: [N, model_channels] timestep_embedding(timesteps, model_channels)
390389
// context: [N, max_position, hidden_size] or [1, max_position, hidden_size]. for example, [N, 77, 768]
391390
// c_concat: [N, in_channels, h, w] or [1, in_channels, h, w]
392391
// y: [N, adm_in_channels] or [1, adm_in_channels]
@@ -417,7 +416,7 @@ class UnetModelBlock : public GGMLBlock {
417416
auto out_0 = std::dynamic_pointer_cast<GroupNorm32>(blocks["out.0"]);
418417
auto out_2 = std::dynamic_pointer_cast<Conv2d>(blocks["out.2"]);
419418

420-
auto t_emb = new_timestep_embedding(ctx, allocr, timesteps, model_channels); // [N, model_channels]
419+
auto t_emb = ggml_nn_timestep_embedding(ctx, timesteps, model_channels); // [N, model_channels]
421420

422421
auto emb = time_embed_0->forward(ctx, t_emb);
423422
emb = ggml_silu_inplace(ctx, emb);
@@ -561,7 +560,7 @@ struct UNetModel : public GGMLModule {
561560
}
562561

563562
struct ggml_cgraph* build_graph(struct ggml_tensor* x,
564-
std::vector<float> timesteps,
563+
struct ggml_tensor* timesteps,
565564
struct ggml_tensor* context,
566565
struct ggml_tensor* c_concat = NULL,
567566
struct ggml_tensor* y = NULL,
@@ -577,6 +576,7 @@ struct UNetModel : public GGMLModule {
577576
x = to_backend(x);
578577
context = to_backend(context);
579578
y = to_backend(y);
579+
timesteps = to_backend(timesteps);
580580

581581
for (int i = 0; i < controls.size(); i++) {
582582
controls[i] = to_backend(controls[i]);
@@ -600,7 +600,7 @@ struct UNetModel : public GGMLModule {
600600

601601
void compute(int n_threads,
602602
struct ggml_tensor* x,
603-
std::vector<float> timesteps,
603+
struct ggml_tensor* timesteps,
604604
struct ggml_tensor* context,
605605
struct ggml_tensor* c_concat,
606606
struct ggml_tensor* y,
@@ -638,7 +638,8 @@ struct UNetModel : public GGMLModule {
638638
int num_video_frames = 3;
639639

640640
auto x = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, 8, 8, 8, num_video_frames);
641-
std::vector<float> timesteps(num_video_frames, 999.f);
641+
std::vector<float> timesteps_vec(num_video_frames, 999.f);
642+
auto timesteps = vector_to_ggml_tensor(work_ctx, timesteps_vec);
642643
ggml_set_f32(x, 0.5f);
643644
// print_ggml_tensor(x);
644645

0 commit comments

Comments
 (0)