Bug Summary

File:root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp
Warning:line 84, column 5
Value stored to 'b' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O2 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name delta-net-base.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +avx -target-feature +avx2 -target-feature +bmi2 -target-feature +f16c -target-feature +fma -target-feature +sse4.2 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -resource-dir /usr/lib/llvm-23/lib/clang/23 -include /root/firefox-clang/config/gcc_hidden.h -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/stl_wrappers -D _GLIBCXX_ASSERTIONS=1 -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/system_wrappers -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG=1 -D _GNU_SOURCE=1 -D GGML_USE_CPU=1 -D GGML_VERSION="GGML_VERSION" -D GGML_COMMIT="GGML_COMMIT" -D GGML_SHARED=1 -D LLAMA_SHARED=1 -D GGML_BUILD=1 -D LLAMA_BUILD=1 -D GGML_BACKEND_SHARED=1 -D GGML_BACKEND_BUILD=1 -D MOZ_HAS_MOZGLUE -I /root/firefox-clang/third_party/llama.cpp -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -I /root/firefox-clang/third_party/llama.cpp/ggml -I /root/firefox-clang/third_party/llama.cpp/ggml/include -I /root/firefox-clang/third_party/llama.cpp/ggml/src -I /root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu -I /root/firefox-clang/third_party/llama.cpp/include -I /root/firefox-clang/third_party/llama.cpp/src -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/c++/16 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/x86_64-linux-gnu/c++/16 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/c++/16/backward -internal-isystem /usr/lib/llvm-23/lib/clang/23/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=pessimizing-move -Wno-error=large-by-value-copy=128 -Wno-error=implicit-int-float-conversion -Wno-error=thread-safety-analysis -Wno-error=tautological-type-limit-compare -Wno-invalid-offsetof -Wno-range-loop-analysis -Wno-deprecated-anon-enum-enum-conversion -Wno-deprecated-enum-enum-conversion -Wno-inline-new-delete -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-vla-cxx-extension -Wno-unknown-warning-option -Wno-character-conversion -Wno-sign-compare -Wno-unused-function -Wno-tautological-unsigned-enum-zero-compare -Wno-implicit-fallthrough -Wno-unreachable-code -std=gnu++20 -fdeprecated-macro -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fno-rtti -fgnuc-version=4.2.1 -fno-implicit-modules -fskip-odr-check-in-gmf -fno-sized-deallocation -fno-aligned-allocation -fdiagnostics-absolute-paths -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -mllvm -dwarf-linkage-names=Abstract -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-09-01-224014-2642839-1 -x c++ /root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp
1#include "llama-impl.h"
2#include "llama-memory-recurrent.h"
3#include "models.h"
4
5// utility to get one slice from the third dimension
6// input dim: [x, y, c, b]
7// output dim: [x, y, 1, b]
8static ggml_tensor * get_slice_2d(ggml_context * ctx0, ggml_tensor * t, int64_t c) {
9 return ggml_view_4d(ctx0, t, t->ne[0], t->ne[1], 1, t->ne[3],
10 t->nb[1], t->nb[2], t->nb[3], t->nb[2] * c);
11}
12
13llm_build_delta_net_base::llm_build_delta_net_base(const llm_graph_params & params) : llm_graph_context(params) {}
14
15std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net_chunking(
16 ggml_tensor * q,
17 ggml_tensor * k,
18 ggml_tensor * v,
19 ggml_tensor * g,
20 ggml_tensor * b,
21 ggml_tensor * s,
22 int il) {
23 const int64_t S_k = q->ne[0];
24 const int64_t H_k = q->ne[1];
25 const int64_t n_tokens = q->ne[2];
26 const int64_t n_seqs = q->ne[3];
27
28 const int64_t S_v = v->ne[0];
29 const int64_t H_v = v->ne[1];
30 const bool kda = (g->ne[0] == S_k && g->ne[1] == H_k);
31
32 GGML_ASSERT(S_k == S_v)if (!(S_k == S_v)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 32, "GGML_ASSERT(%s) failed", "S_k == S_v");
33 GGML_ASSERT(H_v % H_k == 0)if (!(H_v % H_k == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 33, "GGML_ASSERT(%s) failed", "H_v % H_k == 0");
34
35 GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs)if (!(q->ne[0] == S_k && q->ne[1] == H_k &&
q->ne[2] == n_tokens && q->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 35, "GGML_ASSERT(%s) failed", "q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs"
);
36 GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs)if (!(k->ne[0] == S_k && k->ne[1] == H_k &&
k->ne[2] == n_tokens && k->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 36, "GGML_ASSERT(%s) failed", "k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs"
);
37 GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs)if (!(v->ne[0] == S_v && v->ne[1] == H_v &&
v->ne[2] == n_tokens && v->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 37, "GGML_ASSERT(%s) failed", "v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs"
);
38
39 GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v)if (!(g->ne[0] == 1 || g->ne[0] == S_v)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 39, "GGML_ASSERT(%s) failed", "g->ne[0] == 1 || g->ne[0] == S_v"
);
40 GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs)if (!(g->ne[1] == H_v && g->ne[2] == n_tokens &&
g->ne[3] == n_seqs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 40, "GGML_ASSERT(%s) failed", "g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs"
);
41 GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs)if (!(b->ne[0] == 1 && b->ne[1] == H_v &&
b->ne[2] == n_tokens && b->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 41, "GGML_ASSERT(%s) failed", "b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs"
);
42 GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs)if (!(s->ne[0] == S_v && s->ne[1] == S_v &&
s->ne[2] == H_v && s->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 42, "GGML_ASSERT(%s) failed", "s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs"
);
43
44 const float scale = 1.0f / sqrtf(S_k);
45
46 q = ggml_scale(ctx0, q, scale);
47
48 cb(q, "q_in", il);
49 cb(k, "k_in", il);
50 cb(v, "v_in", il);
51 cb(b, "b_in", il);
52 cb(g, "g_in", il);
53
54 q = ggml_permute(ctx0, q, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]
55 k = ggml_permute(ctx0, k, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]
56 v = ggml_permute(ctx0, v, 0, 2, 1, 3); // [S_v, n_tokens, H_v, n_seqs]
57 g = ggml_permute(ctx0, g, 0, 2, 1, 3); // [g_0, n_tokens, H_v, n_seqs]
58 b = ggml_permute(ctx0, b, 0, 2, 1, 3); // [ 1, n_tokens, H_v, n_seqs]
59
60 const int CS = kda ? 16 : 64; // chunk size
61
62 const int pad = (CS - n_tokens % CS) % CS;
63 const int n_chunks = (n_tokens + pad) / CS;
64
65 q = ggml_pad(ctx0, q, 0, pad, 0, 0);
66 k = ggml_pad(ctx0, k, 0, pad, 0, 0);
67 v = ggml_pad(ctx0, v, 0, pad, 0, 0);
68 g = ggml_pad(ctx0, g, 0, pad, 0, 0);
69 b = ggml_pad(ctx0, b, 0, pad, 0, 0);
70
71 ggml_tensor * v_b = ggml_mul(ctx0, v, b);
72 ggml_tensor * k_b = ggml_mul(ctx0, k, b);
73
74 cb(v_b, "v_b", il);
75 cb(k_b, "k_b", il);
76
77 q = ggml_reshape_4d(ctx0, q, S_k, CS, n_chunks, H_k * n_seqs);
78 k = ggml_reshape_4d(ctx0, k, S_k, CS, n_chunks, H_k * n_seqs);
79 k_b = ggml_reshape_4d(ctx0, k_b, S_k, CS, n_chunks, H_v * n_seqs);
80 v = ggml_reshape_4d(ctx0, v, S_v, CS, n_chunks, H_v * n_seqs);
81 v_b = ggml_reshape_4d(ctx0, v_b, S_v, CS, n_chunks, H_v * n_seqs);
82
83 g = ggml_reshape_4d(ctx0, g, g->ne[0], CS, n_chunks, H_v * n_seqs);
84 b = ggml_reshape_4d(ctx0, b, 1, CS, n_chunks, H_v * n_seqs);
Value stored to 'b' is never read
85
86 // [CS, g_0, n_chunks, H_v * n_seqs]
87 // TODO: extend ggml_cumsum with axis parameter to avoid transpose
88 ggml_tensor * g_cs = ggml_cumsum(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, g)));
89 cb(g_cs, "g_cs", il);
90
91 ggml_tensor * kb = nullptr;
92 ggml_tensor * kq = nullptr;
93 if (kda) {
94 const int64_t CHB = n_chunks * H_k * n_seqs;
95
96 ggml_tensor * g_cs_i = ggml_reshape_4d(ctx0, g_cs, CS, 1, S_k, CHB); // [chunk_size, 1, S_k, CHB]
97 ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, S_k, CHB); // [1, chunk_size, S_k, CHB]
98
99 g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, S_k, CHB); // [1, chunk_size, S_k, CHB] -> [chunk_size, chunk_size, S_k, CHB]
100
101 // decay_mask [chunk_size,chunk_size,S_k,CHB]
102 ggml_tensor * decay_mask;
103 decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i);
104 decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG);
105 decay_mask = ggml_exp(ctx0, decay_mask);
106 cb(decay_mask, "decay_mask", il);
107
108 // decay_mask [S_k,BT_j,BT_i,CHB] *Note* second and third chunk_sizes are switched
109 decay_mask = ggml_cont_4d(ctx0, ggml_permute(ctx0, decay_mask, 2, 1, 0, 3), S_k, CS, CS, CHB);
110
111 ggml_tensor * k_b_i = ggml_reshape_4d(ctx0, k_b, S_k, CS, 1, CHB);
112 ggml_tensor * k_j = ggml_reshape_4d(ctx0, k, S_k, 1, CS, CHB);
113 ggml_tensor * q_i = ggml_reshape_4d(ctx0, q, S_k, CS, 1, CHB);
114
115 ggml_tensor * decay_k_b_i = ggml_mul(ctx0, decay_mask, k_b_i);
116 ggml_tensor * decay_q_i = ggml_mul(ctx0, decay_mask, q_i);
117
118 // decay_k_b_i [S,BT,BT,CHB] @ k_j [S,1,BT,CHB] = Akk [BT,1,BT,CHB]
119 kb = ggml_mul_mat(ctx0, decay_k_b_i, k_j);
120 kq = ggml_mul_mat(ctx0, decay_q_i, k_j);
121
122 kb = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kb, CS, CS, n_chunks, H_v * n_seqs)));
123 kq = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kq, CS, CS, n_chunks, H_v * n_seqs)));
124 } else {
125 ggml_tensor * g_cs_i = g_cs;
126 ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, n_chunks, H_v * n_seqs);
127
128 g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, n_chunks, H_v * n_seqs);
129
130 // [CS, CS, n_chunks, H_v * n_seqs]
131 ggml_tensor * decay_mask;
132 decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i);
133 decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG);
134 decay_mask = ggml_exp(ctx0, decay_mask);
135 cb(decay_mask, "decay_mask", il);
136
137 // [CS, CS, n_chunks, H_k * n_seqs]
138 kb = ggml_mul_mat(ctx0, k, k_b);
139 kb = ggml_mul (ctx0, kb, decay_mask);
140
141 // [CS, CS, n_chunks, H_k * n_seqs]
142 kq = ggml_mul_mat(ctx0, k, q);
143 kq = ggml_mul(ctx0, kq, decay_mask);
144 }
145
146 kq = ggml_tri(ctx0, kq, GGML_TRI_TYPE_LOWER_DIAG);
147 cb(kq, "kq", il);
148
149 // [CS, CS, n_chunks, H_k * n_seqs]
150 ggml_tensor * attn;
151 attn = ggml_tri(ctx0, kb, GGML_TRI_TYPE_LOWER);
152 cb(attn, "attn", il);
153
154 ggml_tensor * identity;
155 identity = ggml_view_1d(ctx0, attn, CS, 0);
156 identity = ggml_fill (ctx0, identity, 1.0f);
157 identity = ggml_diag (ctx0, identity);
158
159 ggml_tensor * lhs = ggml_add(ctx0, attn, identity);
160 cb(lhs, "dnet_add_ch_lhs", il);
161
162 attn = ggml_neg(ctx0, attn);
163 cb(attn, "attn_pre_solve", il);
164
165 ggml_tensor * lin_solve = ggml_solve_tri(ctx0, lhs, attn, true, true, false);
166 attn = ggml_add(ctx0, lin_solve, identity);
167 cb(attn, "dnet_add_ch_attn_solved", il); // [CS, CS, n_chunks, H_k * n_seqs]
168
169 // [S_v, CS, n_chunks, H_v * n_seqs]
170 v = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, v_b)), attn);
171
172 // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs]
173 ggml_tensor * g_exp = ggml_exp(ctx0, g_cs);
174
175 k_b = ggml_cont(ctx0, ggml_transpose(ctx0, k_b));
176
177 // [CS, S_k, n_chunks, H_k * n_seqs]
178 ggml_tensor * kbg = ggml_mul(ctx0, k_b, g_exp);
179 cb(kbg, "k_beta_g_exp", il);
180
181 // [S_k, CS, n_chunks, H_k * n_seqs]
182 ggml_tensor * k_cd = ggml_mul_mat(ctx0, kbg, attn);
183 cb(k_cd, "k_cumdecay", il);
184
185 // [1, CS, n_chunks, H_k * n_seqs] KDA: [S_k, CS, n_chunks, H_k * n_seqs]
186 ggml_tensor * g_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, g_exp));
187 ggml_tensor * q_g_exp = ggml_mul(ctx0, q, g_exp_t);
188
189 // vectorized calculation of key_gdiff
190 // improved from the chunked version:
191 // g_last = torch.clamp(g_cum[:, :, -1], max=50.0).exp().unsqueeze(-1).unsqueeze(-1)
192 // g_diff = torch.clamp(g_cum[:, :, -1:] - g_cum, max=50.0).exp()
193 // key_gdiff = key * g_diff.unsqueeze(-1)
194 // kgdmulvnew = (key_gdiff).transpose(-1, -2) @ v_new
195 // last_recurrent_state = last_recurrent_state * g_last + kgdmulvnew
196
197 // get last element in g_cumsum along CS dimension (ne0)
198 // example: [[x, y, z, ..., last], ...] -> [[last], ...]
199 // [1, 1, n_chunks, H_v * n_seqs] KDA: [1, S_k, n_chunks, H_v * n_seqs]
200 ggml_tensor * g_last = ggml_view_4d(ctx0, g_cs, 1, g_cs->ne[1], g_cs->ne[2], g_cs->ne[3],
201 g_cs->nb[1],
202 g_cs->nb[2],
203 g_cs->nb[3],
204 ggml_row_size(g_cs->type, g_cs->ne[0] - 1));
205 cb(g_last, "g_last", il);
206
207 // TODO: remove this cont when CUDA supports non-cont unary ops
208 g_last = ggml_cont(ctx0, g_last);
209
210 // [1, 1, n_chunks, H_v * n_seqs] KDA: [S_k, 1, n_chunks, H_v * n_seqs]
211 ggml_tensor * g_last_exp_t = ggml_transpose(ctx0, ggml_exp(ctx0, g_last));
212 cb(g_last_exp_t, "g_last_exp_t", il);
213
214 // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs]
215 ggml_tensor * g_diff = ggml_neg(ctx0, ggml_sub(ctx0, g_cs, g_last));
216 cb(g_diff, "g_diff", il);
217
218 ggml_tensor * g_diff_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_exp(ctx0, g_diff)));
219
220 // [S_k, CS, n_chunks, H_v * n_seqs]
221 ggml_tensor * kg = ggml_mul(ctx0, k, g_diff_exp_t);
222 cb(kg, "key_gdiff", il);
223
224 // [CS, S_k, n_chunks, H_v * n_seqs]
225 ggml_tensor * kg_t = ggml_cont(ctx0, ggml_transpose(ctx0, kg));
226 cb(kg_t, "key_gdiff_t", il);
227
228 s = ggml_reshape_4d(ctx0, s, S_v, S_v, 1, H_v * n_seqs);
229 cb(s, "dnet_add_ch_state", il);
230
231 // [CS, S_v, n_chunks, H_v * n_seqs]
232 ggml_tensor * v_t = ggml_cont(ctx0, ggml_transpose(ctx0, v));
233
234 for (int64_t chunk = 0; chunk < n_chunks; chunk++) {
235 ggml_tensor * ch_k_cd = get_slice_2d(ctx0, k_cd, chunk); // [S_k, CS, 1, H_k * n_seqs]
236 ggml_tensor * ch_v_t = get_slice_2d(ctx0, v_t, chunk); // [ CS, S_v, 1, H_v * n_seqs]
237 ggml_tensor * ch_kq = get_slice_2d(ctx0, kq, chunk); // [ CS, CS, 1, H_k * n_seqs]
238 ggml_tensor * ch_q_g_exp = get_slice_2d(ctx0, q_g_exp, chunk); // [S_k, CS, 1, H_k * n_seqs]
239 ggml_tensor * ch_kg_t = get_slice_2d(ctx0, kg_t, chunk); // [ CS, S_k, 1, H_v * n_seqs]
240
241 // [CS, S_v, 1, H_v * n_seqs]
242 ggml_tensor * v_t_p = ggml_mul_mat(ctx0, ch_k_cd, s);
243 cb(v_t_p, "v_prime", il);
244
245 // [CS, S_v, 1, H_v * n_seqs]
246 ggml_tensor * v_t_new = ggml_sub(ctx0, ch_v_t, v_t_p);
247 cb(v_t_new, "v_t_new", il);
248
249 // [S_v, CS, 1, H_v * n_seqs]
250 ggml_tensor * v_attn = ggml_mul_mat(ctx0, v_t_new, ch_kq);
251 cb(v_attn, "v_attn", il);
252
253 // [S_v, CS, 1, H_v * n_seqs]
254 ggml_tensor * attn_inter = ggml_mul_mat(ctx0, s, ch_q_g_exp);
255 cb(attn_inter, "attn_inter", il);
256
257 // [S_v, CS, 1, H_v * n_seqs]
258 ggml_tensor * o_ch = ggml_add(ctx0, attn_inter, v_attn);
259 cb(o_ch, "dnet_add_ch_attn_out", il);
260
261 v = ggml_set_inplace(ctx0, v, o_ch, v->nb[1], v->nb[2], v->nb[3], chunk * v->nb[2]);
262
263 // kgdmulvnew = (key_gdiff).transpose(-1, -2) @ v_new
264 // TODO: head broadcast might not work here - probably will need a transpose
265 ggml_tensor * kgv = ggml_mul_mat(ctx0, ch_kg_t, v_t_new); // [S_k, S_v, 1, H_k * n_seqs]
266
267 // last_recurrent_state = last_recurrent_state * g_last + kgdmulvnew
268 ggml_tensor * ch_g_last_exp_t = get_slice_2d(ctx0, g_last_exp_t, chunk);
269
270 s = ggml_mul(ctx0, s, ch_g_last_exp_t);
271 s = ggml_add(ctx0, s, kgv);
272 cb(s, "dnet_add_ch_state", il);
273 }
274
275 // truncate padded tokens
276 ggml_tensor * o = ggml_view_4d(ctx0, v,
277 S_v, n_tokens, H_v, n_seqs,
278 ggml_row_size(v->type, S_v),
279 ggml_row_size(v->type, S_v * CS * n_chunks),
280 ggml_row_size(v->type, S_v * CS * n_chunks * H_v), 0);
281 o = ggml_permute (ctx0, o, 0, 2, 1, 3); // [S_v, H_v, n_tokens, n_seqs]
282 s = ggml_reshape_4d(ctx0, s, S_v, S_v, H_v, n_seqs);
283 cb(s, "output_state", il);
284
285 return {o, s};
286}
287
288std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net_autoregressive(
289 ggml_tensor * q,
290 ggml_tensor * k,
291 ggml_tensor * v,
292 ggml_tensor * g,
293 ggml_tensor * b, // beta
294 ggml_tensor * s, // state
295 int il) {
296 const int64_t S_k = q->ne[0];
297 const int64_t H_k = q->ne[1];
298 const int64_t n_tokens = q->ne[2];
299 const int64_t n_seqs = q->ne[3];
300
301 const int64_t S_v = v->ne[0];
302 const int64_t H_v = v->ne[1];
303
304 GGML_ASSERT(n_tokens == 1)if (!(n_tokens == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 304, "GGML_ASSERT(%s) failed", "n_tokens == 1");
305
306 GGML_ASSERT(S_k == S_v)if (!(S_k == S_v)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 306, "GGML_ASSERT(%s) failed", "S_k == S_v");
307 GGML_ASSERT(H_v % H_k == 0)if (!(H_v % H_k == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 307, "GGML_ASSERT(%s) failed", "H_v % H_k == 0");
308
309 GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs)if (!(q->ne[0] == S_k && q->ne[1] == H_k &&
q->ne[2] == n_tokens && q->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 309, "GGML_ASSERT(%s) failed", "q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs"
);
310 GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs)if (!(k->ne[0] == S_k && k->ne[1] == H_k &&
k->ne[2] == n_tokens && k->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 310, "GGML_ASSERT(%s) failed", "k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs"
);
311 GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs)if (!(v->ne[0] == S_v && v->ne[1] == H_v &&
v->ne[2] == n_tokens && v->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 311, "GGML_ASSERT(%s) failed", "v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs"
);
312
313 GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v)if (!(g->ne[0] == 1 || g->ne[0] == S_v)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 313, "GGML_ASSERT(%s) failed", "g->ne[0] == 1 || g->ne[0] == S_v"
);
314 GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs)if (!(g->ne[1] == H_v && g->ne[2] == n_tokens &&
g->ne[3] == n_seqs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 314, "GGML_ASSERT(%s) failed", "g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs"
);
315 GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs)if (!(b->ne[0] == 1 && b->ne[1] == H_v &&
b->ne[2] == n_tokens && b->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 315, "GGML_ASSERT(%s) failed", "b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs"
);
316 GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs)if (!(s->ne[0] == S_v && s->ne[1] == S_v &&
s->ne[2] == H_v && s->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 316, "GGML_ASSERT(%s) failed", "s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs"
);
317
318 const float scale = 1.0f / sqrtf(S_k);
319
320 q = ggml_scale(ctx0, q, scale);
321
322 q = ggml_permute(ctx0, q, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]
323 k = ggml_permute(ctx0, k, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]
324 v = ggml_permute(ctx0, v, 0, 2, 1, 3); // [S_v, n_tokens, H_v, n_seqs]
325
326 cb(q, "q_in", il);
327 cb(k, "k_in", il);
328 cb(v, "v_in", il);
329 cb(b, "b_in", il);
330 cb(g, "g_in", il);
331
332 // GDA: [1, 1, H_v, n_seqs]
333 // KDA: [1, S_k, H_v, n_seqs]
334 g = ggml_reshape_4d(ctx0, g, 1, g->ne[0], H_v, n_seqs);
335 b = ggml_reshape_4d(ctx0, b, 1, 1, H_v, n_seqs);
336
337 // [S_v, S_v, H_v, n_seqs]
338 g = ggml_exp(ctx0, g);
339 s = ggml_mul(ctx0, s, g);
340
341 // [1, S_v, H_v, n_seqs]
342 ggml_tensor * sk;
343 sk = ggml_mul (ctx0, s, k);
344 sk = ggml_sum_rows(ctx0, sk);
345
346 // [S_v, 1, H_v, n_seqs]
347 ggml_tensor * d;
348 d = ggml_sub(ctx0, v, ggml_transpose(ctx0, sk));
349 d = ggml_mul(ctx0, d, b);
350
351 // [1, S_v, H_v, n_seqs]
352 ggml_tensor * d_t;
353 d_t = ggml_transpose(ctx0, d);
354
355 // [S_v, S_v, H_v, n_seqs]
356 ggml_tensor * kd;
357 k = ggml_repeat(ctx0, k, s);
358 kd = ggml_mul (ctx0, k, d_t);
359
360 s = ggml_add(ctx0, s, kd);
361
362 cb(s, "dnet_add_ar_state", il);
363
364 ggml_tensor * s_q = ggml_mul (ctx0, s, q);
365 ggml_tensor * o = ggml_sum_rows(ctx0, s_q);
366
367 o = ggml_permute (ctx0, o, 2, 0, 1, 3); // [S_v, H_v, n_tokens, n_seqs]
368
369 return {o, s};
370}
371
372std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net_fused(
373 ggml_tensor * q,
374 ggml_tensor * k,
375 ggml_tensor * v,
376 ggml_tensor * g,
377 ggml_tensor * b,
378 ggml_tensor * s,
379 int il) {
380 const int64_t S_k = q->ne[0];
381 const int64_t H_k = q->ne[1];
382 const int64_t n_tokens = q->ne[2];
383 const int64_t n_seqs = q->ne[3];
384
385 const int64_t S_v = v->ne[0];
386 const int64_t H_v = v->ne[1];
387
388 GGML_ASSERT(S_k == S_v)if (!(S_k == S_v)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 388, "GGML_ASSERT(%s) failed", "S_k == S_v");
389 GGML_ASSERT(H_v % H_k == 0)if (!(H_v % H_k == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 389, "GGML_ASSERT(%s) failed", "H_v % H_k == 0");
390
391 GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs)if (!(q->ne[0] == S_k && q->ne[1] == H_k &&
q->ne[2] == n_tokens && q->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 391, "GGML_ASSERT(%s) failed", "q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs"
);
392 GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs)if (!(k->ne[0] == S_k && k->ne[1] == H_k &&
k->ne[2] == n_tokens && k->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 392, "GGML_ASSERT(%s) failed", "k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs"
);
393 GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs)if (!(v->ne[0] == S_v && v->ne[1] == H_v &&
v->ne[2] == n_tokens && v->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 393, "GGML_ASSERT(%s) failed", "v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs"
);
394
395 GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v)if (!(g->ne[0] == 1 || g->ne[0] == S_v)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 395, "GGML_ASSERT(%s) failed", "g->ne[0] == 1 || g->ne[0] == S_v"
);
396 GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs)if (!(g->ne[1] == H_v && g->ne[2] == n_tokens &&
g->ne[3] == n_seqs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 396, "GGML_ASSERT(%s) failed", "g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs"
);
397 GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs)if (!(b->ne[0] == 1 && b->ne[1] == H_v &&
b->ne[2] == n_tokens && b->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 397, "GGML_ASSERT(%s) failed", "b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs"
);
398 GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs)if (!(s->ne[0] == S_v && s->ne[1] == S_v &&
s->ne[2] == H_v && s->ne[3] == n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/models/delta-net-base.cpp"
, 398, "GGML_ASSERT(%s) failed", "s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs"
);
399
400 // K=1: output carries the final state only. state s is 4D [S_v, S_v, H_v, n_seqs].
401 ggml_tensor * result = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, /*K=*/1);
402 if (n_tokens == 1) {
403 cb(result, LLAMA_TENSOR_NAME_FGDN_AR"__fgdn_ar__", il);
404 } else {
405 cb(result, LLAMA_TENSOR_NAME_FGDN_CH"__fgdn_ch__", il);
406 }
407
408 ggml_tensor * output = ggml_view_4d(ctx0, result,
409 S_v, H_v, n_tokens, n_seqs,
410 ggml_row_size(result->type, S_v),
411 ggml_row_size(result->type, S_v * H_v),
412 ggml_row_size(result->type, S_v * H_v * n_tokens), 0);
413
414 ggml_tensor * new_state = ggml_view_4d(ctx0, result,
415 S_v, S_v, H_v, n_seqs,
416 ggml_row_size(result->type, S_v),
417 ggml_row_size(result->type, S_v * S_v),
418 ggml_row_size(result->type, S_v * S_v * H_v),
419 ggml_row_size(result->type, S_v * H_v * n_tokens * n_seqs));
420
421 return {output, new_state};
422}
423
424std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net(
425 ggml_tensor * q,
426 ggml_tensor * k,
427 ggml_tensor * v,
428 ggml_tensor * g,
429 ggml_tensor * b,
430 ggml_tensor * s,
431 int il) {
432 const int64_t n_seq_tokens = q->ne[2];
433
434 if (n_seq_tokens == 1) {
435 if (cparams.fused_gdn_ar) {
436 return build_delta_net_fused(q, k, v, g, b, s, il);
437 }
438 return build_delta_net_autoregressive(q, k, v, g, b, s, il);
439 }
440
441 if (cparams.fused_gdn_ch) {
442 return build_delta_net_fused(q, k, v, g, b, s, il);
443 }
444
445 return build_delta_net_chunking(q, k, v, g, b, s, il);
446}
447
448ggml_tensor * llm_build_delta_net_base::build_conv_state(
449 llm_graph_input_rs * inp,
450 ggml_tensor * conv_states_all,
451 ggml_tensor * qkv_mixed,
452 int64_t conv_kernel_size,
453 int64_t conv_channels,
454 int il) {
455 const auto * mctx_cur = inp->mctx;
456
457 const auto kv_head = mctx_cur->get_head();
458 const auto mem_size = mctx_cur->get_size();
459
460 const int64_t n_seqs = ubatch.n_seqs;
461
462 ggml_tensor * conv_states = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);
463 cb(conv_states, "conv_states", il);
464
465 conv_states = ggml_reshape_3d(ctx0, conv_states, conv_kernel_size - 1, conv_channels, n_seqs);
466 cb(conv_states, "conv_states_reshaped", il);
467
468 qkv_mixed = ggml_transpose(ctx0, qkv_mixed);
469 cb(qkv_mixed, "qkv_mixed_transposed", il);
470
471 ggml_tensor * conv_input = ggml_concat(ctx0, conv_states, qkv_mixed, 0);
472 cb(conv_input, "conv_input", il);
473
474 const int64_t row_count = (conv_kernel_size - 1) * conv_channels;
475
476 const size_t row_size = ggml_row_size(conv_states_all->type, row_count);
477
478 if (cparams.n_rs_seq == 0) {
479 const int64_t s_idx = conv_input->ne[0] - conv_states->ne[0];
480 const int64_t s_slot = 0;
481
482 ggml_tensor * conv_state_last =
483 ggml_view_3d(ctx0, conv_input,
484 conv_kernel_size - 1, conv_channels, n_seqs,
485 conv_input->nb[1], conv_input->nb[2],
486 ggml_row_size(conv_input->type, s_idx));
487 cb(conv_state_last, "conv_state_last", il);
488
489 ggml_tensor * conv_state_update =
490 ggml_view_2d(ctx0, conv_states_all,
491 row_count, n_seqs, conv_states_all->nb[1],
492 (s_slot * mem_size + kv_head) * row_size);
493 cb(conv_state_update, "conv_state_update", il);
494
495 ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_state_last, conv_state_update));
496 } else {
497 // [TAG_RECURRENT_ROLLBACK_SPLITS]
498 // TODO: this logic incorrectly assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are
499 // inside the same ubatch. currently with `split_equal()` this is not correct
500
501 const int64_t K = (int64_t) cparams.n_rs_seq + 1;
502
503 for (int64_t t = 1; t <= K; ++t) {
504 const int64_t s_idx = std::max<int64_t>(0, conv_input->ne[0] - conv_states->ne[0] - K + t);
505 const int64_t s_slot = K - t;
506
507 ggml_tensor * conv_state_last =
508 ggml_view_3d(ctx0, conv_input,
509 conv_kernel_size - 1, conv_channels, n_seqs,
510 conv_input->nb[1], conv_input->nb[2],
511 ggml_row_size(conv_input->type, s_idx));
512
513 ggml_tensor * conv_state_update =
514 ggml_view_2d(ctx0,
515 conv_states_all, row_count, n_seqs,
516 conv_states_all->nb[1],
517 (s_slot * mem_size + kv_head) * row_size);
518
519 ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_state_last, conv_state_update));
520 }
521 }
522
523 return conv_input;
524}
525
526ggml_tensor * llm_build_delta_net_base::build_recurrent_attn(
527 llm_graph_input_rs * inp,
528 ggml_tensor * ssm_states_all,
529 ggml_tensor * q,
530 ggml_tensor * k,
531 ggml_tensor * v,
532 ggml_tensor * g,
533 ggml_tensor * b,
534 ggml_tensor * s,
535 int il) {
536 const auto * mctx_cur = inp->mctx;
537 const auto kv_head = mctx_cur->get_head();
538 const uint32_t mem_size = mctx_cur->get_size();
539
540 const int64_t S_v = s->ne[0];
541 const int64_t H_v = s->ne[2];
542 const int64_t n_seqs = s->ne[3];
543 const int64_t n_seq_tokens = q->ne[2];
544
545 const bool keep = cparams.n_rs_seq > 0;
546
547 if (!keep) {
548 auto attn_out = build_delta_net(q, k, v, g, b, s, il);
549 ggml_tensor * output = attn_out.first;
550 ggml_tensor * new_state = attn_out.second;
551 cb(output, "attn_output", il);
552 cb(new_state, "new_state", il);
553
554 ggml_build_forward_expand(gf,
555 ggml_cpy(ctx0, new_state,
556 ggml_view_2d(ctx0, ssm_states_all, hparams.n_embd_s(), n_seqs, ssm_states_all->nb[1],
557 kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));
558
559 return output;
560 }
561
562 const int64_t D = S_v * S_v * H_v;
563 const int64_t K = cparams.n_rs_seq + 1;
564
565 // state s is 4D [S_v, S_v, H_v, n_seqs]; K snapshot slots are written into the output.
566 ggml_tensor * gdn_out = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, K);
567 if (n_seq_tokens > 1) {
568 cb(gdn_out, LLAMA_TENSOR_NAME_FGDN_CH"__fgdn_ch__", il);
569 } else {
570 cb(gdn_out, LLAMA_TENSOR_NAME_FGDN_AR"__fgdn_ar__", il);
571 }
572
573 const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs;
574 const int64_t state_size_per_snap = S_v * S_v * H_v * n_seqs;
575
576 ggml_tensor * output = ggml_view_4d(ctx0, gdn_out,
577 S_v, H_v, n_seq_tokens, n_seqs,
578 ggml_row_size(gdn_out->type, S_v),
579 ggml_row_size(gdn_out->type, S_v * H_v),
580 ggml_row_size(gdn_out->type, S_v * H_v * n_seq_tokens),
581 0);
582 cb(output, "attn_output", il);
583
584 const size_t row_size = hparams.n_embd_s() * ggml_element_size(ssm_states_all);
585
586 // op writes the last min(n_seq_tokens, K) snapshots; trailing slots are left unwritten
587 const int64_t n_written = std::min<int64_t>(n_seq_tokens, K);
588
589 // write the produced snapshots into the recurrent cache (snapshot slot i -> rollback group i)
590 ggml_tensor * src = ggml_view_3d(ctx0, gdn_out,
591 D, n_seqs, n_written,
592 ggml_row_size(gdn_out->type, D),
593 ggml_row_size(gdn_out->type, state_size_per_snap),
594 ggml_row_size(gdn_out->type, attn_score_elems));
595
596 ggml_tensor * dst = ggml_view_3d(ctx0, ssm_states_all,
597 D, n_seqs, n_written,
598 ssm_states_all->nb[1],
599 (size_t) mem_size * row_size,
600 (size_t) kv_head * row_size);
601
602 ggml_build_forward_expand(gf, ggml_cpy(ctx0, src, dst));
603
604 return output;
605}