Bug Summary

File:root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp
Warning:line 3806, column 45
Division by zero

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O2 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ops.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +avx -target-feature +avx2 -target-feature +bmi2 -target-feature +f16c -target-feature +fma -target-feature +sse4.2 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -resource-dir /usr/lib/llvm-23/lib/clang/23 -include /root/firefox-clang/config/gcc_hidden.h -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/stl_wrappers -D _GLIBCXX_ASSERTIONS=1 -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/system_wrappers -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG=1 -D _GNU_SOURCE=1 -D GGML_USE_CPU=1 -D GGML_VERSION="GGML_VERSION" -D GGML_COMMIT="GGML_COMMIT" -D GGML_SHARED=1 -D LLAMA_SHARED=1 -D GGML_BUILD=1 -D LLAMA_BUILD=1 -D GGML_BACKEND_SHARED=1 -D GGML_BACKEND_BUILD=1 -D MOZ_HAS_MOZGLUE -I /root/firefox-clang/third_party/llama.cpp -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -I /root/firefox-clang/third_party/llama.cpp/ggml -I /root/firefox-clang/third_party/llama.cpp/ggml/include -I /root/firefox-clang/third_party/llama.cpp/ggml/src -I /root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu -I /root/firefox-clang/third_party/llama.cpp/include -I /root/firefox-clang/third_party/llama.cpp/src -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/c++/16 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/x86_64-linux-gnu/c++/16 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/c++/16/backward -internal-isystem /usr/lib/llvm-23/lib/clang/23/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=pessimizing-move -Wno-error=large-by-value-copy=128 -Wno-error=implicit-int-float-conversion -Wno-error=thread-safety-analysis -Wno-error=tautological-type-limit-compare -Wno-invalid-offsetof -Wno-range-loop-analysis -Wno-deprecated-anon-enum-enum-conversion -Wno-deprecated-enum-enum-conversion -Wno-inline-new-delete -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-vla-cxx-extension -Wno-unknown-warning-option -Wno-character-conversion -Wno-sign-compare -Wno-unused-function -Wno-tautological-unsigned-enum-zero-compare -Wno-implicit-fallthrough -Wno-unreachable-code -std=gnu++20 -fdeprecated-macro -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fno-rtti -fgnuc-version=4.2.1 -fno-implicit-modules -fskip-odr-check-in-gmf -fno-sized-deallocation -fno-aligned-allocation -fdiagnostics-absolute-paths -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -mllvm -dwarf-linkage-names=Abstract -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-09-01-224014-2642839-1 -x c++ /root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp
1#include "ops.h"
2
3#include "ggml-cpu.h"
4#include "ggml-impl.h"
5#include "binary-ops.h"
6#include "simd-gemm.h"
7#include "ggml.h"
8#include "unary-ops.h"
9#include "vec.h"
10
11#include <algorithm>
12#include <cfloat>
13#include <cmath>
14
15// ggml_compute_forward_dup
16
17static void ggml_compute_forward_dup_same_cont(
18 const ggml_compute_params * params,
19 ggml_tensor * dst) {
20
21 const ggml_tensor * src0 = dst->src[0];
22
23 GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0))if (!(ggml_nelements(dst) == ggml_nelements(src0))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 23, "GGML_ASSERT(%s) failed", "ggml_nelements(dst) == ggml_nelements(src0)"
);
24 GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0))if (!(ggml_is_contiguous(dst) && ggml_is_contiguous(src0
))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 24, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dst) && ggml_is_contiguous(src0)"
);
25 GGML_ASSERT(src0->type == dst->type)if (!(src0->type == dst->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 25, "GGML_ASSERT(%s) failed", "src0->type == dst->type"
);
26
27 const size_t nb0 = ggml_type_size(src0->type);
28
29 const int ith = params->ith; // thread index
30 const int nth = params->nth; // number of threads
31
32 // parallelize by blocks
33 const int nk = ggml_nelements(src0)/ggml_blck_size(src0->type);
34 const int dr = (nk + nth - 1) / nth;
35 const int k0 = dr * ith;
36 const int k1 = MIN(k0 + dr, nk)((k0 + dr) < (nk) ? (k0 + dr) : (nk));
37
38 if (k0 < k1) {
39 memcpy(
40 ((char *) dst->data + k0*nb0),
41 ((char *) src0->data + k0*nb0),
42 (k1 - k0) * nb0);
43 }
44}
45
46template<typename src_t, typename dst_t>
47static void ggml_compute_forward_dup_flt(
48 const ggml_compute_params * params,
49 ggml_tensor * dst) {
50
51 const ggml_tensor * src0 = dst->src[0];
52
53 GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0))if (!(ggml_nelements(dst) == ggml_nelements(src0))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 53, "GGML_ASSERT(%s) failed", "ggml_nelements(dst) == ggml_nelements(src0)"
);
54 GGML_ASSERT(!ggml_is_quantized(src0->type) && !ggml_is_quantized(dst->type))if (!(!ggml_is_quantized(src0->type) && !ggml_is_quantized
(dst->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 54, "GGML_ASSERT(%s) failed", "!ggml_is_quantized(src0->type) && !ggml_is_quantized(dst->type)"
);
55
56 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
57
58 const int ith = params->ith; // thread index
59 const int nth = params->nth; // number of threads
60
61 // parallelize by rows
62 const int nr = ne01;
63 // number of rows per thread
64 const int dr = (nr + nth - 1) / nth;
65 // row range for this thread
66 const int ir0 = dr * ith;
67 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
68
69 // case: type & row size equal
70 if (src0->type == dst->type &&
71 ne00 == ne0 &&
72 nb00 == ggml_type_size(src0->type) && nb0 == ggml_type_size(dst->type)) {
73 // copy by rows
74 const size_t rs = ne00*nb00;
75 for (int64_t i03 = 0; i03 < ne03; i03++) {
76 for (int64_t i02 = 0; i02 < ne02; i02++) {
77 for (int64_t i01 = ir0; i01 < ir1; i01++) {
78 memcpy(
79 ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),
80 ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),
81 rs);
82 }
83 }
84 }
85 return;
86 }
87
88 // case: dst tensor is contiguous
89 if (ggml_is_contiguous(dst)) {
90 if (nb00 == sizeof(src_t)) {
91 if constexpr (std::is_same_v<dst_t, src_t>) {
92 // same type
93 size_t id = 0;
94 const size_t rs = ne00 * nb00;
95 char * dst_ptr = (char *) dst->data;
96
97 for (int i03 = 0; i03 < ne03; i03++) {
98 for (int i02 = 0; i02 < ne02; i02++) {
99 id += rs * ir0;
100 for (int i01 = ir0; i01 < ir1; i01++) {
101 const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
102 memcpy(dst_ptr + id, src0_ptr, rs);
103 id += rs;
104 }
105 id += rs * (ne01 - ir1);
106 }
107 }
108 } else {
109 // casting between non-quantized types
110 size_t id = 0;
111 dst_t * dst_ptr = (dst_t *) dst->data;
112
113 for (int i03 = 0; i03 < ne03; i03++) {
114 for (int i02 = 0; i02 < ne02; i02++) {
115 id += ne00 * ir0;
116 for (int i01 = ir0; i01 < ir1; i01++) {
117 const src_t * src0_ptr = (src_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
118 for (int i00 = 0; i00 < ne00; i00++) {
119 float tmp = type_conversion_table<src_t>::to_f32(src0_ptr[i00]);
120 dst_ptr[id] = type_conversion_table<dst_t>::from_f32(tmp);
121 id++;
122 }
123 }
124 id += ne00 * (ne01 - ir1);
125 }
126 }
127 }
128 } else {
129 //printf("%s: this is not optimal - fix me\n", __func__);
130
131 size_t id = 0;
132 dst_t * dst_ptr = (dst_t *) dst->data;
133
134 for (int i03 = 0; i03 < ne03; i03++) {
135 for (int i02 = 0; i02 < ne02; i02++) {
136 id += ne00 * ir0;
137 for (int i01 = ir0; i01 < ir1; i01++) {
138 for (int i00 = 0; i00 < ne00; i00++) {
139 const src_t * src0_ptr = (src_t *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
140
141 float tmp = type_conversion_table<src_t>::to_f32(*src0_ptr);
142 dst_ptr[id] = type_conversion_table<dst_t>::from_f32(tmp);
143 id++;
144 }
145 }
146 id += ne00 * (ne01 - ir1);
147 }
148 }
149 }
150 return;
151 }
152
153 // dst counters
154 int64_t i10 = 0;
155 int64_t i11 = 0;
156 int64_t i12 = 0;
157 int64_t i13 = 0;
158
159 if constexpr (std::is_same_v<dst_t, src_t>) {
160 for (int64_t i03 = 0; i03 < ne03; i03++) {
161 for (int64_t i02 = 0; i02 < ne02; i02++) {
162 i10 += ne00 * ir0;
163 while (i10 >= ne0) {
164 i10 -= ne0;
165 if (++i11 == ne1) {
166 i11 = 0;
167 if (++i12 == ne2) {
168 i12 = 0;
169 if (++i13 == ne3) {
170 i13 = 0;
171 }
172 }
173 }
174 }
175 for (int64_t i01 = ir0; i01 < ir1; i01++) {
176 for (int64_t i00 = 0; i00 < ne00; i00++) {
177 const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
178 char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
179
180 memcpy(dst_ptr, src0_ptr, sizeof(dst_t));
181
182 if (++i10 == ne00) {
183 i10 = 0;
184 if (++i11 == ne01) {
185 i11 = 0;
186 if (++i12 == ne02) {
187 i12 = 0;
188 if (++i13 == ne03) {
189 i13 = 0;
190 }
191 }
192 }
193 }
194 }
195 }
196 i10 += ne00 * (ne01 - ir1);
197 while (i10 >= ne0) {
198 i10 -= ne0;
199 if (++i11 == ne1) {
200 i11 = 0;
201 if (++i12 == ne2) {
202 i12 = 0;
203 if (++i13 == ne3) {
204 i13 = 0;
205 }
206 }
207 }
208 }
209 }
210 }
211
212 } else {
213 for (int64_t i03 = 0; i03 < ne03; i03++) {
214 for (int64_t i02 = 0; i02 < ne02; i02++) {
215 i10 += ne00 * ir0;
216 while (i10 >= ne0) {
217 i10 -= ne0;
218 if (++i11 == ne1) {
219 i11 = 0;
220 if (++i12 == ne2) {
221 i12 = 0;
222 if (++i13 == ne3) {
223 i13 = 0;
224 }
225 }
226 }
227 }
228 for (int64_t i01 = ir0; i01 < ir1; i01++) {
229 for (int64_t i00 = 0; i00 < ne00; i00++) {
230 const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
231 char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
232
233 float tmp = type_conversion_table<src_t>::to_f32(*(const src_t *) src0_ptr);
234 *(dst_t *) dst_ptr = type_conversion_table<dst_t>::from_f32(tmp);
235
236 if (++i10 == ne0) {
237 i10 = 0;
238 if (++i11 == ne1) {
239 i11 = 0;
240 if (++i12 == ne2) {
241 i12 = 0;
242 if (++i13 == ne3) {
243 i13 = 0;
244 }
245 }
246 }
247 }
248 }
249 }
250 i10 += ne00 * (ne01 - ir1);
251 while (i10 >= ne0) {
252 i10 -= ne0;
253 if (++i11 == ne1) {
254 i11 = 0;
255 if (++i12 == ne2) {
256 i12 = 0;
257 if (++i13 == ne3) {
258 i13 = 0;
259 }
260 }
261 }
262 }
263 }
264 }
265 }
266}
267
268
269template<typename src_t>
270static void ggml_compute_forward_dup_to_q(
271 const ggml_compute_params * params,
272 ggml_tensor * dst) {
273
274 const ggml_tensor * src0 = dst->src[0];
275
276 GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0))if (!(ggml_nelements(dst) == ggml_nelements(src0))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 276, "GGML_ASSERT(%s) failed", "ggml_nelements(dst) == ggml_nelements(src0)"
);
277 GGML_ASSERT(!ggml_is_quantized(src0->type))if (!(!ggml_is_quantized(src0->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 277, "GGML_ASSERT(%s) failed", "!ggml_is_quantized(src0->type)"
);
278
279 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
280
281 const int ith = params->ith; // thread index
282 const int nth = params->nth; // number of threads
283
284 // parallelize by rows
285 const int nr = ne01;
286 // number of rows per thread
287 const int dr = (nr + nth - 1) / nth;
288 // row range for this thread
289 const int ir0 = dr * ith;
290 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
291
292 if (ggml_is_contiguous(dst) &&
293 nb00 == sizeof(src_t) &&
294 ggml_get_type_traits_cpu(dst->type)->from_float) {
295 // casting non-quantized types --> intermediate f32 --> quantized
296 ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(dst->type)->from_float;
297 float * src0_f32 = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;
298
299 size_t id = 0;
300 size_t rs = nb0 * (ne00 / ggml_blck_size(dst->type));
301 char * dst_ptr = (char *) dst->data;
302
303 for (int i03 = 0; i03 < ne03; i03++) {
304 for (int i02 = 0; i02 < ne02; i02++) {
305 id += rs * ir0;
306 for (int i01 = ir0; i01 < ir1; i01++) {
307 const src_t * src0_ptr = (src_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
308
309 for (int i00 = 0; i00 < ne00; i00++) {
310 src0_f32[i00] = type_conversion_table<src_t>::to_f32(src0_ptr[i00]);
311 }
312
313 quantize_row_q(src0_f32, dst_ptr + id, ne00);
314 id += rs;
315 }
316 id += rs * (ne01 - ir1);
317 }
318 }
319 } else {
320 // printf("%s %s\n", ggml_type_name(src0->type), ggml_type_name(dst->type));
321 GGML_ABORT("not implemented")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 321, "not implemented");
322 }
323}
324
325// A simplified version of ggml_compute_forward_dup that doesn't do float upcasting, and just plain old memcpy.
326static void ggml_compute_forward_dup_bytes(
327 const ggml_compute_params * params,
328 ggml_tensor * dst) {
329 const ggml_tensor * src0 = dst->src[0];
330
331 GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0))if (!(ggml_nelements(dst) == ggml_nelements(src0))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 331, "GGML_ASSERT(%s) failed", "ggml_nelements(dst) == ggml_nelements(src0)"
);
332 GGML_ASSERT(src0->type == dst->type)if (!(src0->type == dst->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 332, "GGML_ASSERT(%s) failed", "src0->type == dst->type"
);
333
334 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);;
335
336 if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {
337 ggml_compute_forward_dup_same_cont(params, dst);
338 return;
339 }
340
341 const size_t type_size = ggml_type_size(src0->type);
342
343 const int ith = params->ith; // thread index
344 const int nth = params->nth; // number of threads
345
346 // parallelize by rows
347 const int nr = ne01;
348 // number of rows per thread
349 const int dr = (nr + nth - 1) / nth;
350 // row range for this thread
351 const int ir0 = dr * ith;
352 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
353
354 if (src0->type == dst->type &&
355 ggml_are_same_shape(src0, dst) &&
356 nb00 == type_size && nb0 == type_size) {
357 // copy by rows
358 const size_t rs = ggml_row_size(src0->type, ne00);
359 for (int64_t i03 = 0; i03 < ne03; i03++) {
360 for (int64_t i02 = 0; i02 < ne02; i02++) {
361 for (int64_t i01 = ir0; i01 < ir1; i01++) {
362 memcpy(
363 ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),
364 ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),
365 rs);
366 }
367 }
368 }
369 return;
370 }
371
372 if (ggml_is_contiguous(dst)) {
373 size_t id = 0;
374 char * dst_ptr = (char *) dst->data;
375 const size_t rs = ne00 * type_size;
376
377 if (nb00 == type_size) {
378 // src0 is contiguous on first dimension, copy by rows
379 for (int64_t i03 = 0; i03 < ne03; i03++) {
380 for (int64_t i02 = 0; i02 < ne02; i02++) {
381 id += rs * ir0;
382 for (int64_t i01 = ir0; i01 < ir1; i01++) {
383 const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
384 memcpy(dst_ptr + id, src0_ptr, rs);
385 id += rs;
386 }
387 id += rs * (ne01 - ir1);
388 }
389 }
390 } else {
391 //printf("%s: this is not optimal - fix me\n", __func__);
392
393 for (int64_t i03 = 0; i03 < ne03; i03++) {
394 for (int64_t i02 = 0; i02 < ne02; i02++) {
395 id += rs * ir0;
396 for (int64_t i01 = ir0; i01 < ir1; i01++) {
397 for (int64_t i00 = 0; i00 < ne00; i00++) {
398 const char * src0_ptr = (char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03;
399 memcpy(dst_ptr + id, src0_ptr, type_size);
400
401 id += type_size;
402 }
403 }
404 id += rs * (ne01 - ir1);
405 }
406 }
407 }
408
409 return;
410 }
411
412 // dst counters
413 int64_t k10 = 0;
414 int64_t i11 = 0;
415 int64_t i12 = 0;
416 int64_t i13 = 0;
417
418 // number of blocks in a row
419 const int64_t nk00 = ne00 / ggml_blck_size(src0->type);
420 const int64_t nk0 = ne0 / ggml_blck_size(dst->type);
421
422 for (int64_t i03 = 0; i03 < ne03; i03++) {
423 for (int64_t i02 = 0; i02 < ne02; i02++) {
424 k10 += nk00 * ir0;
425 while (k10 >= nk0) {
426 k10 -= nk0;
427 if (++i11 == ne1) {
428 i11 = 0;
429 if (++i12 == ne2) {
430 i12 = 0;
431 if (++i13 == ne3) {
432 i13 = 0;
433 }
434 }
435 }
436 }
437 for (int64_t i01 = ir0; i01 < ir1; i01++) {
438 for (int64_t k00 = 0; k00 < nk00; k00++) {
439 const char * src0_ptr = ((char *) src0->data + k00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
440 char * dst_ptr = ((char *) dst->data + k10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
441
442 memcpy(dst_ptr, src0_ptr, type_size);
443
444 if (++k10 == nk0) {
445 k10 = 0;
446 if (++i11 == ne1) {
447 i11 = 0;
448 if (++i12 == ne2) {
449 i12 = 0;
450 if (++i13 == ne3) {
451 i13 = 0;
452 }
453 }
454 }
455 }
456 }
457 }
458 k10 += nk00 * (ne01 - ir1);
459 while (k10 >= nk0) {
460 k10 -= nk0;
461 if (++i11 == ne1) {
462 i11 = 0;
463 if (++i12 == ne2) {
464 i12 = 0;
465 if (++i13 == ne3) {
466 i13 = 0;
467 }
468 }
469 }
470 }
471 }
472 }
473}
474
475static void ggml_compute_forward_dup_from_q(
476 const ggml_compute_params * params,
477 ggml_tensor * dst) {
478
479 const ggml_tensor * src0 = dst->src[0];
480 const ggml_tensor * src1 = dst->src[1];
481
482 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
483
484 const ggml_type type = src0->type;
485 ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;
486
487 size_t qk = ggml_blck_size(type);
488 const int64_t nr = ggml_nelements(src1) / qk;
489
490 // destination must be contiguous in the first dimension
491 GGML_ASSERT(nb10 == ggml_type_size(dst->type))if (!(nb10 == ggml_type_size(dst->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 491, "GGML_ASSERT(%s) failed", "nb10 == ggml_type_size(dst->type)"
);
492 // must either have first dimension large enough to hold a row, or fully contiguous
493 GGML_ASSERT((ne10 % qk) == 0 || ggml_is_contiguous(dst))if (!((ne10 % qk) == 0 || ggml_is_contiguous(dst))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 493, "GGML_ASSERT(%s) failed", "(ne10 % qk) == 0 || ggml_is_contiguous(dst)"
);
494
495 const int ith = params->ith;
496 const int nth = params->nth;
497
498 const int dr = (nr + nth - 1)/nth;
499
500 // row range for this thread
501 const int ir0 = dr*ith;
502 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
503
504 for (int64_t ir = ir0; ir < ir1; ++ir) {
505
506 uint32_t i = ir * qk;
507
508 const int64_t i03 = i/(ne00 * ne01 * ne02);
509 const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);
510 const int64_t i01 = (i - i03*ne00*ne01*ne02 - i02*ne01*ne00) / ne00;
511 const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;
512 const int64_t x_offset = (i00/qk)*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;
513
514 const int64_t i13 = i/(ne10 * ne11 * ne12);
515 const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);
516 const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;
517 const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;
518 const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13;
519
520 dequantize_row_q(
521 (const void *) ((char *) src0->data + x_offset),
522 (float *) ((char *) dst->data + dst_offset), qk);
523 }
524}
525
526void ggml_compute_forward_dup(
527 const ggml_compute_params * params,
528 ggml_tensor * dst) {
529
530 const ggml_tensor * src0 = dst->src[0];
531
532 if (src0->type == dst->type) {
533 ggml_compute_forward_dup_bytes(params, dst);
534 return;
535 }
536
537 switch (src0->type) {
538 case GGML_TYPE_F16:
539 {
540 /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_fp16_t>(params, dst);
541 else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_bf16_t>(params, dst);
542 else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_fp16_t, float >(params, dst);
543 else ggml_compute_forward_dup_to_q<ggml_fp16_t>(params, dst);
544 } break;
545 case GGML_TYPE_BF16:
546 {
547 /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp16_t>(params, dst);
548 else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_bf16_t>(params, dst);
549 else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_bf16_t, float >(params, dst);
550 else ggml_compute_forward_dup_to_q<ggml_bf16_t>(params, dst);
551 } break;
552 case GGML_TYPE_F32:
553 {
554 /**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<float, ggml_fp16_t>(params, dst);
555 else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<float, ggml_bf16_t>(params, dst);
556 else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<float, float >(params, dst);
557 else if (dst->type == GGML_TYPE_I32) ggml_compute_forward_dup_flt<float, int32_t >(params, dst);
558 else ggml_compute_forward_dup_to_q<float>(params, dst);
559 } break;
560 case GGML_TYPE_I32:
561 {
562 if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<int32_t, float>(params, dst);
563 else GGML_ABORT("not implemented")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 563, "not implemented");
564 } break;
565 default:
566 {
567 if (ggml_is_quantized(src0->type) && dst->type == GGML_TYPE_F32) {
568 ggml_compute_forward_dup_from_q(params, dst);
569 break;
570 }
571 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 571, "fatal error");
572 }
573 }
574}
575
576// ggml_compute_forward_add
577
578static void ggml_compute_forward_add_q_f32(
579 const ggml_compute_params * params,
580 ggml_tensor * dst) {
581
582 const ggml_tensor * src0 = dst->src[0];
583 const ggml_tensor * src1 = dst->src[1];
584
585 GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, src1) && ggml_are_same_shape
(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 585, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)"
);
586
587 const int nr = ggml_nrows(src0);
588
589 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
590
591 const int ith = params->ith;
592 const int nth = params->nth;
593
594 const ggml_type type = src0->type;
595 const ggml_type dtype = dst->type;
596 ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;
597 ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(dtype)->from_float;
598
599 // we don't support permuted src0 or src1
600 GGML_ASSERT(nb00 == ggml_type_size(type))if (!(nb00 == ggml_type_size(type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 600, "GGML_ASSERT(%s) failed", "nb00 == ggml_type_size(type)"
);
601 GGML_ASSERT(nb10 == sizeof(float))if (!(nb10 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 601, "GGML_ASSERT(%s) failed", "nb10 == sizeof(float)");
602
603 // dst cannot be transposed or permuted
604 GGML_ASSERT(nb0 <= nb1)if (!(nb0 <= nb1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 604, "GGML_ASSERT(%s) failed", "nb0 <= nb1");
605 GGML_ASSERT(nb1 <= nb2)if (!(nb1 <= nb2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 605, "GGML_ASSERT(%s) failed", "nb1 <= nb2");
606 GGML_ASSERT(nb2 <= nb3)if (!(nb2 <= nb3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 606, "GGML_ASSERT(%s) failed", "nb2 <= nb3");
607
608 GGML_ASSERT(ggml_is_quantized(src0->type))if (!(ggml_is_quantized(src0->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 608, "GGML_ASSERT(%s) failed", "ggml_is_quantized(src0->type)"
);
609 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 609, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
610
611 // rows per thread
612 const int dr = (nr + nth - 1)/nth;
613
614 // row range for this thread
615 const int ir0 = dr*ith;
616 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
617
618 float * wdata = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;
619
620 for (int ir = ir0; ir < ir1; ++ir) {
621 // src0 indices
622 const int i03 = ir/(ne02*ne01);
623 const int i02 = (ir - i03*ne02*ne01)/ne01;
624 const int i01 = (ir - i03*ne02*ne01 - i02*ne01);
625
626 // src1 and dst are same shape as src0 => same indices
627 const int i13 = i03;
628 const int i12 = i02;
629 const int i11 = i01;
630
631 const int i3 = i03;
632 const int i2 = i02;
633 const int i1 = i01;
634
635 void * src0_row = (void *) ((char *) src0->data + (i01*nb01 + i02*nb02 + i03*nb03));
636 float * src1_row = (float *)((char *) src1->data + (i11*nb11 + i12*nb12 + i13*nb13));
637 void * dst_row = (void *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
638
639 assert(ne00 % 32 == 0)(static_cast <bool> (ne00 % 32 == 0) ? void (0) : __assert_fail
("ne00 % 32 == 0", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
640
641 // unquantize row from src0 to temp buffer
642 dequantize_row_q(src0_row, wdata, ne00);
643 // add src1
644 ggml_vec_acc_f32(ne00, wdata, src1_row);
645 // quantize row to dst
646 if (quantize_row_q != NULL__null) {
647 quantize_row_q(wdata, dst_row, ne00);
648 } else {
649 memcpy(dst_row, wdata, ne0*nb0);
650 }
651 }
652}
653
654void ggml_compute_forward_add(
655 const ggml_compute_params * params,
656 ggml_tensor * dst) {
657
658 const ggml_tensor * src0 = dst->src[0];
659
660 switch (src0->type) {
661 case GGML_TYPE_F32:
662 case GGML_TYPE_F16:
663 case GGML_TYPE_BF16:
664 {
665 ggml_compute_forward_add_non_quantized(params, dst);
666 } break;
667 case GGML_TYPE_Q1_0:
668 case GGML_TYPE_Q4_0:
669 case GGML_TYPE_Q4_1:
670 case GGML_TYPE_Q5_0:
671 case GGML_TYPE_Q5_1:
672 case GGML_TYPE_Q8_0:
673 case GGML_TYPE_MXFP4:
674 case GGML_TYPE_NVFP4:
675 case GGML_TYPE_Q2_K:
676 case GGML_TYPE_Q3_K:
677 case GGML_TYPE_Q4_K:
678 case GGML_TYPE_Q5_K:
679 case GGML_TYPE_Q6_K:
680 case GGML_TYPE_TQ1_0:
681 case GGML_TYPE_TQ2_0:
682 case GGML_TYPE_IQ2_XXS:
683 case GGML_TYPE_IQ2_XS:
684 case GGML_TYPE_IQ3_XXS:
685 case GGML_TYPE_IQ1_S:
686 case GGML_TYPE_IQ1_M:
687 case GGML_TYPE_IQ4_NL:
688 case GGML_TYPE_IQ4_XS:
689 case GGML_TYPE_IQ3_S:
690 case GGML_TYPE_IQ2_S:
691 {
692 ggml_compute_forward_add_q_f32(params, dst);
693 } break;
694 default:
695 {
696 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 696, "fatal error");
697 }
698 }
699}
700
701// ggml_compute_forward_add_id
702
703static void ggml_compute_forward_add_id_f32(
704 const ggml_compute_params * params,
705 ggml_tensor * dst) {
706
707 const ggml_tensor * src0 = dst->src[0];
708 const ggml_tensor * src1 = dst->src[1];
709 const ggml_tensor * src2 = dst->src[2];
710
711 GGML_ASSERT(dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 711, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
712 GGML_ASSERT(src0->type == GGML_TYPE_F32)if (!(src0->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 712, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F32"
);
713 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 713, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
714 GGML_ASSERT(src2->type == GGML_TYPE_I32)if (!(src2->type == GGML_TYPE_I32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 714, "GGML_ASSERT(%s) failed", "src2->type == GGML_TYPE_I32"
);
715
716 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 716, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
717 GGML_ASSERT(src1->nb[0] == sizeof(float))if (!(src1->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 717, "GGML_ASSERT(%s) failed", "src1->nb[0] == sizeof(float)"
);
718
719 const int ith = params->ith;
720 const int nth = params->nth;
721
722 const int nr = ggml_nrows(src0);
723
724 GGML_TENSOR_TERNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne20 = (src2) ? (src2)->ne[0] : 0; (void)(ne20); const int64_t
ne21 = (src2) ? (src2)->ne[1] : 0; (void)(ne21); const int64_t
ne22 = (src2) ? (src2)->ne[2] : 0; (void)(ne22); const int64_t
ne23 = (src2) ? (src2)->ne[3] : 0; (void)(ne23); const size_t
nb20 = (src2) ? (src2)->nb[0] : 0; (void)(nb20); const size_t
nb21 = (src2) ? (src2)->nb[1] : 0; (void)(nb21); const size_t
nb22 = (src2) ? (src2)->nb[2] : 0; (void)(nb22); const size_t
nb23 = (src2) ? (src2)->nb[3] : 0; (void)(nb23); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
725
726 GGML_ASSERT( nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 726, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
727 GGML_ASSERT(nb10 == sizeof(float))if (!(nb10 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 727, "GGML_ASSERT(%s) failed", "nb10 == sizeof(float)");
728
729 // rows per thread
730 const int dr = (nr + nth - 1)/nth;
731
732 // row range for this thread
733 const int ir0 = dr*ith;
734 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
735
736 for (int ir = ir0; ir < ir1; ++ir) {
737 // src0 indices
738 const int i3 = ir/(ne2*ne1);
739 const int i2 = (ir - i3*ne2*ne1)/ne1;
740 const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
741
742 // src1 indices
743 const int i11 = *(int32_t *) ((char *) src2->data + i1*nb20 + i2*nb21);
744
745 GGML_ASSERT(i11 >= 0 && i11 < ne11)if (!(i11 >= 0 && i11 < ne11)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 745, "GGML_ASSERT(%s) failed", "i11 >= 0 && i11 < ne11"
);
746
747 ggml_vec_add_f32(ne0,
748 (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ),
749 (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01),
750 (float *) ((char *) src1->data + i11*nb11));
751 }
752}
753
754void ggml_compute_forward_add_id(
755 const ggml_compute_params * params,
756 ggml_tensor * dst) {
757
758 const ggml_tensor * src0 = dst->src[0];
759
760 switch (src0->type) {
761 case GGML_TYPE_F32:
762 {
763 ggml_compute_forward_add_id_f32(params, dst);
764 } break;
765 default:
766 {
767 GGML_ABORT("unsupported type for ggml_compute_forward_add_id: %s", ggml_type_name(src0->type))ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 767, "unsupported type for ggml_compute_forward_add_id: %s"
, ggml_type_name(src0->type));
768 }
769 }
770}
771
772// ggml_compute_forward_add1
773
774static void ggml_compute_forward_add1_f32(
775 const ggml_compute_params * params,
776 ggml_tensor * dst) {
777
778 const ggml_tensor * src0 = dst->src[0];
779 const ggml_tensor * src1 = dst->src[1];
780
781 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 781, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
782 GGML_ASSERT(ggml_is_scalar(src1))if (!(ggml_is_scalar(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 782, "GGML_ASSERT(%s) failed", "ggml_is_scalar(src1)");
783
784 const int ith = params->ith;
785 const int nth = params->nth;
786
787 const int nr = ggml_nrows(src0);
788
789 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
790
791 GGML_ASSERT( nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 791, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
792 GGML_ASSERT(nb00 == sizeof(float))if (!(nb00 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 792, "GGML_ASSERT(%s) failed", "nb00 == sizeof(float)");
793
794 // rows per thread
795 const int dr = (nr + nth - 1)/nth;
796
797 // row range for this thread
798 const int ir0 = dr*ith;
799 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
800
801 for (int ir = ir0; ir < ir1; ++ir) {
802 // src0 and dst are same shape => same indices
803 const int i3 = ir/(ne2*ne1);
804 const int i2 = (ir - i3*ne2*ne1)/ne1;
805 const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
806
807#ifdef GGML_USE_ACCELERATE
808 GGML_UNUSED(ggml_vec_add1_f32)(void)(ggml_vec_add1_f32);
809
810 vDSP_vadd(
811 (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01), 1,
812 (float *) ((char *) src1->data), 0,
813 (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ), 1,
814 ne0);
815#else
816 ggml_vec_add1_f32(ne0,
817 (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ),
818 (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01),
819 *(float *) src1->data);
820#endif
821 }
822}
823
824static void ggml_compute_forward_add1_f16_f32(
825 const ggml_compute_params * params,
826 ggml_tensor * dst) {
827
828 const ggml_tensor * src0 = dst->src[0];
829 const ggml_tensor * src1 = dst->src[1];
830
831 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 831, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
832 GGML_ASSERT(ggml_is_scalar(src1))if (!(ggml_is_scalar(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 832, "GGML_ASSERT(%s) failed", "ggml_is_scalar(src1)");
833
834 // scalar to add
835 const float v = *(float *) src1->data;
836
837 const int ith = params->ith;
838 const int nth = params->nth;
839
840 const int nr = ggml_nrows(src0);
841
842 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
843
844 GGML_ASSERT(src0->type == GGML_TYPE_F16)if (!(src0->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 844, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F16"
);
845 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 845, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
846 GGML_ASSERT(dst->type == GGML_TYPE_F16)if (!(dst->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 846, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F16"
);
847
848 GGML_ASSERT( nb0 == sizeof(ggml_fp16_t))if (!(nb0 == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 848, "GGML_ASSERT(%s) failed", "nb0 == sizeof(ggml_fp16_t)"
);
849 GGML_ASSERT(nb00 == sizeof(ggml_fp16_t))if (!(nb00 == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 849, "GGML_ASSERT(%s) failed", "nb00 == sizeof(ggml_fp16_t)"
);
850
851 // rows per thread
852 const int dr = (nr + nth - 1)/nth;
853
854 // row range for this thread
855 const int ir0 = dr*ith;
856 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
857
858 for (int ir = ir0; ir < ir1; ++ir) {
859 // src0 and dst are same shape => same indices
860 const int i3 = ir/(ne2*ne1);
861 const int i2 = (ir - i3*ne2*ne1)/ne1;
862 const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
863
864 ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
865 ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
866 for (int i = 0; i < ne0; i++) {
867 dst_ptr[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(src0_ptr[i]) + v)ggml_compute_fp32_to_fp16(ggml_lookup_fp16_to_fp32(src0_ptr[i
]) + v);
868 }
869 }
870}
871
872static void ggml_compute_forward_add1_f16_f16(
873 const ggml_compute_params * params,
874 ggml_tensor * dst) {
875
876 const ggml_tensor * src0 = dst->src[0];
877 const ggml_tensor * src1 = dst->src[1];
878
879 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 879, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
880 GGML_ASSERT(ggml_is_scalar(src1))if (!(ggml_is_scalar(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 880, "GGML_ASSERT(%s) failed", "ggml_is_scalar(src1)");
881
882 // scalar to add
883 const float v = GGML_CPU_FP16_TO_FP32(*(ggml_fp16_t *) src1->data)ggml_lookup_fp16_to_fp32(*(ggml_fp16_t *) src1->data);
884
885 const int ith = params->ith;
886 const int nth = params->nth;
887
888 const int nr = ggml_nrows(src0);
889
890 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
891
892 GGML_ASSERT(src0->type == GGML_TYPE_F16)if (!(src0->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 892, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F16"
);
893 GGML_ASSERT(src1->type == GGML_TYPE_F16)if (!(src1->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 893, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F16"
);
894 GGML_ASSERT(dst->type == GGML_TYPE_F16)if (!(dst->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 894, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F16"
);
895
896 GGML_ASSERT( nb0 == sizeof(ggml_fp16_t))if (!(nb0 == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 896, "GGML_ASSERT(%s) failed", "nb0 == sizeof(ggml_fp16_t)"
);
897 GGML_ASSERT(nb00 == sizeof(ggml_fp16_t))if (!(nb00 == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 897, "GGML_ASSERT(%s) failed", "nb00 == sizeof(ggml_fp16_t)"
);
898
899 // rows per thread
900 const int dr = (nr + nth - 1)/nth;
901
902 // row range for this thread
903 const int ir0 = dr*ith;
904 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
905
906 for (int ir = ir0; ir < ir1; ++ir) {
907 // src0 and dst are same shape => same indices
908 const int i3 = ir/(ne2*ne1);
909 const int i2 = (ir - i3*ne2*ne1)/ne1;
910 const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
911
912 ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
913 ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
914 for (int i = 0; i < ne0; i++) {
915 dst_ptr[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(src0_ptr[i]) + v)ggml_compute_fp32_to_fp16(ggml_lookup_fp16_to_fp32(src0_ptr[i
]) + v);
916 }
917 }
918}
919
920static void ggml_compute_forward_add1_q_f32(
921 const ggml_compute_params * params,
922 ggml_tensor * dst) {
923
924 const ggml_tensor * src0 = dst->src[0];
925 const ggml_tensor * src1 = dst->src[1];
926
927 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 927, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
928 GGML_ASSERT(ggml_is_scalar(src1))if (!(ggml_is_scalar(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 928, "GGML_ASSERT(%s) failed", "ggml_is_scalar(src1)");
929
930 // scalar to add
931 const float v = *(float *) src1->data;
932
933 const int ith = params->ith;
934 const int nth = params->nth;
935
936 const int nr = ggml_nrows(src0);
937
938 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
939
940 const ggml_type type = src0->type;
941 ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;
942 ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(type)->from_float;
943
944 // we don't support permuted src0
945 GGML_ASSERT(nb00 == ggml_type_size(type))if (!(nb00 == ggml_type_size(type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 945, "GGML_ASSERT(%s) failed", "nb00 == ggml_type_size(type)"
);
946
947 // dst cannot be transposed or permuted
948 GGML_ASSERT(nb0 <= nb1)if (!(nb0 <= nb1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 948, "GGML_ASSERT(%s) failed", "nb0 <= nb1");
949 GGML_ASSERT(nb1 <= nb2)if (!(nb1 <= nb2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 949, "GGML_ASSERT(%s) failed", "nb1 <= nb2");
950 GGML_ASSERT(nb2 <= nb3)if (!(nb2 <= nb3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 950, "GGML_ASSERT(%s) failed", "nb2 <= nb3");
951
952 GGML_ASSERT(ggml_is_quantized(src0->type))if (!(ggml_is_quantized(src0->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 952, "GGML_ASSERT(%s) failed", "ggml_is_quantized(src0->type)"
);
953 GGML_ASSERT(dst->type == src0->type)if (!(dst->type == src0->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 953, "GGML_ASSERT(%s) failed", "dst->type == src0->type"
);
954 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 954, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
955
956 // rows per thread
957 const int dr = (nr + nth - 1)/nth;
958
959 // row range for this thread
960 const int ir0 = dr*ith;
961 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
962
963 float * wdata = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32) * ith;
964
965 for (int ir = ir0; ir < ir1; ++ir) {
966 // src0 and dst are same shape => same indices
967 const int i3 = ir/(ne2*ne1);
968 const int i2 = (ir - i3*ne2*ne1)/ne1;
969 const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
970
971 void * src0_row = (void *) ((char *) src0->data + (i1*nb01 + i2*nb02 + i3*nb03));
972 void * dst_row = (void *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb0 ));
973
974 assert(ne0 % 32 == 0)(static_cast <bool> (ne0 % 32 == 0) ? void (0) : __assert_fail
("ne0 % 32 == 0", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
975
976 // unquantize row from src0 to temp buffer
977 dequantize_row_q(src0_row, wdata, ne0);
978 // add src1
979 ggml_vec_acc1_f32(ne0, wdata, v);
980 // quantize row to dst
981 quantize_row_q(wdata, dst_row, ne0);
982 }
983}
984
985static void ggml_compute_forward_add1_bf16_f32(
986 const ggml_compute_params * params,
987 ggml_tensor * dst) {
988
989 const ggml_tensor * src0 = dst->src[0];
990 const ggml_tensor * src1 = dst->src[1];
991
992 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 992, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
993 GGML_ASSERT(ggml_is_scalar(src1))if (!(ggml_is_scalar(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 993, "GGML_ASSERT(%s) failed", "ggml_is_scalar(src1)");
994
995 // scalar to add
996 const float v = *(float *) src1->data;
997
998 const int ith = params->ith;
999 const int nth = params->nth;
1000
1001 const int nr = ggml_nrows(src0);
1002
1003 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
1004
1005 GGML_ASSERT(src0->type == GGML_TYPE_BF16)if (!(src0->type == GGML_TYPE_BF16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1005, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_BF16"
);
1006 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1006, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
1007 GGML_ASSERT(dst->type == GGML_TYPE_BF16)if (!(dst->type == GGML_TYPE_BF16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1007, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_BF16"
);
1008
1009 GGML_ASSERT( nb0 == sizeof(ggml_bf16_t))if (!(nb0 == sizeof(ggml_bf16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1009, "GGML_ASSERT(%s) failed", "nb0 == sizeof(ggml_bf16_t)"
);
1010 GGML_ASSERT(nb00 == sizeof(ggml_bf16_t))if (!(nb00 == sizeof(ggml_bf16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1010, "GGML_ASSERT(%s) failed", "nb00 == sizeof(ggml_bf16_t)"
);
1011
1012 // rows per thread
1013 const int dr = (nr + nth - 1)/nth;
1014
1015 // row range for this thread
1016 const int ir0 = dr*ith;
1017 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
1018
1019 for (int ir = ir0; ir < ir1; ++ir) {
1020 // src0 and dst are same shape => same indices
1021 const int i3 = ir/(ne2*ne1);
1022 const int i2 = (ir - i3*ne2*ne1)/ne1;
1023 const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
1024
1025 ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
1026 ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
1027 for (int i = 0; i < ne0; i++) {
1028 dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v)ggml_compute_fp32_to_bf16(ggml_compute_bf16_to_fp32(src0_ptr[
i]) + v);
1029 }
1030 }
1031}
1032
1033static void ggml_compute_forward_add1_bf16_bf16(
1034 const ggml_compute_params * params,
1035 ggml_tensor * dst) {
1036
1037 const ggml_tensor * src0 = dst->src[0];
1038 const ggml_tensor * src1 = dst->src[1];
1039
1040 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1040, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
1041 GGML_ASSERT(ggml_is_scalar(src1))if (!(ggml_is_scalar(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1041, "GGML_ASSERT(%s) failed", "ggml_is_scalar(src1)");
1042
1043 // scalar to add
1044 const float v = GGML_BF16_TO_FP32(*(ggml_bf16_t *) src1->data)ggml_compute_bf16_to_fp32(*(ggml_bf16_t *) src1->data);
1045
1046 const int ith = params->ith;
1047 const int nth = params->nth;
1048
1049 const int nr = ggml_nrows(src0);
1050
1051 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
1052
1053 GGML_ASSERT(src0->type == GGML_TYPE_BF16)if (!(src0->type == GGML_TYPE_BF16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1053, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_BF16"
);
1054 GGML_ASSERT(src1->type == GGML_TYPE_BF16)if (!(src1->type == GGML_TYPE_BF16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1054, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_BF16"
);
1055 GGML_ASSERT(dst->type == GGML_TYPE_BF16)if (!(dst->type == GGML_TYPE_BF16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1055, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_BF16"
);
1056
1057 GGML_ASSERT( nb0 == sizeof(ggml_bf16_t))if (!(nb0 == sizeof(ggml_bf16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1057, "GGML_ASSERT(%s) failed", "nb0 == sizeof(ggml_bf16_t)"
);
1058 GGML_ASSERT(nb00 == sizeof(ggml_bf16_t))if (!(nb00 == sizeof(ggml_bf16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1058, "GGML_ASSERT(%s) failed", "nb00 == sizeof(ggml_bf16_t)"
);
1059
1060 // rows per thread
1061 const int dr = (nr + nth - 1)/nth;
1062
1063 // row range for this thread
1064 const int ir0 = dr*ith;
1065 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
1066
1067 for (int ir = ir0; ir < ir1; ++ir) {
1068 // src0 and dst are same shape => same indices
1069 const int i3 = ir/(ne2*ne1);
1070 const int i2 = (ir - i3*ne2*ne1)/ne1;
1071 const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
1072
1073 ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
1074 ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
1075 for (int i = 0; i < ne0; i++) {
1076 dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v)ggml_compute_fp32_to_bf16(ggml_compute_bf16_to_fp32(src0_ptr[
i]) + v);
1077 }
1078 }
1079}
1080
1081void ggml_compute_forward_add1(
1082 const ggml_compute_params * params,
1083 ggml_tensor * dst) {
1084
1085 const ggml_tensor * src0 = dst->src[0];
1086 const ggml_tensor * src1 = dst->src[1];
1087
1088 switch (src0->type) {
1089 case GGML_TYPE_F32:
1090 {
1091 ggml_compute_forward_add1_f32(params, dst);
1092 } break;
1093 case GGML_TYPE_F16:
1094 {
1095 if (src1->type == GGML_TYPE_F16) {
1096 ggml_compute_forward_add1_f16_f16(params, dst);
1097 }
1098 else if (src1->type == GGML_TYPE_F32) {
1099 ggml_compute_forward_add1_f16_f32(params, dst);
1100 }
1101 else {
1102 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1102, "fatal error");
1103 }
1104 } break;
1105 case GGML_TYPE_BF16:
1106 {
1107 if (src1->type == GGML_TYPE_BF16) {
1108 ggml_compute_forward_add1_bf16_bf16(params, dst);
1109 }
1110 else if (src1->type == GGML_TYPE_F32) {
1111 ggml_compute_forward_add1_bf16_f32(params, dst);
1112 }
1113 else {
1114 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1114, "fatal error");
1115 }
1116 } break;
1117 case GGML_TYPE_Q1_0:
1118 case GGML_TYPE_Q4_0:
1119 case GGML_TYPE_Q4_1:
1120 case GGML_TYPE_Q5_0:
1121 case GGML_TYPE_Q5_1:
1122 case GGML_TYPE_Q8_0:
1123 case GGML_TYPE_Q8_1:
1124 case GGML_TYPE_MXFP4:
1125 case GGML_TYPE_NVFP4:
1126 case GGML_TYPE_Q2_K:
1127 case GGML_TYPE_Q3_K:
1128 case GGML_TYPE_Q4_K:
1129 case GGML_TYPE_Q5_K:
1130 case GGML_TYPE_Q6_K:
1131 case GGML_TYPE_TQ1_0:
1132 case GGML_TYPE_TQ2_0:
1133 case GGML_TYPE_IQ2_XXS:
1134 case GGML_TYPE_IQ2_XS:
1135 case GGML_TYPE_IQ3_XXS:
1136 case GGML_TYPE_IQ1_S:
1137 case GGML_TYPE_IQ1_M:
1138 case GGML_TYPE_IQ4_NL:
1139 case GGML_TYPE_IQ4_XS:
1140 case GGML_TYPE_IQ3_S:
1141 case GGML_TYPE_IQ2_S:
1142 {
1143 ggml_compute_forward_add1_q_f32(params, dst);
1144 } break;
1145 default:
1146 {
1147 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1147, "fatal error");
1148 }
1149 }
1150}
1151
1152// ggml_compute_forward_acc
1153
1154static void ggml_compute_forward_acc_f32(
1155 const ggml_compute_params * params,
1156 ggml_tensor * dst) {
1157
1158 const ggml_tensor * src0 = dst->src[0];
1159 const ggml_tensor * src1 = dst->src[1];
1160
1161 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1161, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
1162 GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0))if (!(ggml_is_contiguous(dst) && ggml_is_contiguous(src0
))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1162, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dst) && ggml_is_contiguous(src0)"
);
1163
1164 // view src0 and dst with these strides and data offset inbytes during acc
1165 // nb0 is implicitly element_size because src0 and dst are contiguous
1166 size_t nb1 = ((int32_t *) dst->op_params)[0];
1167 size_t nb2 = ((int32_t *) dst->op_params)[1];
1168 size_t nb3 = ((int32_t *) dst->op_params)[2];
1169 size_t offset = ((int32_t *) dst->op_params)[3];
1170 bool inplace = (bool) ((int32_t *) dst->op_params)[4];
1171
1172 if (!inplace) {
1173 if (params->ith == 0) {
1174 // memcpy needs to be synchronized across threads to avoid race conditions.
1175 // => do it in INIT phase
1176 memcpy(
1177 ((char *) dst->data),
1178 ((char *) src0->data),
1179 ggml_nbytes(dst));
1180 }
1181 ggml_barrier(params->threadpool);
1182 }
1183
1184 const int ith = params->ith;
1185 const int nth = params->nth;
1186
1187 const int nr = ggml_nrows(src1);
1188 const int nc = src1->ne[0];
1189
1190 GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne)const int64_t ne10 = (src1) ? (src1)->ne[0] : 0; (void)(ne10
); const int64_t ne11 = (src1) ? (src1)->ne[1] : 0; (void)
(ne11); const int64_t ne12 = (src1) ? (src1)->ne[2] : 0; (
void)(ne12); const int64_t ne13 = (src1) ? (src1)->ne[3] :
0; (void)(ne13);
1191 GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)const size_t nb10 = (src1) ? (src1)->nb[0] : 0; (void)(nb10
); const size_t nb11 = (src1) ? (src1)->nb[1] : 0; (void)(
nb11); const size_t nb12 = (src1) ? (src1)->nb[2] : 0; (void
)(nb12); const size_t nb13 = (src1) ? (src1)->nb[3] : 0; (
void)(nb13);
1192
1193 // src0 and dst as viewed during acc
1194 const size_t nb0 = ggml_element_size(src0);
1195
1196 const size_t nb00 = nb0;
1197 const size_t nb01 = nb1;
1198 const size_t nb02 = nb2;
1199 const size_t nb03 = nb3;
1200
1201 GGML_ASSERT(offset + (ne10 == 0 ? 0 : ne10-1)*nb0 + (ne11 == 0 ? 0 : ne11-1)*nb1 + (ne12 == 0 ? 0 : ne12-1)*nb2 + (ne13 == 0 ? 0 : ne13-1)*nb3 < ggml_nbytes(dst))if (!(offset + (ne10 == 0 ? 0 : ne10-1)*nb0 + (ne11 == 0 ? 0 :
ne11-1)*nb1 + (ne12 == 0 ? 0 : ne12-1)*nb2 + (ne13 == 0 ? 0 :
ne13-1)*nb3 < ggml_nbytes(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1201, "GGML_ASSERT(%s) failed", "offset + (ne10 == 0 ? 0 : ne10-1)*nb0 + (ne11 == 0 ? 0 : ne11-1)*nb1 + (ne12 == 0 ? 0 : ne12-1)*nb2 + (ne13 == 0 ? 0 : ne13-1)*nb3 < ggml_nbytes(dst)"
);
1202 GGML_ASSERT(offset + (ne10 == 0 ? 0 : ne10-1)*nb00 + (ne11 == 0 ? 0 : ne11-1)*nb01 + (ne12 == 0 ? 0 : ne12-1)*nb02 + (ne13 == 0 ? 0 : ne13-1)*nb03 < ggml_nbytes(src0))if (!(offset + (ne10 == 0 ? 0 : ne10-1)*nb00 + (ne11 == 0 ? 0
: ne11-1)*nb01 + (ne12 == 0 ? 0 : ne12-1)*nb02 + (ne13 == 0 ?
0 : ne13-1)*nb03 < ggml_nbytes(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1202, "GGML_ASSERT(%s) failed", "offset + (ne10 == 0 ? 0 : ne10-1)*nb00 + (ne11 == 0 ? 0 : ne11-1)*nb01 + (ne12 == 0 ? 0 : ne12-1)*nb02 + (ne13 == 0 ? 0 : ne13-1)*nb03 < ggml_nbytes(src0)"
);
1203
1204 GGML_ASSERT(nb10 == sizeof(float))if (!(nb10 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1204, "GGML_ASSERT(%s) failed", "nb10 == sizeof(float)");
1205
1206 // rows per thread
1207 const int dr = (nr + nth - 1)/nth;
1208
1209 // row range for this thread
1210 const int ir0 = dr*ith;
1211 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
1212
1213 for (int ir = ir0; ir < ir1; ++ir) {
1214 // src0 and dst are viewed with shape of src1 and offset
1215 // => same indices
1216 const int i3 = ir/(ne12*ne11);
1217 const int i2 = (ir - i3*ne12*ne11)/ne11;
1218 const int i1 = (ir - i3*ne12*ne11 - i2*ne11);
1219
1220#ifdef GGML_USE_ACCELERATE
1221 vDSP_vadd(
1222 (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + offset), 1,
1223 (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11), 1,
1224 (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + offset), 1, nc);
1225#else
1226 ggml_vec_add_f32(nc,
1227 (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + offset),
1228 (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + offset),
1229 (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));
1230#endif
1231 }
1232}
1233
1234void ggml_compute_forward_acc(
1235 const ggml_compute_params * params,
1236 ggml_tensor * dst) {
1237
1238 const ggml_tensor * src0 = dst->src[0];
1239
1240 switch (src0->type) {
1241 case GGML_TYPE_F32:
1242 {
1243 ggml_compute_forward_acc_f32(params, dst);
1244 } break;
1245 case GGML_TYPE_F16:
1246 case GGML_TYPE_BF16:
1247 case GGML_TYPE_Q1_0:
1248 case GGML_TYPE_Q4_0:
1249 case GGML_TYPE_Q4_1:
1250 case GGML_TYPE_Q5_0:
1251 case GGML_TYPE_Q5_1:
1252 case GGML_TYPE_Q8_0:
1253 case GGML_TYPE_Q8_1:
1254 case GGML_TYPE_MXFP4:
1255 case GGML_TYPE_NVFP4:
1256 case GGML_TYPE_Q2_K:
1257 case GGML_TYPE_Q3_K:
1258 case GGML_TYPE_Q4_K:
1259 case GGML_TYPE_Q5_K:
1260 case GGML_TYPE_Q6_K:
1261 case GGML_TYPE_TQ1_0:
1262 case GGML_TYPE_TQ2_0:
1263 case GGML_TYPE_IQ2_XXS:
1264 case GGML_TYPE_IQ2_XS:
1265 case GGML_TYPE_IQ3_XXS:
1266 case GGML_TYPE_IQ1_S:
1267 case GGML_TYPE_IQ1_M:
1268 case GGML_TYPE_IQ4_NL:
1269 case GGML_TYPE_IQ4_XS:
1270 case GGML_TYPE_IQ3_S:
1271 case GGML_TYPE_IQ2_S:
1272 default:
1273 {
1274 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1274, "fatal error");
1275 }
1276 }
1277}
1278
1279// ggml_compute_forward_sum
1280
1281static void ggml_compute_forward_sum_f32(
1282 const ggml_compute_params * params,
1283 ggml_tensor * dst) {
1284
1285 const ggml_tensor * src0 = dst->src[0];
1286
1287 if (params->ith != 0) {
1288 return;
1289 }
1290
1291 assert(ggml_is_scalar(dst))(static_cast <bool> (ggml_is_scalar(dst)) ? void (0) : __assert_fail
("ggml_is_scalar(dst)", __builtin_FILE (), __builtin_LINE ()
, __extension__ __PRETTY_FUNCTION__));
1292 assert(src0->nb[0] == sizeof(float))(static_cast <bool> (src0->nb[0] == sizeof(float)) ?
void (0) : __assert_fail ("src0->nb[0] == sizeof(float)",
__builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
1293
1294 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
1295 GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)const size_t nb00 = (src0) ? (src0)->nb[0] : 0; (void)(nb00
); const size_t nb01 = (src0) ? (src0)->nb[1] : 0; (void)(
nb01); const size_t nb02 = (src0) ? (src0)->nb[2] : 0; (void
)(nb02); const size_t nb03 = (src0) ? (src0)->nb[3] : 0; (
void)(nb03);
1296
1297 ggml_float sum = 0;
1298 ggml_float row_sum = 0;
1299
1300 for (int64_t i03 = 0; i03 < ne03; i03++) {
1301 for (int64_t i02 = 0; i02 < ne02; i02++) {
1302 for (int64_t i01 = 0; i01 < ne01; i01++) {
1303 ggml_vec_sum_f32_ggf(ne00,
1304 &row_sum,
1305 (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
1306 sum += row_sum;
1307 }
1308 }
1309 }
1310 ((float *) dst->data)[0] = sum;
1311}
1312
1313static void ggml_compute_forward_sum_f16(
1314 const ggml_compute_params * params,
1315 ggml_tensor * dst) {
1316
1317 const ggml_tensor * src0 = dst->src[0];
1318
1319 if (params->ith != 0) {
1320 return;
1321 }
1322
1323 assert(ggml_is_scalar(dst))(static_cast <bool> (ggml_is_scalar(dst)) ? void (0) : __assert_fail
("ggml_is_scalar(dst)", __builtin_FILE (), __builtin_LINE ()
, __extension__ __PRETTY_FUNCTION__));
1324
1325 assert(src0->nb[0] == sizeof(ggml_fp16_t))(static_cast <bool> (src0->nb[0] == sizeof(ggml_fp16_t
)) ? void (0) : __assert_fail ("src0->nb[0] == sizeof(ggml_fp16_t)"
, __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
1326
1327 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
1328 GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)const size_t nb00 = (src0) ? (src0)->nb[0] : 0; (void)(nb00
); const size_t nb01 = (src0) ? (src0)->nb[1] : 0; (void)(
nb01); const size_t nb02 = (src0) ? (src0)->nb[2] : 0; (void
)(nb02); const size_t nb03 = (src0) ? (src0)->nb[3] : 0; (
void)(nb03);
1329
1330 float sum = 0;
1331 float row_sum = 0;
1332
1333 for (int64_t i03 = 0; i03 < ne03; i03++) {
1334 for (int64_t i02 = 0; i02 < ne02; i02++) {
1335 for (int64_t i01 = 0; i01 < ne01; i01++) {
1336 ggml_vec_sum_f16_ggf(ne00,
1337 &row_sum,
1338 (ggml_fp16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));
1339 sum += row_sum;
1340 }
1341 }
1342 }
1343 ((ggml_fp16_t *) dst->data)[0] = GGML_CPU_FP32_TO_FP16(sum)ggml_compute_fp32_to_fp16(sum);
1344}
1345
1346static void ggml_compute_forward_sum_bf16(
1347 const ggml_compute_params * params,
1348 ggml_tensor * dst) {
1349
1350 const ggml_tensor * src0 = dst->src[0];
1351
1352 if (params->ith != 0) {
1353 return;
1354 }
1355
1356 assert(ggml_is_scalar(dst))(static_cast <bool> (ggml_is_scalar(dst)) ? void (0) : __assert_fail
("ggml_is_scalar(dst)", __builtin_FILE (), __builtin_LINE ()
, __extension__ __PRETTY_FUNCTION__));
1357
1358 assert(src0->nb[0] == sizeof(ggml_bf16_t))(static_cast <bool> (src0->nb[0] == sizeof(ggml_bf16_t
)) ? void (0) : __assert_fail ("src0->nb[0] == sizeof(ggml_bf16_t)"
, __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
1359
1360 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
1361 GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)const size_t nb00 = (src0) ? (src0)->nb[0] : 0; (void)(nb00
); const size_t nb01 = (src0) ? (src0)->nb[1] : 0; (void)(
nb01); const size_t nb02 = (src0) ? (src0)->nb[2] : 0; (void
)(nb02); const size_t nb03 = (src0) ? (src0)->nb[3] : 0; (
void)(nb03);
1362
1363 float sum = 0;
1364 float row_sum = 0;
1365
1366 for (int64_t i03 = 0; i03 < ne03; i03++) {
1367 for (int64_t i02 = 0; i02 < ne02; i02++) {
1368 for (int64_t i01 = 0; i01 < ne01; i01++) {
1369 ggml_vec_sum_bf16_ggf(ne00,
1370 &row_sum,
1371 (ggml_bf16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));
1372 sum += row_sum;
1373 }
1374 }
1375 }
1376 ((ggml_bf16_t *) dst->data)[0] = GGML_FP32_TO_BF16(sum)ggml_compute_fp32_to_bf16(sum);
1377}
1378
1379void ggml_compute_forward_sum(
1380 const ggml_compute_params * params,
1381 ggml_tensor * dst) {
1382
1383 const ggml_tensor * src0 = dst->src[0];
1384
1385 switch (src0->type) {
1386 case GGML_TYPE_F32:
1387 {
1388 ggml_compute_forward_sum_f32(params, dst);
1389 } break;
1390 case GGML_TYPE_F16:
1391 {
1392 ggml_compute_forward_sum_f16(params, dst);
1393 } break;
1394 case GGML_TYPE_BF16:
1395 {
1396 ggml_compute_forward_sum_bf16(params, dst);
1397 } break;
1398 default:
1399 {
1400 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1400, "fatal error");
1401 }
1402 }
1403}
1404
1405// ggml_compute_forward_cumsum
1406
1407static void ggml_compute_forward_cumsum_f32(
1408 const ggml_compute_params * params,
1409 ggml_tensor * dst) {
1410
1411 const ggml_tensor * src0 = dst->src[0];
1412
1413 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1413, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
1414 GGML_ASSERT(dst->nb[0] == sizeof(float))if (!(dst->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1414, "GGML_ASSERT(%s) failed", "dst->nb[0] == sizeof(float)"
);
1415
1416 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
1417
1418 GGML_ASSERT(ne0 == ne00)if (!(ne0 == ne00)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1418, "GGML_ASSERT(%s) failed", "ne0 == ne00");
1419 GGML_ASSERT(ne1 == ne01)if (!(ne1 == ne01)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1419, "GGML_ASSERT(%s) failed", "ne1 == ne01");
1420 GGML_ASSERT(ne2 == ne02)if (!(ne2 == ne02)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1420, "GGML_ASSERT(%s) failed", "ne2 == ne02");
1421 GGML_ASSERT(ne3 == ne03)if (!(ne3 == ne03)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1421, "GGML_ASSERT(%s) failed", "ne3 == ne03");
1422
1423 const auto [ir0, ir1] = get_thread_range(params, src0);
1424
1425 for (int64_t ir = ir0; ir < ir1; ++ir) {
1426 const int64_t i03 = ir/(ne02*ne01);
1427 const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
1428 const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
1429
1430 float * src_row = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
1431 float * dst_row = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
1432
1433 ggml_vec_cumsum_f32(ne00, dst_row, src_row);
1434 }
1435}
1436
1437void ggml_compute_forward_cumsum(
1438 const ggml_compute_params * params,
1439 ggml_tensor * dst) {
1440
1441 const ggml_tensor * src0 = dst->src[0];
1442
1443 switch (src0->type) {
1444 case GGML_TYPE_F32:
1445 {
1446 ggml_compute_forward_cumsum_f32(params, dst);
1447 } break;
1448 default:
1449 {
1450 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1450, "fatal error");
1451 }
1452 }
1453}
1454
1455// ggml_compute_forward_sum_rows
1456
1457static void ggml_compute_forward_sum_rows_f32(
1458 const ggml_compute_params * params,
1459 ggml_tensor * dst) {
1460
1461 const ggml_tensor * src0 = dst->src[0];
1462
1463 if (params->ith != 0) {
1464 return;
1465 }
1466
1467 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1467, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
1468 GGML_ASSERT(dst->nb[0] == sizeof(float))if (!(dst->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1468, "GGML_ASSERT(%s) failed", "dst->nb[0] == sizeof(float)"
);
1469
1470 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
1471
1472 GGML_ASSERT(ne0 == 1)if (!(ne0 == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1472, "GGML_ASSERT(%s) failed", "ne0 == 1");
1473 GGML_ASSERT(ne1 == ne01)if (!(ne1 == ne01)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1473, "GGML_ASSERT(%s) failed", "ne1 == ne01");
1474 GGML_ASSERT(ne2 == ne02)if (!(ne2 == ne02)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1474, "GGML_ASSERT(%s) failed", "ne2 == ne02");
1475 GGML_ASSERT(ne3 == ne03)if (!(ne3 == ne03)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1475, "GGML_ASSERT(%s) failed", "ne3 == ne03");
1476
1477 for (int64_t i3 = 0; i3 < ne03; i3++) {
1478 for (int64_t i2 = 0; i2 < ne02; i2++) {
1479 for (int64_t i1 = 0; i1 < ne01; i1++) {
1480 float * src_row = (float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
1481 float * dst_row = (float *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
1482 float row_sum = 0;
1483 ggml_vec_sum_f32(ne00, &row_sum, src_row);
1484 dst_row[0] = row_sum;
1485 }
1486 }
1487 }
1488}
1489
1490void ggml_compute_forward_sum_rows(
1491 const ggml_compute_params * params,
1492 ggml_tensor * dst) {
1493
1494 const ggml_tensor * src0 = dst->src[0];
1495
1496 switch (src0->type) {
1497 case GGML_TYPE_F32:
1498 {
1499 ggml_compute_forward_sum_rows_f32(params, dst);
1500 } break;
1501 default:
1502 {
1503 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1503, "fatal error");
1504 }
1505 }
1506}
1507
1508// ggml_compute_forward_mean
1509
1510static void ggml_compute_forward_mean_f32(
1511 const ggml_compute_params * params,
1512 ggml_tensor * dst) {
1513
1514 const ggml_tensor * src0 = dst->src[0];
1515
1516 if (params->ith != 0) {
1517 return;
1518 }
1519
1520 assert(src0->nb[0] == sizeof(float))(static_cast <bool> (src0->nb[0] == sizeof(float)) ?
void (0) : __assert_fail ("src0->nb[0] == sizeof(float)",
__builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
1521
1522 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
1523
1524 assert(ne0 == 1)(static_cast <bool> (ne0 == 1) ? void (0) : __assert_fail
("ne0 == 1", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
1525 assert(ne1 == ne01)(static_cast <bool> (ne1 == ne01) ? void (0) : __assert_fail
("ne1 == ne01", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
1526 assert(ne2 == ne02)(static_cast <bool> (ne2 == ne02) ? void (0) : __assert_fail
("ne2 == ne02", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
1527 assert(ne3 == ne03)(static_cast <bool> (ne3 == ne03) ? void (0) : __assert_fail
("ne3 == ne03", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
1528
1529 GGML_UNUSED(ne0)(void)(ne0);
1530 GGML_UNUSED(ne1)(void)(ne1);
1531 GGML_UNUSED(ne2)(void)(ne2);
1532 GGML_UNUSED(ne3)(void)(ne3);
1533
1534 for (int64_t i03 = 0; i03 < ne03; i03++) {
1535 for (int64_t i02 = 0; i02 < ne02; i02++) {
1536 for (int64_t i01 = 0; i01 < ne01; i01++) {
1537 ggml_vec_sum_f32(ne00,
1538 (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),
1539 (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
1540
1541 *(float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3) /= (float) ne00;
1542 }
1543 }
1544 }
1545}
1546
1547void ggml_compute_forward_mean(
1548 const ggml_compute_params * params,
1549 ggml_tensor * dst) {
1550
1551 const ggml_tensor * src0 = dst->src[0];
1552
1553 switch (src0->type) {
1554 case GGML_TYPE_F32:
1555 {
1556 ggml_compute_forward_mean_f32(params, dst);
1557 } break;
1558 default:
1559 {
1560 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1560, "fatal error");
1561 }
1562 }
1563}
1564
1565// ggml_compute_forward_argmax
1566
1567static void ggml_compute_forward_argmax_f32(
1568 const ggml_compute_params * params,
1569 ggml_tensor * dst) {
1570
1571 const ggml_tensor * src0 = dst->src[0];
1572
1573 if (params->ith != 0) {
1574 return;
1575 }
1576
1577 assert(src0->nb[0] == sizeof(float))(static_cast <bool> (src0->nb[0] == sizeof(float)) ?
void (0) : __assert_fail ("src0->nb[0] == sizeof(float)",
__builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
1578 assert(dst->nb[0] == sizeof(float))(static_cast <bool> (dst->nb[0] == sizeof(float)) ? void
(0) : __assert_fail ("dst->nb[0] == sizeof(float)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
1579
1580 const int64_t ne00 = src0->ne[0];
1581 const int64_t ne01 = src0->ne[1];
1582
1583 const size_t nb01 = src0->nb[1];
1584 const size_t nb0 = dst->nb[0];
1585
1586 for (int64_t i1 = 0; i1 < ne01; i1++) {
1587 float * src = (float *) ((char *) src0->data + i1*nb01);
1588 int32_t * dst_ = (int32_t *) ((char *) dst->data + i1*nb0);
1589 int v = 0;
1590 ggml_vec_argmax_f32(ne00, &v, src);
1591 dst_[0] = v;
1592 }
1593}
1594
1595void ggml_compute_forward_argmax(
1596 const ggml_compute_params * params,
1597 ggml_tensor * dst) {
1598
1599 const ggml_tensor * src0 = dst->src[0];
1600
1601 switch (src0->type) {
1602 case GGML_TYPE_F32:
1603 {
1604 ggml_compute_forward_argmax_f32(params, dst);
1605 } break;
1606 default:
1607 {
1608 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1608, "fatal error");
1609 }
1610 }
1611}
1612
1613// ggml_compute_forward_count_equal
1614
1615static void ggml_compute_forward_count_equal_i32(
1616 const ggml_compute_params * params,
1617 ggml_tensor * dst) {
1618
1619 const ggml_tensor * src0 = dst->src[0];
1620 const ggml_tensor * src1 = dst->src[1];
1621
1622 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);;
1623
1624 GGML_ASSERT(src0->type == GGML_TYPE_I32)if (!(src0->type == GGML_TYPE_I32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1624, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_I32"
);
1625 GGML_ASSERT(src1->type == GGML_TYPE_I32)if (!(src1->type == GGML_TYPE_I32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1625, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_I32"
);
1626 GGML_ASSERT(ggml_are_same_shape(src0, src1))if (!(ggml_are_same_shape(src0, src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1626, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, src1)"
);
1627 GGML_ASSERT(ggml_is_scalar(dst))if (!(ggml_is_scalar(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1627, "GGML_ASSERT(%s) failed", "ggml_is_scalar(dst)");
1628 GGML_ASSERT(dst->type == GGML_TYPE_I64)if (!(dst->type == GGML_TYPE_I64)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1628, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_I64"
);
1629
1630 const int64_t nr = ggml_nrows(src0);
1631
1632 const int ith = params->ith;
1633 const int nth = params->nth;
1634
1635 int64_t * sums = (int64_t *) params->wdata;
1636 int64_t sum_thread = 0;
1637
1638 // rows per thread
1639 const int64_t dr = (nr + nth - 1)/nth;
1640
1641 // row range for this thread
1642 const int64_t ir0 = dr*ith;
1643 const int64_t ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
1644
1645 for (int64_t ir = ir0; ir < ir1; ++ir) {
1646 const int64_t i03 = ir / (ne02*ne01);
1647 const int64_t i02 = (ir - i03*ne03) / ne01;
1648 const int64_t i01 = ir - i03*ne03 - i02*ne02;
1649
1650 const char * data0 = (const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01;
1651 const char * data1 = (const char *) src1->data + i03*nb13 + i02*nb12 + i01*nb11;
1652
1653 for (int64_t i00 = 0; i00 < ne00; ++i00) {
1654 const int32_t val0 = *((const int32_t *) (data0 + i00*nb00));
1655 const int32_t val1 = *((const int32_t *) (data1 + i00*nb10));
1656
1657 sum_thread += val0 == val1;
1658 }
1659 }
1660 if (ith != 0) {
1661 sums[ith] = sum_thread;
1662 }
1663 ggml_barrier(params->threadpool);
1664
1665 if (ith != 0) {
1666 return;
1667 }
1668
1669 for (int ith_other = 1; ith_other < nth; ++ith_other) {
1670 sum_thread += sums[ith_other];
1671 }
1672 *((int64_t *) dst->data) = sum_thread;
1673}
1674
1675void ggml_compute_forward_count_equal(
1676 const ggml_compute_params * params,
1677 ggml_tensor * dst) {
1678
1679 const ggml_tensor * src0 = dst->src[0];
1680
1681 switch (src0->type) {
1682 case GGML_TYPE_I32:
1683 {
1684 ggml_compute_forward_count_equal_i32(params, dst);
1685 } break;
1686 default:
1687 {
1688 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1688, "fatal error");
1689 }
1690 }
1691}
1692
1693// ggml_compute_forward_repeat
1694
1695static void ggml_compute_forward_repeat_f32(
1696 const ggml_compute_params * params,
1697 ggml_tensor * dst) {
1698
1699 const ggml_tensor * src0 = dst->src[0];
1700
1701 if (params->ith != 0) {
1702 return;
1703 }
1704
1705 GGML_ASSERT(ggml_can_repeat(src0, dst))if (!(ggml_can_repeat(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1705, "GGML_ASSERT(%s) failed", "ggml_can_repeat(src0, dst)"
);
1706
1707 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
1708
1709 // guaranteed to be an integer due to the check in ggml_can_repeat
1710 const int nr0 = (int)(ne0/ne00);
1711 const int nr1 = (int)(ne1/ne01);
1712 const int nr2 = (int)(ne2/ne02);
1713 const int nr3 = (int)(ne3/ne03);
1714
1715 // TODO: support for transposed / permuted tensors
1716 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1716, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
1717 GGML_ASSERT(nb00 == sizeof(float))if (!(nb00 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1717, "GGML_ASSERT(%s) failed", "nb00 == sizeof(float)");
1718
1719 // TODO: maybe this is not optimal?
1720 for (int i3 = 0; i3 < nr3; i3++) {
1721 for (int k3 = 0; k3 < ne03; k3++) {
1722 for (int i2 = 0; i2 < nr2; i2++) {
1723 for (int k2 = 0; k2 < ne02; k2++) {
1724 for (int i1 = 0; i1 < nr1; i1++) {
1725 for (int k1 = 0; k1 < ne01; k1++) {
1726 for (int i0 = 0; i0 < nr0; i0++) {
1727 ggml_vec_cpy_f32(ne00,
1728 (float *) ((char *) dst->data + (i3*ne03 + k3)*nb3 + (i2*ne02 + k2)*nb2 + (i1*ne01 + k1)*nb1 + (i0*ne00)*nb0),
1729 (float *) ((char *) src0->data + ( k3)*nb03 + ( k2)*nb02 + ( k1)*nb01));
1730 }
1731 }
1732 }
1733 }
1734 }
1735 }
1736 }
1737}
1738
1739static void ggml_compute_forward_repeat_f16(
1740 const ggml_compute_params * params,
1741 ggml_tensor * dst) {
1742
1743 const ggml_tensor * src0 = dst->src[0];
1744
1745 if (params->ith != 0) {
1746 return;
1747 }
1748
1749 GGML_ASSERT(ggml_can_repeat(src0, dst))if (!(ggml_can_repeat(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1749, "GGML_ASSERT(%s) failed", "ggml_can_repeat(src0, dst)"
);
1750
1751 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
1752
1753 // guaranteed to be an integer due to the check in ggml_can_repeat
1754 const int nr0 = (int)(ne0/ne00);
1755 const int nr1 = (int)(ne1/ne01);
1756 const int nr2 = (int)(ne2/ne02);
1757 const int nr3 = (int)(ne3/ne03);
1758
1759 // TODO: support for transposed / permuted tensors
1760 GGML_ASSERT(nb0 == sizeof(ggml_fp16_t))if (!(nb0 == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1760, "GGML_ASSERT(%s) failed", "nb0 == sizeof(ggml_fp16_t)"
);
1761 GGML_ASSERT(nb00 == sizeof(ggml_fp16_t))if (!(nb00 == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1761, "GGML_ASSERT(%s) failed", "nb00 == sizeof(ggml_fp16_t)"
);
1762
1763 // TODO: maybe this is not optimal?
1764 for (int i3 = 0; i3 < nr3; i3++) {
1765 for (int k3 = 0; k3 < ne03; k3++) {
1766 for (int i2 = 0; i2 < nr2; i2++) {
1767 for (int k2 = 0; k2 < ne02; k2++) {
1768 for (int i1 = 0; i1 < nr1; i1++) {
1769 for (int k1 = 0; k1 < ne01; k1++) {
1770 for (int i0 = 0; i0 < nr0; i0++) {
1771 ggml_fp16_t * y = (ggml_fp16_t *) ((char *) dst->data + (i3*ne03 + k3)*nb3 + (i2*ne02 + k2)*nb2 + (i1*ne01 + k1)*nb1 + (i0*ne00)*nb0);
1772 ggml_fp16_t * x = (ggml_fp16_t *) ((char *) src0->data + ( k3)*nb03 + ( k2)*nb02 + ( k1)*nb01);
1773 // ggml_vec_cpy_f16(ne00, y, x)
1774 for (int i = 0; i < ne00; ++i) {
1775 y[i] = x[i];
1776 }
1777 }
1778 }
1779 }
1780 }
1781 }
1782 }
1783 }
1784}
1785
1786void ggml_compute_forward_repeat(
1787 const ggml_compute_params * params,
1788 ggml_tensor * dst) {
1789
1790 const ggml_tensor * src0 = dst->src[0];
1791
1792 switch (src0->type) {
1793 case GGML_TYPE_F16:
1794 case GGML_TYPE_BF16:
1795 case GGML_TYPE_I16:
1796 {
1797 ggml_compute_forward_repeat_f16(params, dst);
1798 } break;
1799 case GGML_TYPE_F32:
1800 case GGML_TYPE_I32:
1801 {
1802 ggml_compute_forward_repeat_f32(params, dst);
1803 } break;
1804 // TODO: templateify the implementation and support for I64
1805 // ref https://github.com/ggml-org/llama.cpp/pull/14274#discussion_r2169492225
1806 //case GGML_TYPE_I64:
1807 // {
1808 // ggml_compute_forward_repeat_i64(params, dst);
1809 // } break;
1810 default:
1811 {
1812 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1812, "fatal error");
1813 }
1814 }
1815}
1816
1817// ggml_compute_forward_repeat_back
1818
1819static void ggml_compute_forward_repeat_back_f32(
1820 const ggml_compute_params * params,
1821 ggml_tensor * dst) {
1822
1823 const ggml_tensor * src0 = dst->src[0];
1824
1825 if (params->ith != 0) {
1826 return;
1827 }
1828
1829 GGML_ASSERT(ggml_can_repeat(dst, src0))if (!(ggml_can_repeat(dst, src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1829, "GGML_ASSERT(%s) failed", "ggml_can_repeat(dst, src0)"
);
1830
1831 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
1832
1833 // guaranteed to be an integer due to the check in ggml_can_repeat
1834 const int nr0 = (int)(ne00/ne0);
1835 const int nr1 = (int)(ne01/ne1);
1836 const int nr2 = (int)(ne02/ne2);
1837 const int nr3 = (int)(ne03/ne3);
1838
1839 // TODO: support for transposed / permuted tensors
1840 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1840, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
1841 GGML_ASSERT(nb00 == sizeof(float))if (!(nb00 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1841, "GGML_ASSERT(%s) failed", "nb00 == sizeof(float)");
1842
1843 if (ggml_is_contiguous(dst)) {
1844 ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0);
1845 } else {
1846 for (int k3 = 0; k3 < ne3; k3++) {
1847 for (int k2 = 0; k2 < ne2; k2++) {
1848 for (int k1 = 0; k1 < ne1; k1++) {
1849 ggml_vec_set_f32(ne0,
1850 (float *) ((char *) dst->data + k1*nb1 + k2*nb2 + k3*nb3),
1851 0);
1852 }
1853 }
1854 }
1855 }
1856
1857 // TODO: maybe this is not optimal?
1858 for (int i3 = 0; i3 < nr3; i3++) {
1859 for (int k3 = 0; k3 < ne3; k3++) {
1860 for (int i2 = 0; i2 < nr2; i2++) {
1861 for (int k2 = 0; k2 < ne2; k2++) {
1862 for (int i1 = 0; i1 < nr1; i1++) {
1863 for (int k1 = 0; k1 < ne1; k1++) {
1864 for (int i0 = 0; i0 < nr0; i0++) {
1865 ggml_vec_acc_f32(ne0,
1866 (float *) ((char *) dst->data + ( k3)*nb3 + ( k2)*nb2 + ( k1)*nb1),
1867 (float *) ((char *) src0->data + (i3*ne3 + k3)*nb03 + (i2*ne2 + k2)*nb02 + (i1*ne1 + k1)*nb01 + (i0*ne0)*nb00));
1868 }
1869 }
1870 }
1871 }
1872 }
1873 }
1874 }
1875}
1876
1877void ggml_compute_forward_repeat_back(
1878 const ggml_compute_params * params,
1879 ggml_tensor * dst) {
1880
1881 const ggml_tensor * src0 = dst->src[0];
1882
1883 switch (src0->type) {
1884 case GGML_TYPE_F32:
1885 {
1886 ggml_compute_forward_repeat_back_f32(params, dst);
1887 } break;
1888 default:
1889 {
1890 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1890, "fatal error");
1891 }
1892 }
1893}
1894
1895// ggml_compute_forward_concat
1896
1897static void ggml_compute_forward_concat_any(
1898 const ggml_compute_params * params,
1899 ggml_tensor * dst) {
1900
1901 const ggml_tensor * src0 = dst->src[0];
1902 const ggml_tensor * src1 = dst->src[1];
1903
1904 const size_t len = ggml_type_size(src0->type);
1905
1906 const int ith = params->ith;
1907 const int nth = params->nth;
1908
1909 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
1910
1911 const int32_t dim = ggml_get_op_params_i32(dst, 0);
1912
1913 GGML_ASSERT(dim >= 0 && dim < 4)if (!(dim >= 0 && dim < 4)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1913, "GGML_ASSERT(%s) failed", "dim >= 0 && dim < 4"
);
1914
1915 int64_t o[4] = {0, 0, 0, 0};
1916 o[dim] = src0->ne[dim];
1917
1918 const char * x;
1919
1920 // TODO: smarter multi-theading
1921 for (int i3 = 0; i3 < ne3; i3++) {
1922 for (int i2 = ith; i2 < ne2; i2 += nth) {
1923 for (int i1 = 0; i1 < ne1; i1++) {
1924 for (int i0 = 0; i0 < ne0; i0++) {
1925 if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
1926 x = (const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03;
1927 } else {
1928 x = (const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13;
1929 }
1930
1931 char * y = (char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3;
1932
1933 memcpy(y, x, len);
1934 }
1935 }
1936 }
1937 }
1938}
1939
1940static void ggml_compute_forward_concat_i8(
1941 const ggml_compute_params * params,
1942 ggml_tensor * dst) {
1943
1944 const ggml_tensor * src0 = dst->src[0];
1945 const ggml_tensor * src1 = dst->src[1];
1946
1947 GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t))if (!(ggml_type_size(src0->type) == sizeof(int8_t))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1947, "GGML_ASSERT(%s) failed", "ggml_type_size(src0->type) == sizeof(int8_t)"
);
1948
1949 const int ith = params->ith;
1950 const int nth = params->nth;
1951
1952 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
1953
1954 const int32_t dim = ggml_get_op_params_i32(dst, 0);
1955
1956 GGML_ASSERT(dim >= 0 && dim < 4)if (!(dim >= 0 && dim < 4)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1956, "GGML_ASSERT(%s) failed", "dim >= 0 && dim < 4"
);
1957
1958 int64_t o[4] = {0, 0, 0, 0};
1959 o[dim] = src0->ne[dim];
1960
1961 const int8_t * x;
1962
1963 // TODO: smarter multi-theading
1964 for (int i3 = 0; i3 < ne3; i3++) {
1965 for (int i2 = ith; i2 < ne2; i2 += nth) {
1966 for (int i1 = 0; i1 < ne1; i1++) {
1967 for (int i0 = 0; i0 < ne0; i0++) {
1968 if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
1969 x = (const int8_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
1970 } else {
1971 x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
1972 }
1973
1974 int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
1975
1976 *y = *x;
1977 }
1978 }
1979 }
1980 }
1981}
1982
1983static void ggml_compute_forward_concat_f16(
1984 const ggml_compute_params * params,
1985 ggml_tensor * dst) {
1986
1987 const ggml_tensor * src0 = dst->src[0];
1988 const ggml_tensor * src1 = dst->src[1];
1989
1990 GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t))if (!(ggml_type_size(src0->type) == sizeof(ggml_fp16_t))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1990, "GGML_ASSERT(%s) failed", "ggml_type_size(src0->type) == sizeof(ggml_fp16_t)"
);
1991
1992 const int ith = params->ith;
1993 const int nth = params->nth;
1994
1995 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
1996
1997 const int32_t dim = ggml_get_op_params_i32(dst, 0);
1998
1999 GGML_ASSERT(dim >= 0 && dim < 4)if (!(dim >= 0 && dim < 4)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 1999, "GGML_ASSERT(%s) failed", "dim >= 0 && dim < 4"
);
2000
2001 int64_t o[4] = {0, 0, 0, 0};
2002 o[dim] = src0->ne[dim];
2003
2004 const ggml_fp16_t * x;
2005
2006 // TODO: smarter multi-theading
2007 for (int i3 = 0; i3 < ne3; i3++) {
2008 for (int i2 = ith; i2 < ne2; i2 += nth) {
2009 for (int i1 = 0; i1 < ne1; i1++) {
2010 for (int i0 = 0; i0 < ne0; i0++) {
2011 if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
2012 x = (const ggml_fp16_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
2013 } else {
2014 x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
2015 }
2016
2017 ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
2018
2019 *y = *x;
2020 }
2021 }
2022 }
2023 }
2024}
2025
2026static void ggml_compute_forward_concat_f32(
2027 const ggml_compute_params * params,
2028 ggml_tensor * dst) {
2029
2030 const ggml_tensor * src0 = dst->src[0];
2031 const ggml_tensor * src1 = dst->src[1];
2032
2033 GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float))if (!(ggml_type_size(src0->type) == sizeof(float))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2033, "GGML_ASSERT(%s) failed", "ggml_type_size(src0->type) == sizeof(float)"
);
2034
2035 const int ith = params->ith;
2036 const int nth = params->nth;
2037
2038 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
2039
2040 const int32_t dim = ggml_get_op_params_i32(dst, 0);
2041
2042 GGML_ASSERT(dim >= 0 && dim < 4)if (!(dim >= 0 && dim < 4)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2042, "GGML_ASSERT(%s) failed", "dim >= 0 && dim < 4"
);
2043
2044 int64_t o[4] = {0, 0, 0, 0};
2045 o[dim] = src0->ne[dim];
2046
2047 const float * x;
2048
2049 // TODO: smarter multi-theading
2050 for (int i3 = 0; i3 < ne3; i3++) {
2051 for (int i2 = ith; i2 < ne2; i2 += nth) {
2052 for (int i1 = 0; i1 < ne1; i1++) {
2053 for (int i0 = 0; i0 < ne0; i0++) {
2054 if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
2055 x = (const float *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
2056 } else {
2057 x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
2058 }
2059
2060 float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
2061
2062 *y = *x;
2063 }
2064 }
2065 }
2066 }
2067}
2068
2069void ggml_compute_forward_concat(
2070 const ggml_compute_params * params,
2071 ggml_tensor * dst) {
2072
2073 const ggml_tensor * src0 = dst->src[0];
2074
2075 switch (src0->type) {
2076 case GGML_TYPE_F16:
2077 case GGML_TYPE_BF16:
2078 case GGML_TYPE_I16:
2079 {
2080 ggml_compute_forward_concat_f16(params, dst);
2081 } break;
2082 case GGML_TYPE_I8:
2083 {
2084 ggml_compute_forward_concat_i8(params, dst);
2085 } break;
2086 case GGML_TYPE_F32:
2087 case GGML_TYPE_I32:
2088 {
2089 ggml_compute_forward_concat_f32(params, dst);
2090 } break;
2091 default:
2092 {
2093 ggml_compute_forward_concat_any(params, dst);
2094 }
2095 }
2096}
2097
2098// ggml_compute_forward_gelu
2099
2100static void ggml_compute_forward_gelu_f32(
2101 const ggml_compute_params * params,
2102 ggml_tensor * dst) {
2103
2104 const ggml_tensor * src0 = dst->src[0];
2105
2106 assert(ggml_is_contiguous_rows(src0))(static_cast <bool> (ggml_is_contiguous_rows(src0)) ? void
(0) : __assert_fail ("ggml_is_contiguous_rows(src0)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2107 assert(ggml_are_same_shape(src0, dst))(static_cast <bool> (ggml_are_same_shape(src0, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src0, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2108
2109 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
2110 GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)const size_t nb00 = (src0) ? (src0)->nb[0] : 0; (void)(nb00
); const size_t nb01 = (src0) ? (src0)->nb[1] : 0; (void)(
nb01); const size_t nb02 = (src0) ? (src0)->nb[2] : 0; (void
)(nb02); const size_t nb03 = (src0) ? (src0)->nb[3] : 0; (
void)(nb03);
2111 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
2112 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
2113
2114 const int ith = params->ith;
2115 const int nth = params->nth;
2116
2117 const int nc = src0->ne[0];
2118 const int nr = ggml_nrows(src0);
2119
2120 // rows per thread
2121 const int dr = (nr + nth - 1)/nth;
2122
2123 // row range for this thread
2124 const int ir0 = dr*ith;
2125 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2126
2127 for (int ir = ir0; ir < ir1; ++ir) {
2128 const int i3 = ir/(ne02*ne01);
2129 const int i2 = (ir - i3*ne02*ne01)/ne01;
2130 const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
2131
2132 ggml_vec_gelu_f32(nc,
2133 (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
2134 (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
2135
2136#ifndef NDEBUG
2137 for (int k = 0; k < nc; k++) {
2138 const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];
2139 GGML_UNUSED(x)(void)(x);
2140 assert(!isnan(x))(static_cast <bool> (!isnan(x)) ? void (0) : __assert_fail
("!isnan(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2141 assert(!isinf(x))(static_cast <bool> (!isinf(x)) ? void (0) : __assert_fail
("!isinf(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2142 }
2143#endif // NDEBUG
2144 }
2145}
2146
2147static void ggml_compute_forward_gelu_f16(
2148 const ggml_compute_params * params,
2149 ggml_tensor * dst) {
2150
2151 const ggml_tensor * src0 = dst->src[0];
2152
2153 assert(ggml_is_contiguous_rows(src0))(static_cast <bool> (ggml_is_contiguous_rows(src0)) ? void
(0) : __assert_fail ("ggml_is_contiguous_rows(src0)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2154 assert(ggml_are_same_shape(src0, dst))(static_cast <bool> (ggml_are_same_shape(src0, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src0, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2155
2156 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
2157 GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)const size_t nb00 = (src0) ? (src0)->nb[0] : 0; (void)(nb00
); const size_t nb01 = (src0) ? (src0)->nb[1] : 0; (void)(
nb01); const size_t nb02 = (src0) ? (src0)->nb[2] : 0; (void
)(nb02); const size_t nb03 = (src0) ? (src0)->nb[3] : 0; (
void)(nb03);
2158 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
2159 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
2160
2161 const int ith = params->ith;
2162 const int nth = params->nth;
2163
2164 const int nc = src0->ne[0];
2165 const int nr = ggml_nrows(src0);
2166
2167 // rows per thread
2168 const int dr = (nr + nth - 1)/nth;
2169
2170 // row range for this thread
2171 const int ir0 = dr*ith;
2172 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2173
2174 for (int ir = ir0; ir < ir1; ++ir) {
2175 const int i3 = ir/(ne02*ne01);
2176 const int i2 = (ir - i3*ne02*ne01)/ne01;
2177 const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
2178
2179 ggml_vec_gelu_f16(nc,
2180 (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
2181 (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
2182
2183#ifndef NDEBUG
2184 for (int k = 0; k < nc; k++) {
2185 const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];
2186 const float v = GGML_CPU_FP16_TO_FP32(x)ggml_lookup_fp16_to_fp32(x);
2187 GGML_UNUSED(v)(void)(v);
2188 assert(!isnan(v))(static_cast <bool> (!isnan(v)) ? void (0) : __assert_fail
("!isnan(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2189 assert(!isinf(v))(static_cast <bool> (!isinf(v)) ? void (0) : __assert_fail
("!isinf(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2190 }
2191#endif // NDEBUG
2192 }
2193}
2194
2195static void ggml_compute_forward_gelu(
2196 const ggml_compute_params * params,
2197 ggml_tensor * dst) {
2198
2199 const ggml_tensor * src0 = dst->src[0];
2200
2201 switch (src0->type) {
2202 case GGML_TYPE_F32:
2203 {
2204 ggml_compute_forward_gelu_f32(params, dst);
2205 } break;
2206 case GGML_TYPE_F16:
2207 {
2208 ggml_compute_forward_gelu_f16(params, dst);
2209 } break;
2210 default:
2211 {
2212 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2212, "fatal error");
2213 }
2214 }
2215}
2216
2217// ggml_compute_fill
2218
2219static void ggml_compute_forward_fill_f32(const ggml_compute_params * params, ggml_tensor * dst) {
2220 const float c = ggml_get_op_params_f32(dst, 0);
2221
2222 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);;
2223 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);;
2224
2225 const auto [ir0, ir1] = get_thread_range(params, dst);
2226
2227 for (int64_t ir = ir0; ir < ir1; ++ir) {
2228 const int64_t i03 = ir/(ne2*ne1);
2229 const int64_t i02 = (ir - i03*ne2*ne1)/ne1;
2230 const int64_t i01 = (ir - i03*ne2*ne1 - i02*ne1);
2231
2232 float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);
2233
2234 ggml_vec_set_f32(ne0, dst_ptr, c);
2235 }
2236}
2237
2238static void ggml_compute_forward_fill_f16(const ggml_compute_params * params, ggml_tensor * dst) {
2239 const ggml_fp16_t c = GGML_CPU_FP32_TO_FP16(ggml_get_op_params_f32(dst, 0))ggml_compute_fp32_to_fp16(ggml_get_op_params_f32(dst, 0));
2240
2241 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);;
2242 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);;
2243
2244 const auto [ir0, ir1] = get_thread_range(params, dst);
2245
2246 for (int64_t ir = ir0; ir < ir1; ++ir) {
2247 const int64_t i03 = ir/(ne2*ne1);
2248 const int64_t i02 = (ir - i03*ne2*ne1)/ne1;
2249 const int64_t i01 = (ir - i03*ne2*ne1 - i02*ne1);
2250
2251 ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);
2252
2253 ggml_vec_set_f16(ne0, dst_ptr, c);
2254 }
2255}
2256
2257void ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor * dst) {
2258 const ggml_tensor * src0 = dst->src[0];
2259
2260 switch (src0->type) {
2261 case GGML_TYPE_F32:
2262 {
2263 ggml_compute_forward_fill_f32(params, dst);
2264 } break;
2265 case GGML_TYPE_F16:
2266 {
2267 ggml_compute_forward_fill_f16(params, dst);
2268 } break;
2269 default:
2270 {
2271 GGML_ABORT("unsupported type for ggml_compute_forward_fill: %s", ggml_type_name(src0->type))ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2271, "unsupported type for ggml_compute_forward_fill: %s",
ggml_type_name(src0->type));
2272 }
2273 }
2274}
2275
2276// ggml_compute_tri
2277
2278static void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggml_tensor * dst) {
2279 const ggml_tensor * src0 = dst->src[0];
2280
2281 const ggml_tri_type ttype = (ggml_tri_type) ggml_get_op_params_i32(dst, 0);
2282
2283 GGML_ASSERT(ggml_is_contiguous(src0))if (!(ggml_is_contiguous(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2283, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(src0)");
2284
2285 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
2286
2287 const auto [ir0, ir1] = get_thread_range(params, src0);
2288
2289 bool (*bipred)(int, int);
2290
2291 switch (ttype) {
2292 case GGML_TRI_TYPE_LOWER: bipred = [](int i, int r) { return i < r; }; break;
2293 case GGML_TRI_TYPE_LOWER_DIAG: bipred = [](int i, int r) { return i <= r; }; break;
2294 case GGML_TRI_TYPE_UPPER: bipred = [](int i, int r) { return i > r; }; break;
2295 case GGML_TRI_TYPE_UPPER_DIAG: bipred = [](int i, int r) { return i >= r; }; break;
2296 default: GGML_ABORT("invalid tri type")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2296, "invalid tri type");
2297 }
2298
2299 for (int64_t ir = ir0; ir < ir1; ++ir) {
2300 const int64_t i03 = ir/(ne02*ne01);
2301 const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
2302 const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
2303
2304 const float * src_ptr = (const float *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
2305 float * dst_ptr = ( float *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);
2306
2307 for (int i0 = 0; i0 < ne0; ++i0) {
2308 dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : 0.0f;
2309 }
2310 }
2311}
2312
2313void ggml_compute_forward_tri(const ggml_compute_params * params, ggml_tensor * dst) {
2314 const ggml_tensor * src0 = dst->src[0];
2315
2316 switch (src0->type) {
2317 case GGML_TYPE_F32:
2318 {
2319 ggml_compute_forward_tri_f32(params, dst);
2320 } break;
2321 default:
2322 {
2323 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2323, "fatal error");
2324 }
2325 }
2326}
2327
2328// ggml_compute_forward_gelu_erf
2329
2330static void ggml_compute_forward_gelu_erf_f32(
2331 const ggml_compute_params * params,
2332 ggml_tensor * dst) {
2333
2334 const ggml_tensor * src0 = dst->src[0];
2335
2336 assert(ggml_is_contiguous_rows(src0))(static_cast <bool> (ggml_is_contiguous_rows(src0)) ? void
(0) : __assert_fail ("ggml_is_contiguous_rows(src0)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2337 assert(ggml_are_same_shape(src0, dst))(static_cast <bool> (ggml_are_same_shape(src0, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src0, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2338
2339 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
2340 GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)const size_t nb00 = (src0) ? (src0)->nb[0] : 0; (void)(nb00
); const size_t nb01 = (src0) ? (src0)->nb[1] : 0; (void)(
nb01); const size_t nb02 = (src0) ? (src0)->nb[2] : 0; (void
)(nb02); const size_t nb03 = (src0) ? (src0)->nb[3] : 0; (
void)(nb03);
2341 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
2342 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
2343
2344 const int ith = params->ith;
2345 const int nth = params->nth;
2346
2347 const int nc = src0->ne[0];
2348 const int nr = ggml_nrows(src0);
2349
2350 // rows per thread
2351 const int dr = (nr + nth - 1)/nth;
2352
2353 // row range for this thread
2354 const int ir0 = dr*ith;
2355 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2356
2357 for (int ir = ir0; ir < ir1; ++ir) {
2358 const int i3 = ir/(ne02*ne01);
2359 const int i2 = (ir - i3*ne02*ne01)/ne01;
2360 const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
2361
2362 ggml_vec_gelu_erf_f32(nc,
2363 (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
2364 (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
2365
2366#ifndef NDEBUG
2367 for (int k = 0; k < nc; k++) {
2368 const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];
2369 GGML_UNUSED(x)(void)(x);
2370 assert(!isnan(x))(static_cast <bool> (!isnan(x)) ? void (0) : __assert_fail
("!isnan(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2371 assert(!isinf(x))(static_cast <bool> (!isinf(x)) ? void (0) : __assert_fail
("!isinf(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2372 }
2373#endif // NDEBUG
2374 }
2375}
2376
2377static void ggml_compute_forward_gelu_erf_f16(
2378 const ggml_compute_params * params,
2379 ggml_tensor * dst) {
2380
2381 const ggml_tensor * src0 = dst->src[0];
2382
2383 assert(ggml_is_contiguous_rows(src0))(static_cast <bool> (ggml_is_contiguous_rows(src0)) ? void
(0) : __assert_fail ("ggml_is_contiguous_rows(src0)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2384 assert(ggml_are_same_shape(src0, dst))(static_cast <bool> (ggml_are_same_shape(src0, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src0, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2385
2386 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
2387 GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)const size_t nb00 = (src0) ? (src0)->nb[0] : 0; (void)(nb00
); const size_t nb01 = (src0) ? (src0)->nb[1] : 0; (void)(
nb01); const size_t nb02 = (src0) ? (src0)->nb[2] : 0; (void
)(nb02); const size_t nb03 = (src0) ? (src0)->nb[3] : 0; (
void)(nb03);
2388 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
2389 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
2390
2391 const int ith = params->ith;
2392 const int nth = params->nth;
2393
2394 const int nc = src0->ne[0];
2395 const int nr = ggml_nrows(src0);
2396
2397 // rows per thread
2398 const int dr = (nr + nth - 1)/nth;
2399
2400 // row range for this thread
2401 const int ir0 = dr*ith;
2402 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2403
2404 for (int ir = ir0; ir < ir1; ++ir) {
2405 const int i3 = ir/(ne02*ne01);
2406 const int i2 = (ir - i3*ne02*ne01)/ne01;
2407 const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
2408
2409 ggml_vec_gelu_erf_f16(nc,
2410 (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
2411 (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
2412
2413#ifndef NDEBUG
2414 for (int k = 0; k < nc; k++) {
2415 const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];
2416 const float v = GGML_CPU_FP16_TO_FP32(x)ggml_lookup_fp16_to_fp32(x);
2417 GGML_UNUSED(v)(void)(v);
2418 assert(!isnan(v))(static_cast <bool> (!isnan(v)) ? void (0) : __assert_fail
("!isnan(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2419 assert(!isinf(v))(static_cast <bool> (!isinf(v)) ? void (0) : __assert_fail
("!isinf(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2420 }
2421#endif // NDEBUG
2422 }
2423}
2424
2425static void ggml_compute_forward_gelu_erf(
2426 const ggml_compute_params * params,
2427 ggml_tensor * dst) {
2428
2429 const ggml_tensor * src0 = dst->src[0];
2430
2431 switch (src0->type) {
2432 case GGML_TYPE_F32:
2433 {
2434 ggml_compute_forward_gelu_erf_f32(params, dst);
2435 } break;
2436 case GGML_TYPE_F16:
2437 {
2438 ggml_compute_forward_gelu_erf_f16(params, dst);
2439 } break;
2440 default:
2441 {
2442 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2442, "fatal error");
2443 }
2444 }
2445}
2446
2447// ggml_compute_forward_gelu_quick
2448
2449static void ggml_compute_forward_gelu_quick_f32(
2450 const ggml_compute_params * params,
2451 ggml_tensor * dst) {
2452
2453 const ggml_tensor * src0 = dst->src[0];
2454
2455 assert(ggml_is_contiguous_rows(src0))(static_cast <bool> (ggml_is_contiguous_rows(src0)) ? void
(0) : __assert_fail ("ggml_is_contiguous_rows(src0)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2456 assert(ggml_are_same_shape(src0, dst))(static_cast <bool> (ggml_are_same_shape(src0, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src0, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2457
2458 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
2459 GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)const size_t nb00 = (src0) ? (src0)->nb[0] : 0; (void)(nb00
); const size_t nb01 = (src0) ? (src0)->nb[1] : 0; (void)(
nb01); const size_t nb02 = (src0) ? (src0)->nb[2] : 0; (void
)(nb02); const size_t nb03 = (src0) ? (src0)->nb[3] : 0; (
void)(nb03);
2460 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
2461 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
2462
2463 const int ith = params->ith;
2464 const int nth = params->nth;
2465
2466 const int nc = src0->ne[0];
2467 const int nr = ggml_nrows(src0);
2468
2469 // rows per thread
2470 const int dr = (nr + nth - 1)/nth;
2471
2472 // row range for this thread
2473 const int ir0 = dr*ith;
2474 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2475
2476 for (int ir = ir0; ir < ir1; ++ir) {
2477 const int i3 = ir/(ne02*ne01);
2478 const int i2 = (ir - i3*ne02*ne01)/ne01;
2479 const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
2480
2481 ggml_vec_gelu_quick_f32(nc,
2482 (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
2483 (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
2484
2485#ifndef NDEBUG
2486 for (int k = 0; k < nc; k++) {
2487 const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];
2488 GGML_UNUSED(x)(void)(x);
2489 assert(!isnan(x))(static_cast <bool> (!isnan(x)) ? void (0) : __assert_fail
("!isnan(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2490 assert(!isinf(x))(static_cast <bool> (!isinf(x)) ? void (0) : __assert_fail
("!isinf(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2491 }
2492#endif // NDEBUG
2493 }
2494}
2495
2496static void ggml_compute_forward_gelu_quick_f16(
2497 const ggml_compute_params * params,
2498 ggml_tensor * dst) {
2499
2500 const ggml_tensor * src0 = dst->src[0];
2501
2502 assert(ggml_is_contiguous_rows(src0))(static_cast <bool> (ggml_is_contiguous_rows(src0)) ? void
(0) : __assert_fail ("ggml_is_contiguous_rows(src0)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2503 assert(ggml_are_same_shape(src0, dst))(static_cast <bool> (ggml_are_same_shape(src0, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src0, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2504
2505 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
2506 GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)const size_t nb00 = (src0) ? (src0)->nb[0] : 0; (void)(nb00
); const size_t nb01 = (src0) ? (src0)->nb[1] : 0; (void)(
nb01); const size_t nb02 = (src0) ? (src0)->nb[2] : 0; (void
)(nb02); const size_t nb03 = (src0) ? (src0)->nb[3] : 0; (
void)(nb03);
2507 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
2508 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
2509
2510 const int ith = params->ith;
2511 const int nth = params->nth;
2512
2513 const int nc = src0->ne[0];
2514 const int nr = ggml_nrows(src0);
2515
2516 // rows per thread
2517 const int dr = (nr + nth - 1)/nth;
2518
2519 // row range for this thread
2520 const int ir0 = dr*ith;
2521 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2522
2523 for (int ir = ir0; ir < ir1; ++ir) {
2524 const int i3 = ir/(ne02*ne01);
2525 const int i2 = (ir - i3*ne02*ne01)/ne01;
2526 const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
2527
2528 ggml_vec_gelu_quick_f16(nc,
2529 (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
2530 (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
2531
2532#ifndef NDEBUG
2533 for (int k = 0; k < nc; k++) {
2534 const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];
2535 const float v = GGML_CPU_FP16_TO_FP32(x)ggml_lookup_fp16_to_fp32(x);
2536 GGML_UNUSED(v)(void)(v);
2537 assert(!isnan(v))(static_cast <bool> (!isnan(v)) ? void (0) : __assert_fail
("!isnan(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2538 assert(!isinf(v))(static_cast <bool> (!isinf(v)) ? void (0) : __assert_fail
("!isinf(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2539 }
2540#endif // NDEBUG
2541 }
2542}
2543
2544static void ggml_compute_forward_gelu_quick(
2545 const ggml_compute_params * params,
2546 ggml_tensor * dst) {
2547
2548 const ggml_tensor * src0 = dst->src[0];
2549
2550 switch (src0->type) {
2551 case GGML_TYPE_F32:
2552 {
2553 ggml_compute_forward_gelu_quick_f32(params, dst);
2554 } break;
2555 case GGML_TYPE_F16:
2556 {
2557 ggml_compute_forward_gelu_quick_f16(params, dst);
2558 } break;
2559 default:
2560 {
2561 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2561, "fatal error");
2562 }
2563 }
2564}
2565
2566// ggml_compute_forward_silu
2567
2568static void ggml_compute_forward_silu_f32(
2569 const ggml_compute_params * params,
2570 ggml_tensor * dst) {
2571
2572 const ggml_tensor * src0 = dst->src[0];
2573
2574 assert(ggml_is_contiguous_rows(src0))(static_cast <bool> (ggml_is_contiguous_rows(src0)) ? void
(0) : __assert_fail ("ggml_is_contiguous_rows(src0)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2575 assert(ggml_are_same_shape(src0, dst))(static_cast <bool> (ggml_are_same_shape(src0, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src0, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2576
2577 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
2578 GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)const size_t nb00 = (src0) ? (src0)->nb[0] : 0; (void)(nb00
); const size_t nb01 = (src0) ? (src0)->nb[1] : 0; (void)(
nb01); const size_t nb02 = (src0) ? (src0)->nb[2] : 0; (void
)(nb02); const size_t nb03 = (src0) ? (src0)->nb[3] : 0; (
void)(nb03);
2579 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
2580 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
2581
2582 const int ith = params->ith;
2583 const int nth = params->nth;
2584
2585 const int nc = src0->ne[0];
2586 const int nr = ggml_nrows(src0);
2587
2588 // rows per thread
2589 const int dr = (nr + nth - 1)/nth;
2590
2591 // row range for this thread
2592 const int ir0 = dr*ith;
2593 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2594
2595 for (int ir = ir0; ir < ir1; ++ir) {
2596 const int i3 = ir/(ne02*ne01);
2597 const int i2 = (ir - i3*ne02*ne01)/ne01;
2598 const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
2599
2600 ggml_vec_silu_f32(nc,
2601 (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
2602 (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
2603
2604#ifndef NDEBUG
2605 for (int k = 0; k < nc; k++) {
2606 const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];
2607 GGML_UNUSED(x)(void)(x);
2608 assert(!isnan(x))(static_cast <bool> (!isnan(x)) ? void (0) : __assert_fail
("!isnan(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2609 assert(!isinf(x))(static_cast <bool> (!isinf(x)) ? void (0) : __assert_fail
("!isinf(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2610 }
2611#endif // NDEBUG
2612 }
2613}
2614
2615static void ggml_compute_forward_silu_f16(
2616 const ggml_compute_params * params,
2617 ggml_tensor * dst) {
2618
2619 const ggml_tensor * src0 = dst->src[0];
2620
2621 assert(ggml_is_contiguous_rows(src0))(static_cast <bool> (ggml_is_contiguous_rows(src0)) ? void
(0) : __assert_fail ("ggml_is_contiguous_rows(src0)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2622 assert(ggml_are_same_shape(src0, dst))(static_cast <bool> (ggml_are_same_shape(src0, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src0, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2623
2624 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
2625 GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)const size_t nb00 = (src0) ? (src0)->nb[0] : 0; (void)(nb00
); const size_t nb01 = (src0) ? (src0)->nb[1] : 0; (void)(
nb01); const size_t nb02 = (src0) ? (src0)->nb[2] : 0; (void
)(nb02); const size_t nb03 = (src0) ? (src0)->nb[3] : 0; (
void)(nb03);
2626 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
2627 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
2628
2629 const int ith = params->ith;
2630 const int nth = params->nth;
2631
2632 const int nc = src0->ne[0];
2633 const int nr = ggml_nrows(src0);
2634
2635 // rows per thread
2636 const int dr = (nr + nth - 1)/nth;
2637
2638 // row range for this thread
2639 const int ir0 = dr*ith;
2640 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2641
2642 for (int ir = ir0; ir < ir1; ++ir) {
2643 const int i3 = ir/(ne02*ne01);
2644 const int i2 = (ir - i3*ne02*ne01)/ne01;
2645 const int i1 = (ir - i3*ne02*ne01 - i2*ne01);
2646
2647 ggml_vec_silu_f16(nc,
2648 (ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1),
2649 (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
2650
2651#ifndef NDEBUG
2652 for (int k = 0; k < nc; k++) {
2653 const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];
2654 const float v = GGML_CPU_FP16_TO_FP32(x)ggml_lookup_fp16_to_fp32(x);
2655 GGML_UNUSED(v)(void)(v);
2656 assert(!isnan(v))(static_cast <bool> (!isnan(v)) ? void (0) : __assert_fail
("!isnan(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2657 assert(!isinf(v))(static_cast <bool> (!isinf(v)) ? void (0) : __assert_fail
("!isinf(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2658 }
2659#endif // NDEBUG
2660 }
2661}
2662
2663static void ggml_compute_forward_silu(
2664 const ggml_compute_params * params,
2665 ggml_tensor * dst) {
2666
2667 const ggml_tensor * src0 = dst->src[0];
2668
2669 switch (src0->type) {
2670 case GGML_TYPE_F32:
2671 {
2672 ggml_compute_forward_silu_f32(params, dst);
2673 } break;
2674 case GGML_TYPE_F16:
2675 {
2676 ggml_compute_forward_silu_f16(params, dst);
2677 } break;
2678 default:
2679 {
2680 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2680, "fatal error");
2681 }
2682 }
2683}
2684// ggml_compute_forward_leaky_relu
2685
2686static void ggml_compute_forward_leaky_relu_f32(
2687 const ggml_compute_params * params,
2688 ggml_tensor * dst) {
2689
2690 const ggml_tensor * src0 = dst->src[0];
2691
2692 if (params->ith != 0) {
2693 return;
2694 }
2695
2696 assert(ggml_is_contiguous_1(src0))(static_cast <bool> (ggml_is_contiguous_1(src0)) ? void
(0) : __assert_fail ("ggml_is_contiguous_1(src0)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2697 assert(ggml_is_contiguous_1(dst))(static_cast <bool> (ggml_is_contiguous_1(dst)) ? void (
0) : __assert_fail ("ggml_is_contiguous_1(dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2698 assert(ggml_are_same_shape(src0, dst))(static_cast <bool> (ggml_are_same_shape(src0, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src0, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2699
2700 const int n = ggml_nrows(src0);
2701 const int nc = src0->ne[0];
2702
2703 float negative_slope;
2704 memcpy(&negative_slope, dst->op_params, sizeof(float));
2705
2706 assert(dst->nb[0] == sizeof(float))(static_cast <bool> (dst->nb[0] == sizeof(float)) ? void
(0) : __assert_fail ("dst->nb[0] == sizeof(float)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2707 assert(src0->nb[0] == sizeof(float))(static_cast <bool> (src0->nb[0] == sizeof(float)) ?
void (0) : __assert_fail ("src0->nb[0] == sizeof(float)",
__builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
2708
2709 for (int i = 0; i < n; i++) {
2710 ggml_vec_leaky_relu_f32(nc,
2711 (float *) ((char *) dst->data + i*( dst->nb[1])),
2712 (float *) ((char *) src0->data + i*(src0->nb[1])), negative_slope);
2713 }
2714}
2715
2716static void ggml_compute_forward_leaky_relu_f16(
2717 const ggml_compute_params * params,
2718 ggml_tensor * dst) {
2719
2720 const ggml_tensor * src0 = dst->src[0];
2721
2722 if (params->ith != 0) {
2723 return;
2724 }
2725
2726 assert(ggml_is_contiguous_1(src0))(static_cast <bool> (ggml_is_contiguous_1(src0)) ? void
(0) : __assert_fail ("ggml_is_contiguous_1(src0)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2727 assert(ggml_is_contiguous_1(dst))(static_cast <bool> (ggml_is_contiguous_1(dst)) ? void (
0) : __assert_fail ("ggml_is_contiguous_1(dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2728 assert(ggml_are_same_shape(src0, dst))(static_cast <bool> (ggml_are_same_shape(src0, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src0, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2729
2730 const int n = ggml_nrows(src0);
2731 const int nc = src0->ne[0];
2732
2733 float negative_slope;
2734 memcpy(&negative_slope, dst->op_params, sizeof(float));
2735
2736 assert(dst->nb[0] == sizeof(ggml_fp16_t))(static_cast <bool> (dst->nb[0] == sizeof(ggml_fp16_t
)) ? void (0) : __assert_fail ("dst->nb[0] == sizeof(ggml_fp16_t)"
, __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
2737 assert(src0->nb[0] == sizeof(ggml_fp16_t))(static_cast <bool> (src0->nb[0] == sizeof(ggml_fp16_t
)) ? void (0) : __assert_fail ("src0->nb[0] == sizeof(ggml_fp16_t)"
, __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
2738
2739 for (int i = 0; i < n; i++) {
2740 ggml_vec_leaky_relu_f16(nc,
2741 (ggml_fp16_t *) ((char *) dst->data + i*( dst->nb[1])),
2742 (ggml_fp16_t *) ((char *) src0->data + i*(src0->nb[1])), negative_slope);
2743 }
2744}
2745
2746void ggml_compute_forward_leaky_relu(
2747 const ggml_compute_params * params,
2748 ggml_tensor * dst) {
2749
2750 const ggml_tensor * src0 = dst->src[0];
2751
2752 switch (src0->type) {
2753 case GGML_TYPE_F32:
2754 {
2755 ggml_compute_forward_leaky_relu_f32(params, dst);
2756 } break;
2757 case GGML_TYPE_F16:
2758 {
2759 ggml_compute_forward_leaky_relu_f16(params, dst);
2760 } break;
2761 default:
2762 {
2763 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2763, "fatal error");
2764 }
2765 }
2766}
2767
2768// ggml_compute_forward_silu_back
2769
2770static void ggml_compute_forward_silu_back_f32(
2771 const ggml_compute_params * params,
2772 ggml_tensor * dst) {
2773
2774 const ggml_tensor * grad = dst->src[0];
2775 const ggml_tensor * src1 = dst->src[1];
2776
2777 assert(ggml_is_contiguous_1(grad))(static_cast <bool> (ggml_is_contiguous_1(grad)) ? void
(0) : __assert_fail ("ggml_is_contiguous_1(grad)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2778 assert(ggml_is_contiguous_1(src1))(static_cast <bool> (ggml_is_contiguous_1(src1)) ? void
(0) : __assert_fail ("ggml_is_contiguous_1(src1)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2779 assert(ggml_is_contiguous_1(dst))(static_cast <bool> (ggml_is_contiguous_1(dst)) ? void (
0) : __assert_fail ("ggml_is_contiguous_1(dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2780 assert(ggml_are_same_shape(src1, dst))(static_cast <bool> (ggml_are_same_shape(src1, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src1, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2781 assert(ggml_are_same_shape(src1, grad))(static_cast <bool> (ggml_are_same_shape(src1, grad)) ?
void (0) : __assert_fail ("ggml_are_same_shape(src1, grad)",
__builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
2782
2783 const int ith = params->ith;
2784 const int nth = params->nth;
2785
2786 const int nc = src1->ne[0];
2787 const int nr = ggml_nrows(src1);
2788
2789 // rows per thread
2790 const int dr = (nr + nth - 1)/nth;
2791
2792 // row range for this thread
2793 const int ir0 = dr*ith;
2794 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2795
2796 for (int i1 = ir0; i1 < ir1; i1++) {
2797 ggml_vec_silu_backward_f32(nc,
2798 (float *) ((char *) dst->data + i1*( dst->nb[1])),
2799 (float *) ((char *) src1->data + i1*(src1->nb[1])),
2800 (float *) ((char *) grad->data + i1*(grad->nb[1])));
2801
2802#ifndef NDEBUG
2803 for (int k = 0; k < nc; k++) {
2804 const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
2805 GGML_UNUSED(x)(void)(x);
2806 assert(!isnan(x))(static_cast <bool> (!isnan(x)) ? void (0) : __assert_fail
("!isnan(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2807 assert(!isinf(x))(static_cast <bool> (!isinf(x)) ? void (0) : __assert_fail
("!isinf(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2808 }
2809#endif // NDEBUG
2810 }
2811}
2812
2813static void ggml_compute_forward_silu_back_f16(
2814 const ggml_compute_params * params,
2815 ggml_tensor * dst) {
2816
2817 const ggml_tensor * grad = dst->src[0];
2818 const ggml_tensor * src1 = dst->src[1];
2819
2820 assert(ggml_is_contiguous_1(grad))(static_cast <bool> (ggml_is_contiguous_1(grad)) ? void
(0) : __assert_fail ("ggml_is_contiguous_1(grad)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2821 assert(ggml_is_contiguous_1(src1))(static_cast <bool> (ggml_is_contiguous_1(src1)) ? void
(0) : __assert_fail ("ggml_is_contiguous_1(src1)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2822 assert(ggml_is_contiguous_1(dst))(static_cast <bool> (ggml_is_contiguous_1(dst)) ? void (
0) : __assert_fail ("ggml_is_contiguous_1(dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2823 assert(ggml_are_same_shape(src1, dst))(static_cast <bool> (ggml_are_same_shape(src1, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src1, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2824 assert(ggml_are_same_shape(src1, grad))(static_cast <bool> (ggml_are_same_shape(src1, grad)) ?
void (0) : __assert_fail ("ggml_are_same_shape(src1, grad)",
__builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
2825
2826 const int ith = params->ith;
2827 const int nth = params->nth;
2828
2829 const int nc = src1->ne[0];
2830 const int nr = ggml_nrows(src1);
2831
2832 // rows per thread
2833 const int dr = (nr + nth - 1)/nth;
2834
2835 // row range for this thread
2836 const int ir0 = dr*ith;
2837 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2838
2839 for (int i1 = ir0; i1 < ir1; i1++) {
2840 ggml_vec_silu_backward_f16(nc,
2841 (ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])),
2842 (ggml_fp16_t *) ((char *) src1->data + i1*(src1->nb[1])),
2843 (ggml_fp16_t *) ((char *) grad->data + i1*(grad->nb[1])));
2844
2845#ifndef NDEBUG
2846 for (int k = 0; k < nc; k++) {
2847 const float x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];
2848 const float v = GGML_CPU_FP16_TO_FP32(x)ggml_lookup_fp16_to_fp32(x);
2849 GGML_UNUSED(v)(void)(v);
2850 assert(!isnan(v))(static_cast <bool> (!isnan(v)) ? void (0) : __assert_fail
("!isnan(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2851 assert(!isinf(v))(static_cast <bool> (!isinf(v)) ? void (0) : __assert_fail
("!isinf(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2852 }
2853#endif // NDEBUG
2854 }
2855}
2856
2857void ggml_compute_forward_silu_back(
2858 const ggml_compute_params * params,
2859 ggml_tensor * dst) {
2860
2861 const ggml_tensor * src0 = dst->src[0];
2862
2863 switch (src0->type) {
2864 case GGML_TYPE_F32:
2865 {
2866 ggml_compute_forward_silu_back_f32(params, dst);
2867 } break;
2868 case GGML_TYPE_F16:
2869 {
2870 ggml_compute_forward_silu_back_f16(params, dst);
2871 } break;
2872 default:
2873 {
2874 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2874, "fatal error");
2875 }
2876 }
2877}
2878
2879// ggml_compute_forward_reglu
2880
2881static void ggml_compute_forward_reglu_f32(
2882 const ggml_compute_params * params,
2883 ggml_tensor * dst) {
2884
2885 const ggml_tensor * src0 = dst->src[0];
2886 const ggml_tensor * src1 = dst->src[1];
2887 char * src0_d = (char *) src0->data;
2888 char * src1_d = (char *) (src1 ? src1->data : src0->data);
2889 const size_t src0_o = src0->nb[1];
2890 const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
2891
2892 GGML_ASSERT(ggml_is_contiguous_1(src0))if (!(ggml_is_contiguous_1(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2892, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src0)"
);
2893 GGML_ASSERT(ggml_is_contiguous_1(dst))if (!(ggml_is_contiguous_1(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2893, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(dst)"
);
2894
2895 if (src1) {
2896 GGML_ASSERT(ggml_is_contiguous_1(src1))if (!(ggml_is_contiguous_1(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2896, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src1)"
);
2897 GGML_ASSERT(src0->type == src1->type)if (!(src0->type == src1->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2897, "GGML_ASSERT(%s) failed", "src0->type == src1->type"
);
2898 }
2899
2900 const int ith = params->ith;
2901 const int nth = params->nth;
2902
2903 const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
2904 const int nr = ggml_nrows(src0);
2905
2906 GGML_ASSERT(dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2906, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
2907 GGML_ASSERT(ggml_nrows(dst) == nr)if (!(ggml_nrows(dst) == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2907, "GGML_ASSERT(%s) failed", "ggml_nrows(dst) == nr");
2908
2909 const int32_t swapped = ggml_get_op_params_i32(dst, 1);
2910
2911 // rows per thread
2912 const int dr = (nr + nth - 1)/nth;
2913
2914 // row range for this thread
2915 const int ir0 = dr*ith;
2916 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2917
2918 for (int i1 = ir0; i1 < ir1; i1++) {
2919 float * src0_p = (float *) (src0_d + i1*src0_o);
2920 float * src1_p = (float *) (src1_d + i1*src1_o);
2921
2922 if (!src1) {
2923 src0_p += swapped ? nc : 0;
2924 src1_p += swapped ? 0 : nc;
2925 }
2926
2927 ggml_vec_reglu_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);
2928
2929#ifndef NDEBUG
2930 for (int k = 0; k < nc; k++) {
2931 const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
2932 GGML_UNUSED(x)(void)(x);
2933 assert(!isnan(x))(static_cast <bool> (!isnan(x)) ? void (0) : __assert_fail
("!isnan(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2934 assert(!isinf(x))(static_cast <bool> (!isinf(x)) ? void (0) : __assert_fail
("!isinf(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2935 }
2936#endif // NDEBUG
2937 }
2938}
2939
2940static void ggml_compute_forward_reglu_f16(
2941 const ggml_compute_params * params,
2942 ggml_tensor * dst) {
2943
2944 const ggml_tensor * src0 = dst->src[0];
2945 const ggml_tensor * src1 = dst->src[1];
2946 char * src0_d = (char *) src0->data;
2947 char * src1_d = (char *) (src1 ? src1->data : src0->data);
2948 const size_t src0_o = src0->nb[1];
2949 const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
2950
2951 GGML_ASSERT(ggml_is_contiguous_1(src0))if (!(ggml_is_contiguous_1(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2951, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src0)"
);
2952 GGML_ASSERT(ggml_is_contiguous_1(dst))if (!(ggml_is_contiguous_1(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2952, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(dst)"
);
2953
2954 if (src1) {
2955 GGML_ASSERT(ggml_is_contiguous_1(src1))if (!(ggml_is_contiguous_1(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2955, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src1)"
);
2956 GGML_ASSERT(src0->type == src1->type)if (!(src0->type == src1->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2956, "GGML_ASSERT(%s) failed", "src0->type == src1->type"
);
2957 }
2958
2959 const int ith = params->ith;
2960 const int nth = params->nth;
2961
2962 const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
2963 const int nr = ggml_nrows(src0);
2964
2965 GGML_ASSERT(dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2965, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
2966 GGML_ASSERT(ggml_nrows(dst) == nr)if (!(ggml_nrows(dst) == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 2966, "GGML_ASSERT(%s) failed", "ggml_nrows(dst) == nr");
2967
2968 const int32_t swapped = ggml_get_op_params_i32(dst, 1);
2969
2970 // rows per thread
2971 const int dr = (nr + nth - 1)/nth;
2972
2973 // row range for this thread
2974 const int ir0 = dr*ith;
2975 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
2976
2977 for (int i1 = ir0; i1 < ir1; i1++) {
2978 ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
2979 ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
2980
2981 if (!src1) {
2982 src0_p += swapped ? nc : 0;
2983 src1_p += swapped ? 0 : nc;
2984 }
2985
2986 ggml_vec_reglu_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);
2987
2988#ifndef NDEBUG
2989 for (int k = 0; k < nc; k++) {
2990 const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];
2991 const float v = GGML_FP16_TO_FP32(x)ggml_compute_fp16_to_fp32(x);
2992 GGML_UNUSED(v)(void)(v);
2993 assert(!isnan(v))(static_cast <bool> (!isnan(v)) ? void (0) : __assert_fail
("!isnan(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2994 assert(!isinf(v))(static_cast <bool> (!isinf(v)) ? void (0) : __assert_fail
("!isinf(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
2995 }
2996#endif // NDEBUG
2997 }
2998}
2999
3000static void ggml_compute_forward_reglu(
3001 const ggml_compute_params * params,
3002 ggml_tensor * dst) {
3003
3004 const ggml_tensor * src0 = dst->src[0];
3005
3006 switch (src0->type) {
3007 case GGML_TYPE_F32:
3008 {
3009 ggml_compute_forward_reglu_f32(params, dst);
3010 } break;
3011 case GGML_TYPE_F16:
3012 {
3013 ggml_compute_forward_reglu_f16(params, dst);
3014 } break;
3015 default:
3016 {
3017 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3017, "fatal error");
3018 }
3019 }
3020}
3021
3022// ggml_compute_forward_geglu
3023
3024static void ggml_compute_forward_geglu_f32(
3025 const ggml_compute_params * params,
3026 ggml_tensor * dst) {
3027
3028 const ggml_tensor * src0 = dst->src[0];
3029 const ggml_tensor * src1 = dst->src[1];
3030 char * src0_d = (char *) src0->data;
3031 char * src1_d = (char *) (src1 ? src1->data : src0->data);
3032 const size_t src0_o = src0->nb[1];
3033 const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
3034
3035 GGML_ASSERT(ggml_is_contiguous_1(src0))if (!(ggml_is_contiguous_1(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3035, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src0)"
);
3036 GGML_ASSERT(ggml_is_contiguous_1(dst))if (!(ggml_is_contiguous_1(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3036, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(dst)"
);
3037
3038 if (src1) {
3039 GGML_ASSERT(ggml_is_contiguous_1(src1))if (!(ggml_is_contiguous_1(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3039, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src1)"
);
3040 GGML_ASSERT(src0->type == src1->type)if (!(src0->type == src1->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3040, "GGML_ASSERT(%s) failed", "src0->type == src1->type"
);
3041 }
3042
3043 const int ith = params->ith;
3044 const int nth = params->nth;
3045
3046 const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
3047 const int nr = ggml_nrows(src0);
3048
3049 GGML_ASSERT(dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3049, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
3050 GGML_ASSERT(ggml_nrows(dst) == nr)if (!(ggml_nrows(dst) == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3050, "GGML_ASSERT(%s) failed", "ggml_nrows(dst) == nr");
3051
3052 const int32_t swapped = ggml_get_op_params_i32(dst, 1);
3053
3054 // rows per thread
3055 const int dr = (nr + nth - 1)/nth;
3056
3057 // row range for this thread
3058 const int ir0 = dr*ith;
3059 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
3060
3061 for (int i1 = ir0; i1 < ir1; i1++) {
3062 float * src0_p = (float *) (src0_d + i1*src0_o);
3063 float * src1_p = (float *) (src1_d + i1*src1_o);
3064
3065 if (!src1) {
3066 src0_p += swapped ? nc : 0;
3067 src1_p += swapped ? 0 : nc;
3068 }
3069
3070 ggml_vec_geglu_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);
3071
3072#ifndef NDEBUG
3073 for (int k = 0; k < nc; k++) {
3074 const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
3075 GGML_UNUSED(x)(void)(x);
3076 assert(!isnan(x))(static_cast <bool> (!isnan(x)) ? void (0) : __assert_fail
("!isnan(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3077 assert(!isinf(x))(static_cast <bool> (!isinf(x)) ? void (0) : __assert_fail
("!isinf(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3078 }
3079#endif // NDEBUG
3080 }
3081}
3082
3083static void ggml_compute_forward_geglu_f16(
3084 const ggml_compute_params * params,
3085 ggml_tensor * dst) {
3086
3087 const ggml_tensor * src0 = dst->src[0];
3088 const ggml_tensor * src1 = dst->src[1];
3089 char * src0_d = (char *) src0->data;
3090 char * src1_d = (char *) (src1 ? src1->data : src0->data);
3091 const size_t src0_o = src0->nb[1];
3092 const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
3093
3094 GGML_ASSERT(ggml_is_contiguous_1(src0))if (!(ggml_is_contiguous_1(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3094, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src0)"
);
3095 GGML_ASSERT(ggml_is_contiguous_1(dst))if (!(ggml_is_contiguous_1(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3095, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(dst)"
);
3096
3097 if (src1) {
3098 GGML_ASSERT(ggml_is_contiguous_1(src1))if (!(ggml_is_contiguous_1(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3098, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src1)"
);
3099 GGML_ASSERT(src0->type == src1->type)if (!(src0->type == src1->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3099, "GGML_ASSERT(%s) failed", "src0->type == src1->type"
);
3100 }
3101
3102 const int ith = params->ith;
3103 const int nth = params->nth;
3104
3105 const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
3106 const int nr = ggml_nrows(src0);
3107
3108 GGML_ASSERT(dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3108, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
3109 GGML_ASSERT(ggml_nrows(dst) == nr)if (!(ggml_nrows(dst) == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3109, "GGML_ASSERT(%s) failed", "ggml_nrows(dst) == nr");
3110
3111 const int32_t swapped = ggml_get_op_params_i32(dst, 1);
3112
3113 // rows per thread
3114 const int dr = (nr + nth - 1)/nth;
3115
3116 // row range for this thread
3117 const int ir0 = dr*ith;
3118 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
3119
3120 for (int i1 = ir0; i1 < ir1; i1++) {
3121 ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
3122 ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
3123
3124 if (!src1) {
3125 src0_p += swapped ? nc : 0;
3126 src1_p += swapped ? 0 : nc;
3127 }
3128
3129 ggml_vec_geglu_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);
3130
3131#ifndef NDEBUG
3132 for (int k = 0; k < nc; k++) {
3133 const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];
3134 const float v = GGML_FP16_TO_FP32(x)ggml_compute_fp16_to_fp32(x);
3135 GGML_UNUSED(v)(void)(v);
3136 assert(!isnan(v))(static_cast <bool> (!isnan(v)) ? void (0) : __assert_fail
("!isnan(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3137 assert(!isinf(v))(static_cast <bool> (!isinf(v)) ? void (0) : __assert_fail
("!isinf(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3138 }
3139#endif // NDEBUG
3140 }
3141}
3142
3143static void ggml_compute_forward_geglu(
3144 const ggml_compute_params * params,
3145 ggml_tensor * dst) {
3146
3147 const ggml_tensor * src0 = dst->src[0];
3148
3149 switch (src0->type) {
3150 case GGML_TYPE_F32:
3151 {
3152 ggml_compute_forward_geglu_f32(params, dst);
3153 } break;
3154 case GGML_TYPE_F16:
3155 {
3156 ggml_compute_forward_geglu_f16(params, dst);
3157 } break;
3158 default:
3159 {
3160 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3160, "fatal error");
3161 }
3162 }
3163}
3164
3165// ggml_compute_forward_swiglu
3166
3167static void ggml_compute_forward_swiglu_f32(
3168 const ggml_compute_params * params,
3169 ggml_tensor * dst) {
3170
3171 const ggml_tensor * src0 = dst->src[0];
3172 const ggml_tensor * src1 = dst->src[1];
3173 char * src0_d = (char *) src0->data;
3174 char * src1_d = (char *) (src1 ? src1->data : src0->data);
3175 const size_t src0_o = src0->nb[1];
3176 const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
3177
3178 GGML_ASSERT(ggml_is_contiguous_1(src0))if (!(ggml_is_contiguous_1(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3178, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src0)"
);
3179 GGML_ASSERT(ggml_is_contiguous_1(dst))if (!(ggml_is_contiguous_1(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3179, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(dst)"
);
3180
3181 if (src1) {
3182 GGML_ASSERT(ggml_is_contiguous_1(src1))if (!(ggml_is_contiguous_1(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3182, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src1)"
);
3183 GGML_ASSERT(src0->type == src1->type)if (!(src0->type == src1->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3183, "GGML_ASSERT(%s) failed", "src0->type == src1->type"
);
3184 }
3185
3186 const int ith = params->ith;
3187 const int nth = params->nth;
3188
3189 const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
3190 const int nr = ggml_nrows(src0);
3191
3192 GGML_ASSERT(dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3192, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
3193 GGML_ASSERT(ggml_nrows(dst) == nr)if (!(ggml_nrows(dst) == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3193, "GGML_ASSERT(%s) failed", "ggml_nrows(dst) == nr");
3194
3195 const int32_t swapped = ggml_get_op_params_i32(dst, 1);
3196
3197 // rows per thread
3198 const int dr = (nr + nth - 1)/nth;
3199
3200 // row range for this thread
3201 const int ir0 = dr*ith;
3202 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
3203
3204 for (int i1 = ir0; i1 < ir1; i1++) {
3205 float * src0_p = (float *) (src0_d + i1*src0_o);
3206 float * src1_p = (float *) (src1_d + i1*src1_o);
3207
3208 if (!src1) {
3209 src0_p += swapped ? nc : 0;
3210 src1_p += swapped ? 0 : nc;
3211 }
3212
3213 ggml_vec_swiglu_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);
3214
3215#ifndef NDEBUG
3216 for (int k = 0; k < nc; k++) {
3217 const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
3218 GGML_UNUSED(x)(void)(x);
3219 assert(!isnan(x))(static_cast <bool> (!isnan(x)) ? void (0) : __assert_fail
("!isnan(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3220 assert(!isinf(x))(static_cast <bool> (!isinf(x)) ? void (0) : __assert_fail
("!isinf(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3221 }
3222#endif // NDEBUG
3223 }
3224}
3225
3226static void ggml_compute_forward_swiglu_f16(
3227 const ggml_compute_params * params,
3228 ggml_tensor * dst) {
3229
3230 const ggml_tensor * src0 = dst->src[0];
3231 const ggml_tensor * src1 = dst->src[1];
3232 char * src0_d = (char *) src0->data;
3233 char * src1_d = (char *) (src1 ? src1->data : src0->data);
3234 const size_t src0_o = src0->nb[1];
3235 const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
3236
3237 GGML_ASSERT(ggml_is_contiguous_1(src0))if (!(ggml_is_contiguous_1(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3237, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src0)"
);
3238 GGML_ASSERT(ggml_is_contiguous_1(dst))if (!(ggml_is_contiguous_1(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3238, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(dst)"
);
3239
3240 if (src1) {
3241 GGML_ASSERT(ggml_is_contiguous_1(src1))if (!(ggml_is_contiguous_1(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3241, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src1)"
);
3242 GGML_ASSERT(src0->type == src1->type)if (!(src0->type == src1->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3242, "GGML_ASSERT(%s) failed", "src0->type == src1->type"
);
3243 }
3244
3245 const int ith = params->ith;
3246 const int nth = params->nth;
3247
3248 const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
3249 const int nr = ggml_nrows(src0);
3250
3251 GGML_ASSERT(dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3251, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
3252 GGML_ASSERT(ggml_nrows(dst) == nr)if (!(ggml_nrows(dst) == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3252, "GGML_ASSERT(%s) failed", "ggml_nrows(dst) == nr");
3253
3254 const int32_t swapped = ggml_get_op_params_i32(dst, 1);
3255
3256 // rows per thread
3257 const int dr = (nr + nth - 1)/nth;
3258
3259 // row range for this thread
3260 const int ir0 = dr*ith;
3261 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
3262
3263 for (int i1 = ir0; i1 < ir1; i1++) {
3264 ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
3265 ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
3266
3267 if (!src1) {
3268 src0_p += swapped ? nc : 0;
3269 src1_p += swapped ? 0 : nc;
3270 }
3271
3272 ggml_vec_swiglu_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);
3273
3274#ifndef NDEBUG
3275 for (int k = 0; k < nc; k++) {
3276 const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];
3277 const float v = GGML_FP16_TO_FP32(x)ggml_compute_fp16_to_fp32(x);
3278 GGML_UNUSED(v)(void)(v);
3279 assert(!isnan(v))(static_cast <bool> (!isnan(v)) ? void (0) : __assert_fail
("!isnan(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3280 assert(!isinf(v))(static_cast <bool> (!isinf(v)) ? void (0) : __assert_fail
("!isinf(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3281 }
3282#endif // NDEBUG
3283 }
3284}
3285
3286static void ggml_compute_forward_swiglu(
3287 const ggml_compute_params * params,
3288 ggml_tensor * dst) {
3289
3290 const ggml_tensor * src0 = dst->src[0];
3291
3292 switch (src0->type) {
3293 case GGML_TYPE_F32:
3294 {
3295 ggml_compute_forward_swiglu_f32(params, dst);
3296 } break;
3297 case GGML_TYPE_F16:
3298 {
3299 ggml_compute_forward_swiglu_f16(params, dst);
3300 } break;
3301 default:
3302 {
3303 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3303, "fatal error");
3304 }
3305 }
3306}
3307
3308// ggml_compute_forward_swiglu_oai
3309
3310static void ggml_compute_forward_swiglu_oai_f32(
3311 const ggml_compute_params * params,
3312 ggml_tensor * dst) {
3313
3314 const ggml_tensor * src0 = dst->src[0];
3315 const ggml_tensor * src1 = dst->src[1];
3316 char * src0_d = (char *) src0->data;
3317 char * src1_d = (char *) (src1 ? src1->data : src0->data);
3318 const size_t src0_o = src0->nb[1];
3319 const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
3320
3321 GGML_ASSERT(ggml_is_contiguous_1(src0))if (!(ggml_is_contiguous_1(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3321, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src0)"
);
3322 GGML_ASSERT(ggml_is_contiguous_1(dst))if (!(ggml_is_contiguous_1(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3322, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(dst)"
);
3323
3324 if (src1) {
3325 GGML_ASSERT(ggml_is_contiguous_1(src1))if (!(ggml_is_contiguous_1(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3325, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src1)"
);
3326 GGML_ASSERT(src0->type == src1->type)if (!(src0->type == src1->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3326, "GGML_ASSERT(%s) failed", "src0->type == src1->type"
);
3327 }
3328
3329 const int ith = params->ith;
3330 const int nth = params->nth;
3331
3332 const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
3333 const int nr = ggml_nrows(src0);
3334
3335 GGML_ASSERT(dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3335, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
3336 GGML_ASSERT(ggml_nrows(dst) == nr)if (!(ggml_nrows(dst) == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3336, "GGML_ASSERT(%s) failed", "ggml_nrows(dst) == nr");
3337
3338 const int32_t swapped = ggml_get_op_params_i32(dst, 1);
3339 const float alpha = ggml_get_op_params_f32(dst, 2);
3340 const float limit = ggml_get_op_params_f32(dst, 3);
3341
3342 // rows per thread
3343 const int dr = (nr + nth - 1)/nth;
3344
3345 // row range for this thread
3346 const int ir0 = dr*ith;
3347 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
3348
3349 for (int i1 = ir0; i1 < ir1; i1++) {
3350 float * src0_p = (float *) (src0_d + i1*src0_o);
3351 float * src1_p = (float *) (src1_d + i1*src1_o);
3352 float * dst_p = (float *) ((char *) dst->data + i1*(dst->nb[1]));
3353
3354 if (!src1) {
3355 src0_p += swapped ? nc : 0;
3356 src1_p += swapped ? 0 : nc;
3357 }
3358
3359 for (int k = 0; k < nc; k++) {
3360 const float x = std::min(src0_p[k], limit);
3361 const float y = std::clamp(src1_p[k], -limit, limit);
3362 const float out_glu = x / (1.f + expf(alpha * (-x)));
3363 dst_p[k] = out_glu * (y + 1.f);
3364 }
3365
3366#ifndef NDEBUG
3367 for (int k = 0; k < nc; k++) {
3368 const float x = dst_p[k];
3369 GGML_UNUSED(x)(void)(x);
3370 assert(!isnan(x))(static_cast <bool> (!isnan(x)) ? void (0) : __assert_fail
("!isnan(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3371 assert(!isinf(x))(static_cast <bool> (!isinf(x)) ? void (0) : __assert_fail
("!isinf(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3372 }
3373#endif // NDEBUG
3374 }
3375}
3376
3377static void ggml_compute_forward_swiglu_oai(
3378 const ggml_compute_params * params,
3379 ggml_tensor * dst) {
3380
3381 const ggml_tensor * src0 = dst->src[0];
3382
3383 switch (src0->type) {
3384 case GGML_TYPE_F32:
3385 {
3386 ggml_compute_forward_swiglu_oai_f32(params, dst);
3387 } break;
3388 default:
3389 {
3390 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3390, "fatal error");
3391 }
3392 }
3393}
3394
3395// ggml_compute_forward_geglu_erf
3396
3397static void ggml_compute_forward_geglu_erf_f32(
3398 const ggml_compute_params * params,
3399 ggml_tensor * dst) {
3400
3401 const ggml_tensor * src0 = dst->src[0];
3402 const ggml_tensor * src1 = dst->src[1];
3403 char * src0_d = (char *) src0->data;
3404 char * src1_d = (char *) (src1 ? src1->data : src0->data);
3405 const size_t src0_o = src0->nb[1];
3406 const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
3407
3408 GGML_ASSERT(ggml_is_contiguous_1(src0))if (!(ggml_is_contiguous_1(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3408, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src0)"
);
3409 GGML_ASSERT(ggml_is_contiguous_1(dst))if (!(ggml_is_contiguous_1(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3409, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(dst)"
);
3410
3411 if (src1) {
3412 GGML_ASSERT(ggml_is_contiguous_1(src1))if (!(ggml_is_contiguous_1(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3412, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src1)"
);
3413 GGML_ASSERT(src0->type == src1->type)if (!(src0->type == src1->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3413, "GGML_ASSERT(%s) failed", "src0->type == src1->type"
);
3414 }
3415
3416 const int ith = params->ith;
3417 const int nth = params->nth;
3418
3419 const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
3420 const int nr = ggml_nrows(src0);
3421
3422 GGML_ASSERT(dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3422, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
3423 GGML_ASSERT(ggml_nrows(dst) == nr)if (!(ggml_nrows(dst) == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3423, "GGML_ASSERT(%s) failed", "ggml_nrows(dst) == nr");
3424
3425 const int32_t swapped = ggml_get_op_params_i32(dst, 1);
3426
3427 // rows per thread
3428 const int dr = (nr + nth - 1)/nth;
3429
3430 // row range for this thread
3431 const int ir0 = dr*ith;
3432 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
3433
3434 for (int i1 = ir0; i1 < ir1; i1++) {
3435 float * src0_p = (float *) (src0_d + i1*src0_o);
3436 float * src1_p = (float *) (src1_d + i1*src1_o);
3437
3438 if (!src1) {
3439 src0_p += swapped ? nc : 0;
3440 src1_p += swapped ? 0 : nc;
3441 }
3442
3443 ggml_vec_geglu_erf_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);
3444
3445#ifndef NDEBUG
3446 for (int k = 0; k < nc; k++) {
3447 const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
3448 GGML_UNUSED(x)(void)(x);
3449 assert(!isnan(x))(static_cast <bool> (!isnan(x)) ? void (0) : __assert_fail
("!isnan(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3450 assert(!isinf(x))(static_cast <bool> (!isinf(x)) ? void (0) : __assert_fail
("!isinf(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3451 }
3452#endif // NDEBUG
3453 }
3454}
3455
3456static void ggml_compute_forward_geglu_erf_f16(
3457 const ggml_compute_params * params,
3458 ggml_tensor * dst) {
3459
3460 const ggml_tensor * src0 = dst->src[0];
3461 const ggml_tensor * src1 = dst->src[1];
3462 char * src0_d = (char *) src0->data;
3463 char * src1_d = (char *) (src1 ? src1->data : src0->data);
3464 const size_t src0_o = src0->nb[1];
3465 const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
3466
3467 GGML_ASSERT(ggml_is_contiguous_1(src0))if (!(ggml_is_contiguous_1(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3467, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src0)"
);
3468 GGML_ASSERT(ggml_is_contiguous_1(dst))if (!(ggml_is_contiguous_1(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3468, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(dst)"
);
3469
3470 if (src1) {
3471 GGML_ASSERT(ggml_is_contiguous_1(src1))if (!(ggml_is_contiguous_1(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3471, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src1)"
);
3472 GGML_ASSERT(src0->type == src1->type)if (!(src0->type == src1->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3472, "GGML_ASSERT(%s) failed", "src0->type == src1->type"
);
3473 }
3474
3475 const int ith = params->ith;
3476 const int nth = params->nth;
3477
3478 const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
3479 const int nr = ggml_nrows(src0);
3480
3481 GGML_ASSERT(dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3481, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
3482 GGML_ASSERT(ggml_nrows(dst) == nr)if (!(ggml_nrows(dst) == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3482, "GGML_ASSERT(%s) failed", "ggml_nrows(dst) == nr");
3483
3484 const int32_t swapped = ggml_get_op_params_i32(dst, 1);
3485
3486 // rows per thread
3487 const int dr = (nr + nth - 1)/nth;
3488
3489 // row range for this thread
3490 const int ir0 = dr*ith;
3491 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
3492
3493 for (int i1 = ir0; i1 < ir1; i1++) {
3494 ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
3495 ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
3496
3497 if (!src1) {
3498 src0_p += swapped ? nc : 0;
3499 src1_p += swapped ? 0 : nc;
3500 }
3501
3502 ggml_vec_geglu_erf_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);
3503
3504#ifndef NDEBUG
3505 for (int k = 0; k < nc; k++) {
3506 const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];
3507 const float v = GGML_FP16_TO_FP32(x)ggml_compute_fp16_to_fp32(x);
3508 GGML_UNUSED(v)(void)(v);
3509 assert(!isnan(v))(static_cast <bool> (!isnan(v)) ? void (0) : __assert_fail
("!isnan(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3510 assert(!isinf(v))(static_cast <bool> (!isinf(v)) ? void (0) : __assert_fail
("!isinf(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3511 }
3512#endif // NDEBUG
3513 }
3514}
3515
3516static void ggml_compute_forward_geglu_erf(
3517 const ggml_compute_params * params,
3518 ggml_tensor * dst) {
3519
3520 const ggml_tensor * src0 = dst->src[0];
3521
3522 switch (src0->type) {
3523 case GGML_TYPE_F32:
3524 {
3525 ggml_compute_forward_geglu_erf_f32(params, dst);
3526 } break;
3527 case GGML_TYPE_F16:
3528 {
3529 ggml_compute_forward_geglu_erf_f16(params, dst);
3530 } break;
3531 default:
3532 {
3533 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3533, "fatal error");
3534 }
3535 }
3536}
3537
3538// ggml_compute_forward_geglu_quick
3539
3540static void ggml_compute_forward_geglu_quick_f32(
3541 const ggml_compute_params * params,
3542 ggml_tensor * dst) {
3543
3544 const ggml_tensor * src0 = dst->src[0];
3545 const ggml_tensor * src1 = dst->src[1];
3546 char * src0_d = (char *) src0->data;
3547 char * src1_d = (char *) (src1 ? src1->data : src0->data);
3548 const size_t src0_o = src0->nb[1];
3549 const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
3550
3551 GGML_ASSERT(ggml_is_contiguous_1(src0))if (!(ggml_is_contiguous_1(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3551, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src0)"
);
3552 GGML_ASSERT(ggml_is_contiguous_1(dst))if (!(ggml_is_contiguous_1(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3552, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(dst)"
);
3553
3554 if (src1) {
3555 GGML_ASSERT(ggml_is_contiguous_1(src1))if (!(ggml_is_contiguous_1(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3555, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src1)"
);
3556 GGML_ASSERT(src0->type == src1->type)if (!(src0->type == src1->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3556, "GGML_ASSERT(%s) failed", "src0->type == src1->type"
);
3557 }
3558
3559 const int ith = params->ith;
3560 const int nth = params->nth;
3561
3562 const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
3563 const int nr = ggml_nrows(src0);
3564
3565 GGML_ASSERT(dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3565, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
3566 GGML_ASSERT(ggml_nrows(dst) == nr)if (!(ggml_nrows(dst) == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3566, "GGML_ASSERT(%s) failed", "ggml_nrows(dst) == nr");
3567
3568 const int32_t swapped = ggml_get_op_params_i32(dst, 1);
3569
3570 // rows per thread
3571 const int dr = (nr + nth - 1)/nth;
3572
3573 // row range for this thread
3574 const int ir0 = dr*ith;
3575 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
3576
3577 for (int i1 = ir0; i1 < ir1; i1++) {
3578 float * src0_p = (float *) (src0_d + i1*src0_o);
3579 float * src1_p = (float *) (src1_d + i1*src1_o);
3580
3581 if (!src1) {
3582 src0_p += swapped ? nc : 0;
3583 src1_p += swapped ? 0 : nc;
3584 }
3585
3586 ggml_vec_geglu_quick_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);
3587
3588#ifndef NDEBUG
3589 for (int k = 0; k < nc; k++) {
3590 const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
3591 GGML_UNUSED(x)(void)(x);
3592 assert(!isnan(x))(static_cast <bool> (!isnan(x)) ? void (0) : __assert_fail
("!isnan(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3593 assert(!isinf(x))(static_cast <bool> (!isinf(x)) ? void (0) : __assert_fail
("!isinf(x)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3594 }
3595#endif // NDEBUG
3596 }
3597}
3598
3599static void ggml_compute_forward_geglu_quick_f16(
3600 const ggml_compute_params * params,
3601 ggml_tensor * dst) {
3602
3603 const ggml_tensor * src0 = dst->src[0];
3604 const ggml_tensor * src1 = dst->src[1];
3605 char * src0_d = (char *) src0->data;
3606 char * src1_d = (char *) (src1 ? src1->data : src0->data);
3607 const size_t src0_o = src0->nb[1];
3608 const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];
3609
3610 GGML_ASSERT(ggml_is_contiguous_1(src0))if (!(ggml_is_contiguous_1(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3610, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src0)"
);
3611 GGML_ASSERT(ggml_is_contiguous_1(dst))if (!(ggml_is_contiguous_1(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3611, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(dst)"
);
3612
3613 if (src1) {
3614 GGML_ASSERT(ggml_is_contiguous_1(src1))if (!(ggml_is_contiguous_1(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3614, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(src1)"
);
3615 GGML_ASSERT(src0->type == src1->type)if (!(src0->type == src1->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3615, "GGML_ASSERT(%s) failed", "src0->type == src1->type"
);
3616 }
3617
3618 const int ith = params->ith;
3619 const int nth = params->nth;
3620
3621 const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;
3622 const int nr = ggml_nrows(src0);
3623
3624 GGML_ASSERT(dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3624, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
3625 GGML_ASSERT(ggml_nrows(dst) == nr)if (!(ggml_nrows(dst) == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3625, "GGML_ASSERT(%s) failed", "ggml_nrows(dst) == nr");
3626
3627 const int32_t swapped = ggml_get_op_params_i32(dst, 1);
3628
3629 // rows per thread
3630 const int dr = (nr + nth - 1)/nth;
3631
3632 // row range for this thread
3633 const int ir0 = dr*ith;
3634 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
3635
3636 for (int i1 = ir0; i1 < ir1; i1++) {
3637 ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
3638 ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
3639
3640 if (!src1) {
3641 src0_p += swapped ? nc : 0;
3642 src1_p += swapped ? 0 : nc;
3643 }
3644
3645 ggml_vec_geglu_quick_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);
3646
3647#ifndef NDEBUG
3648 for (int k = 0; k < nc; k++) {
3649 const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];
3650 const float v = GGML_FP16_TO_FP32(x)ggml_compute_fp16_to_fp32(x);
3651 GGML_UNUSED(v)(void)(v);
3652 assert(!isnan(v))(static_cast <bool> (!isnan(v)) ? void (0) : __assert_fail
("!isnan(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3653 assert(!isinf(v))(static_cast <bool> (!isinf(v)) ? void (0) : __assert_fail
("!isinf(v)", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
3654 }
3655#endif // NDEBUG
3656 }
3657}
3658
3659static void ggml_compute_forward_geglu_quick(
3660 const ggml_compute_params * params,
3661 ggml_tensor * dst) {
3662
3663 const ggml_tensor * src0 = dst->src[0];
3664
3665 switch (src0->type) {
3666 case GGML_TYPE_F32:
3667 {
3668 ggml_compute_forward_geglu_quick_f32(params, dst);
3669 } break;
3670 case GGML_TYPE_F16:
3671 {
3672 ggml_compute_forward_geglu_quick_f16(params, dst);
3673 } break;
3674 default:
3675 {
3676 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3676, "fatal error");
3677 }
3678 }
3679}
3680
3681// ggml_compute_forward_norm
3682
3683static void ggml_compute_forward_norm_f32(
3684 const ggml_compute_params * params,
3685 ggml_tensor * dst) {
3686
3687 const ggml_tensor * src0 = dst->src[0];
3688
3689 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3689, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
3690
3691 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3691, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
3692
3693 const int ith = params->ith;
3694 const int nth = params->nth;
3695
3696 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
3697
3698 float eps;
3699 memcpy(&eps, dst->op_params, sizeof(float));
3700
3701 GGML_ASSERT(eps >= 0.0f)if (!(eps >= 0.0f)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3701, "GGML_ASSERT(%s) failed", "eps >= 0.0f");
3702
3703 for (int64_t i03 = 0; i03 < ne03; i03++) {
3704 for (int64_t i02 = 0; i02 < ne02; i02++) {
3705 for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
3706 const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
3707
3708 float sum = 0.0;
3709 ggml_vec_sum_f32(ne00, &sum, x);
3710 float mean = sum/ne00;
3711
3712 float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
3713 float variance = 0;
3714
3715#ifdef GGML_USE_ACCELERATE
3716 mean = -mean;
3717 vDSP_vsadd(x, 1, &mean, y, 1, ne00);
3718 vDSP_measqv(y, 1, &variance, ne00);
3719#else
3720 variance = ggml_vec_cvar_f32(ne00, y, x, mean);
3721#endif //GGML_USE_ACCELERATE
3722
3723 const float scale = 1.0f/sqrtf(variance + eps);
3724 ggml_vec_scale_f32(ne00, y, scale);
3725 }
3726 }
3727 }
3728}
3729
3730void ggml_compute_forward_norm(
3731 const ggml_compute_params * params,
3732 ggml_tensor * dst) {
3733
3734 const ggml_tensor * src0 = dst->src[0];
3735
3736 switch (src0->type) {
3737 case GGML_TYPE_F32:
3738 {
3739 ggml_compute_forward_norm_f32(params, dst);
3740 } break;
3741 default:
3742 {
3743 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3743, "fatal error");
3744 }
3745 }
3746}
3747
3748// ggml_compute_forward_group_rms_norm
3749
3750// fusion kinds that can be combined with the rms_norm computation in a single pass.
3751// extend this enum when adding new fused variants (e.g. FUSE_ADD, FUSE_MUL_ADD, ...).
3752enum ggml_rms_norm_fuse_op {
3753 GGML_RMS_NORM_FUSE_OP_NONE,
3754 GGML_RMS_NORM_FUSE_OP_MUL,
3755};
3756
3757template <ggml_rms_norm_fuse_op FUSE_OP>
3758static void ggml_compute_forward_rms_norm_f32(
3759 const ggml_compute_params * params,
3760 ggml_tensor * dst_rms_norm,
3761 ggml_tensor * dst_fused = nullptr) {
3762
3763 const ggml_tensor * src0 = dst_rms_norm->src[0];
3764 const ggml_tensor * src1 = nullptr;
3765 ggml_tensor * dst = dst_rms_norm;
3766
3767 if constexpr (FUSE_OP
0.1
1 is equal to GGML_RMS_NORM_FUSE_OP_MUL
 == GGML_RMS_NORM_FUSE_OP_MUL) {
3768 src1 = (dst_fused->src[0] == dst_rms_norm) ? dst_fused->src[1] : dst_fused->src[0];
1
Assuming the condition is false
→
2
←
'?' condition is false
→
3769 dst = dst_fused;
3770 }
3771
3772 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3772, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
3
←
Assuming the condition is false
→
4
←
Taking false branch
→
3773
3774 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3774, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
5
←
Assuming the condition is true
→
6
←
Taking false branch
→
3775
3776 const int ith = params->ith;
3777 const int nth = params->nth;
3778
3779 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
7
←
Assuming 'src1' is null
→
8
←
'?' condition is false
→
9
←
'?' condition is false
→
10
←
'ne11' initialized to 0
→
11
←
'?' condition is false
→
12
←
'?' condition is false
→
13
←
'?' condition is false
→
14
←
'?' condition is false
→
15
←
'?' condition is false
→
16
←
'?' condition is false
→
3780
3781 float eps;
3782 memcpy(&eps, dst_rms_norm->op_params, sizeof(float));
3783 GGML_ASSERT(eps >= 0.0f)if (!(eps >= 0.0f)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3783, "GGML_ASSERT(%s) failed", "eps >= 0.0f");
17
←
Assuming the condition is false
→
18
←
Taking false branch
→
3784
3785 // TODO: optimize
3786 for (int64_t i03 = 0; i03 < ne03; i03++) {
19
←
Assuming 'i03' is < 'ne03'
→
20
←
Loop condition is true. Entering loop body
→
3787 for (int64_t i02 = 0; i02 < ne02; i02++) {
21
←
Assuming 'i02' is < 'ne02'
→
22
←
Loop condition is true. Entering loop body
→
3788 for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
23
←
Assuming 'i01' is < 'ne01'
→
24
←
Loop condition is true. Entering loop body
→
3789 const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
3790
3791 ggml_float sum = 0.0;
3792 // worth switching to explicit SIMD?
3793 for (int64_t i00 = 0; i00 < ne00; i00++) {
25
←
Assuming 'i00' is >= 'ne00'
→
26
←
Loop condition is false. Execution continues on line 3797
→
3794 sum += (ggml_float)(x[i00] * x[i00]);
3795 }
3796
3797 const float mean = sum/ne00;
3798 const float scale = 1.0f/sqrtf(mean + eps);
3799
3800 // if you hit this, likely you got an inf somewhere earlier
3801 assert(scale > 0.0f)(static_cast <bool> (scale > 0.0f) ? void (0) : __assert_fail
("scale > 0.0f", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
27
←
Assuming the condition is true
→
28
←
'?' condition is true
→
3802
3803 float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
3804
3805 if constexpr (FUSE_OP
28.1
1 is equal to GGML_RMS_NORM_FUSE_OP_MUL
 == GGML_RMS_NORM_FUSE_OP_MUL) {
29
←
Taking true branch
→
3806 const int64_t i11 = i01 % ne11;
30
←
Division by zero
3807 const int64_t i12 = i02 % ne12;
3808 const int64_t i13 = i03 % ne13;
3809 const float * w = (float *) ((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13);
3810
3811 for (int64_t i00 = 0; i00 < ne00; i00++) {
3812 y[i00] = x[i00] * scale * w[i00];
3813 }
3814 } else {
3815 memcpy(y, x, ne00 * sizeof(float));
3816 ggml_vec_scale_f32(ne00, y, scale);
3817 }
3818 }
3819 }
3820 }
3821}
3822
3823void ggml_compute_forward_rms_norm(
3824 const ggml_compute_params * params,
3825 ggml_tensor * dst) {
3826
3827 const ggml_tensor * src0 = dst->src[0];
3828
3829 switch (src0->type) {
3830 case GGML_TYPE_F32:
3831 {
3832 ggml_compute_forward_rms_norm_f32<GGML_RMS_NORM_FUSE_OP_NONE>(params, dst);
3833 } break;
3834 default:
3835 {
3836 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3836, "fatal error");
3837 }
3838 }
3839}
3840
3841// Fused RMS_NORM + MUL: computes dst = rms_norm(src0) * src1 in a single pass.
3842// This avoids materializing the intermediate rms_norm result in memory.
3843void ggml_compute_forward_rms_norm_mul_fused(
3844 const ggml_compute_params * params,
3845 ggml_tensor * dst_rms_norm,
3846 ggml_tensor * dst_mul) {
3847
3848 GGML_ASSERT(dst_mul != nullptr)if (!(dst_mul != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3848, "GGML_ASSERT(%s) failed", "dst_mul != nullptr");
3849 GGML_ASSERT(dst_mul->src[0] == dst_rms_norm || dst_mul->src[1] == dst_rms_norm)if (!(dst_mul->src[0] == dst_rms_norm || dst_mul->src[1
] == dst_rms_norm)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3849, "GGML_ASSERT(%s) failed", "dst_mul->src[0] == dst_rms_norm || dst_mul->src[1] == dst_rms_norm"
);
3850
3851 const ggml_tensor * src0 = dst_rms_norm->src[0];
3852
3853 switch (src0->type) {
3854 case GGML_TYPE_F32:
3855 {
3856 ggml_compute_forward_rms_norm_f32<GGML_RMS_NORM_FUSE_OP_MUL>(params, dst_rms_norm, dst_mul);
3857 } break;
3858 default:
3859 {
3860 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3860, "fatal error");
3861 }
3862 }
3863}
3864
3865static void ggml_compute_forward_rms_norm_back_f32(
3866 const ggml_compute_params * params,
3867 ggml_tensor * dst) {
3868
3869 const ggml_tensor * src0 = dst->src[0]; // gradients from forward pass output
3870 const ggml_tensor * src1 = dst->src[1]; // src1 from forward pass
3871
3872 GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_are_same_shape(src0, src1))if (!(ggml_are_same_shape(src0, dst) && ggml_are_same_shape
(src0, src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3872, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst) && ggml_are_same_shape(src0, src1)"
);
3873
3874 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3874, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
3875 GGML_ASSERT(src1->nb[0] == sizeof(float))if (!(src1->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 3875, "GGML_ASSERT(%s) failed", "src1->nb[0] == sizeof(float)"
);
3876
3877 const int ith = params->ith;
3878 const int nth = params->nth;
3879
3880 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
3881
3882 float eps;
3883 memcpy(&eps, dst->op_params, sizeof(float));
3884
3885 // TODO: optimize
3886 for (int64_t i03 = 0; i03 < ne03; i03++) {
3887 for (int64_t i02 = 0; i02 < ne02; i02++) {
3888 for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
3889 // src1 is same shape as src0 => same indices
3890 const int64_t i11 = i01;
3891 const int64_t i12 = i02;
3892 const int64_t i13 = i03;
3893
3894 const float * dz = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
3895 const float * x = (float *) ((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13);
3896
3897 ggml_float sum_xx = 0.0;
3898 ggml_float sum_xdz = 0.0;
3899
3900 for (int64_t i00 = 0; i00 < ne00; i00++) {
3901 sum_xx += (ggml_float)(x[i00] * x[i00]);
3902 sum_xdz += (ggml_float)(x[i00] * dz[i00]);
3903 }
3904
3905 //const float mean = (float)(sum_xx)/ne00;
3906 const float mean_eps = (float)(sum_xx)/ne00 + eps;
3907 const float sum_eps = (float)(sum_xx) + eps*ne00;
3908 //const float mean_xdz = (float)(sum_xdz)/ne00;
3909 // we could cache rms from forward pass to improve performance.
3910 // to do this implement ggml_rms and compose ggml_rms_norm using ggml_rms.
3911 //const float rms = sqrtf(mean_eps);
3912 const float rrms = 1.0f / sqrtf(mean_eps);
3913 //const float scale = -rrms/(ne00 * mean_eps); // -1/(n*rms**3)
3914
3915 {
3916 // z = rms_norm(x)
3917 //
3918 // rms_norm(src1) =
3919 // scale(
3920 // src1,
3921 // div(
3922 // 1,
3923 // sqrt(
3924 // add(
3925 // scale(
3926 // sum(
3927 // sqr(
3928 // src1)),
3929 // (1.0/N)),
3930 // eps))));
3931
3932 // postorder:
3933 // ## op args grad
3934 // 00 param src1 grad[#00]
3935 // 01 const 1
3936 // 02 sqr (#00) grad[#02]
3937 // 03 sum (#02) grad[#03]
3938 // 04 const 1/N
3939 // 05 scale (#03, #04) grad[#05]
3940 // 06 const eps
3941 // 07 add (#05, #06) grad[#07]
3942 // 08 sqrt (#07) grad[#08]
3943 // 09 div (#01,#08) grad[#09]
3944 // 10 scale (#00,#09) grad[#10]
3945 //
3946 // backward pass, given grad[#10]
3947 // #10: scale
3948 // grad[#00] += scale(grad[#10],#09)
3949 // grad[#09] += sum(mul(grad[#10],#00))
3950 // #09: div
3951 // grad[#08] += neg(mul(grad[#09], div(#09,#08)))
3952 // #08: sqrt
3953 // grad[#07] += mul(grad[#08], div(0.5, #08))
3954 // #07: add
3955 // grad[#05] += grad[#07]
3956 // #05: scale
3957 // grad[#03] += scale(grad[#05],#04)
3958 // #03: sum
3959 // grad[#02] += repeat(grad[#03], #02)
3960 // #02:
3961 // grad[#00] += scale(mul(#00, grad[#02]), 2.0)
3962 //
3963 // substitute and simplify:
3964 // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, grad[#02]), 2.0)
3965 // grad[#02] = repeat(grad[#03], #02)
3966 // grad[#02] = repeat(scale(grad[#05],#04), #02)
3967 // grad[#02] = repeat(scale(grad[#07],#04), #02)
3968 // grad[#02] = repeat(scale(mul(grad[#08], div(0.5, #08)),#04), #02)
3969 // grad[#02] = repeat(scale(mul(neg(mul(grad[#09], div(#09,#08))), div(0.5, #08)),#04), #02)
3970 // grad[#02] = repeat(scale(mul(neg(mul(sum(mul(grad[#10],#00)), div(#09,#08))), div(0.5, #08)),#04), #02)
3971 // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(#09,#08) * div(0.5, #08) * (1/N)), #02)
3972 // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(div(#01,#08),#08) * div(0.5, #08) * (1/N)), #02)
3973 // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(1,#08*#08) * div(0.5, #08) * (1/N)), #02)
3974 // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N)), #02)
3975 // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, grad[#02]), 2.0)
3976 // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, repeat(-(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N)), #02)), 2.0)
3977 // grad[#00] = scale(grad(#10), #09) + scale(scale(#00, -(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N))), 2.0)
3978 // grad[#00] = scale(grad(#10), #09) + scale(#00, -(sum(mul(grad[#10],#00)) * div(1,#07) * div(1,#08) * (1/N)))
3979 // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,#07*#08) * (-1/N))
3980 // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,#07*#08) * (-1/N))
3981 // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,mean_eps*rms) * (-1/N))
3982 // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*mean_eps))
3983 // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*(sum_xx/N+eps)))
3984 // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*sum_xx+rms*N*eps))
3985 // grad[#00] = scale(dz, rrms) + scale(x, sum(mul(dz,x)) * div(-1,rms*N*mean_eps))
3986 // grad[#00] = scale(dz, rrms) + scale(x, sum_xdz * div(-1,rms*N*mean_eps))
3987 // a = b*c + d*e
3988 // a = b*c*f/f + d*e*f/f
3989 // a = (b*c*f + d*e*f)*(1/f)
3990 // a = (b*c*(1/c) + d*e*(1/c))*(1/(1/c))
3991 // a = (b + d*e/c)*c
3992 // b = dz, c = rrms, d = x, e = sum_xdz * div(-1,rms*N*mean_eps)
3993 // a = (dz + x*sum_xdz * div(-1,rms*N*mean_eps)/rrms)*rrms
3994 // a = (dz + x*sum_xdz * div(-1,rms*N*mean_eps)*rms)*rrms
3995 // a = (dz + x*sum_xdz * div(-rms,rms*N*mean_eps))*rrms
3996 // a = (dz + x*sum_xdz * div(-1,N*mean_eps))*rrms
3997 // a = (dz + x*div(-sum_xdz,N*mean_eps))*rrms
3998 // a = (dz + x*div(-mean_xdz,mean_eps))*rrms
3999 // grad[#00] = scale(dz + scale(x, div(-mean_xdz,mean_eps)),rrms)
4000 // grad[#00] = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)
4001 // dx = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)
4002 }
4003 // dx = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)
4004 // post-order:
4005 // dx := x
4006 // dx := scale(dx,-mean_xdz/mean_eps)
4007 // dx := add(dx, dz)
4008 // dx := scale(dx, rrms)
4009 float * dx = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
4010
4011 // dx[i00] = (dz + x*(-sum_xdz/sum_eps)) * rrms
4012 // note: https://github.com/ggml-org/ggml/issues/1491
4013 const float scale_x = (float) (-sum_xdz) / sum_eps;
4014 for (int64_t i00 = 0; i00 < ne00; i00++) {
4015 dx[i00] = (dz[i00] + x[i00] * scale_x) * rrms;
4016 }
4017 }
4018 }
4019 }
4020}
4021
4022void ggml_compute_forward_rms_norm_back(
4023 const ggml_compute_params * params,
4024 ggml_tensor * dst) {
4025
4026 const ggml_tensor * src0 = dst->src[0];
4027
4028 switch (src0->type) {
4029 case GGML_TYPE_F32:
4030 {
4031 ggml_compute_forward_rms_norm_back_f32(params, dst);
4032 } break;
4033 default:
4034 {
4035 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4035, "fatal error");
4036 }
4037 }
4038}
4039
4040// ggml_compute_forward_group_norm
4041
4042static void ggml_compute_forward_group_norm_f32(
4043 const ggml_compute_params * params,
4044 ggml_tensor * dst) {
4045
4046 const ggml_tensor * src0 = dst->src[0];
4047
4048 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4048, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
4049
4050 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4050, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
4051
4052 const int ith = params->ith;
4053 const int nth = params->nth;
4054
4055 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
4056
4057 // TODO: optimize
4058
4059 float eps;
4060 memcpy(&eps, dst->op_params + 1, sizeof(float));
4061
4062 int n_channels = src0->ne[2];
4063 int n_groups = dst->op_params[0];
4064 int n_channels_per_group = (n_channels + n_groups - 1) / n_groups;
4065 for (int i = ith; i < n_groups; i += nth) {
4066 int start = i * n_channels_per_group;
4067 int end = start + n_channels_per_group;
4068 if (end > n_channels) {
4069 end = n_channels;
4070 }
4071 int step = end - start;
4072
4073 for (int64_t i03 = 0; i03 < ne03; i03++) {
4074 ggml_float sum = 0.0;
4075 for (int64_t i02 = start; i02 < end; i02++) {
4076 for (int64_t i01 = 0; i01 < ne01; i01++) {
4077 const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03);
4078
4079 ggml_float sumr = 0.0;
4080 for (int64_t i00 = 0; i00 < ne00; i00++) {
4081 sumr += (ggml_float)x[i00];
4082 }
4083 sum += sumr;
4084 }
4085 }
4086 const float mean = sum / (ne00 * ne01 * step);
4087
4088 ggml_float sum2 = 0.0;
4089 for (int64_t i02 = start; i02 < end; i02++) {
4090 for (int64_t i01 = 0; i01 < ne01; i01++) {
4091 const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03);
4092
4093 float * y = (float *)((char *) dst->data + i01 * nb1 + i02 * nb2 + i03 * nb3);
4094
4095 ggml_float sumr = 0.0;
4096 for (int64_t i00 = 0; i00 < ne00; i00++) {
4097 float v = x[i00] - mean;
4098 y[i00] = v;
4099 sumr += (ggml_float)(v * v);
4100 }
4101 sum2 += sumr;
4102 }
4103 }
4104 const float variance = sum2 / (ne00 * ne01 * step);
4105 const float scale = 1.0f / sqrtf(variance + eps);
4106
4107 for (int64_t i02 = start; i02 < end; i02++) {
4108 for (int64_t i01 = 0; i01 < ne01; i01++) {
4109 float * y = (float *)((char *) dst->data + i01 * nb1 + i02 * nb2 + i03 * nb3);
4110 ggml_vec_scale_f32(ne00, y, scale);
4111 }
4112 }
4113 }
4114 }
4115}
4116
4117void ggml_compute_forward_group_norm(
4118 const ggml_compute_params * params,
4119 ggml_tensor * dst) {
4120
4121 const ggml_tensor * src0 = dst->src[0];
4122
4123 switch (src0->type) {
4124 case GGML_TYPE_F32:
4125 {
4126 ggml_compute_forward_group_norm_f32(params, dst);
4127 } break;
4128 default:
4129 {
4130 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4130, "fatal error");
4131 }
4132 }
4133}
4134
4135// ggml_compute_forward_l2_norm
4136
4137static void ggml_compute_forward_l2_norm_f32(
4138 const ggml_compute_params * params,
4139 ggml_tensor * dst) {
4140
4141 const ggml_tensor * src0 = dst->src[0];
4142
4143 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4143, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
4144
4145 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4145, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
4146
4147 const int ith = params->ith;
4148 const int nth = params->nth;
4149
4150 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
4151
4152 float eps;
4153 memcpy(&eps, dst->op_params, sizeof(float));
4154
4155 GGML_ASSERT(eps >= 0.0f)if (!(eps >= 0.0f)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4155, "GGML_ASSERT(%s) failed", "eps >= 0.0f");
4156
4157 // TODO: optimize
4158 for (int64_t i03 = 0; i03 < ne03; i03++) {
4159 for (int64_t i02 = 0; i02 < ne02; i02++) {
4160 for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
4161 const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
4162
4163 ggml_float sum = 0.0;
4164 for (int64_t i00 = 0; i00 < ne00; i00++) {
4165 sum += (ggml_float)(x[i00] * x[i00]);
4166 }
4167
4168 float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
4169
4170 memcpy(y, x, ne00 * sizeof(float));
4171
4172 const float scale = 1.0f/fmaxf(sqrtf(sum), eps);
4173
4174 ggml_vec_scale_f32(ne00, y, scale);
4175 }
4176 }
4177 }
4178}
4179
4180void ggml_compute_forward_l2_norm(
4181 const ggml_compute_params * params,
4182 ggml_tensor * dst) {
4183
4184 const ggml_tensor * src0 = dst->src[0];
4185
4186 switch (src0->type) {
4187 case GGML_TYPE_F32:
4188 {
4189 ggml_compute_forward_l2_norm_f32(params, dst);
4190 } break;
4191 default:
4192 {
4193 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4193, "fatal error");
4194 }
4195 }
4196}
4197
4198// ggml_compute_forward_out_prod
4199
4200static void ggml_compute_forward_out_prod_f32(
4201 const ggml_compute_params * params,
4202 ggml_tensor * dst) {
4203
4204 const ggml_tensor * src0 = dst->src[0];
4205 const ggml_tensor * src1 = dst->src[1];
4206
4207 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
4208
4209 GGML_ASSERT(dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4209, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
4210 GGML_ASSERT(src0->type == GGML_TYPE_F32)if (!(src0->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4210, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F32"
);
4211 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4211, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
4212
4213 const int ith = params->ith;
4214 const int nth = params->nth;
4215
4216 GGML_ASSERT(ne0 == ne00)if (!(ne0 == ne00)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4216, "GGML_ASSERT(%s) failed", "ne0 == ne00");
4217 GGML_ASSERT(ne1 == ne10)if (!(ne1 == ne10)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4217, "GGML_ASSERT(%s) failed", "ne1 == ne10");
4218 GGML_ASSERT(ne2 == ne12)if (!(ne2 == ne12)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4218, "GGML_ASSERT(%s) failed", "ne2 == ne12");
4219 GGML_ASSERT(ne3 == ne13)if (!(ne3 == ne13)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4219, "GGML_ASSERT(%s) failed", "ne3 == ne13");
4220
4221 GGML_ASSERT(ne2 % ne02 == 0)if (!(ne2 % ne02 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4221, "GGML_ASSERT(%s) failed", "ne2 % ne02 == 0");
4222 GGML_ASSERT(ne3 % ne03 == 0)if (!(ne3 % ne03 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4222, "GGML_ASSERT(%s) failed", "ne3 % ne03 == 0");
4223
4224 // we don't support permuted src0 or src1
4225 GGML_ASSERT(nb00 == sizeof(float))if (!(nb00 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4225, "GGML_ASSERT(%s) failed", "nb00 == sizeof(float)");
4226
4227 // dst cannot be transposed or permuted
4228 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4228, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
4229 // GGML_ASSERT(nb0 <= nb1);
4230 // GGML_ASSERT(nb1 <= nb2);
4231 // GGML_ASSERT(nb2 <= nb3);
4232
4233 // nb01 >= nb00 - src0 is not transposed
4234 // compute by src0 rows
4235
4236 if (ith == 0) {
4237 ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0);
4238 }
4239 ggml_barrier(params->threadpool);
4240
4241 // dst[:,:,:,:] = 0
4242 // for i2,i3:
4243 // for i1:
4244 // for i01:
4245 // for i0:
4246 // dst[i0,i1,i2,i3] += src0[i0,i01,i2,i3] * src1[i1,i01,i2,i3]
4247
4248 // parallelize by last three dimensions
4249
4250 // total rows in dst
4251 const int64_t nr = ne1*ne2*ne3;
4252
4253 // rows per thread
4254 const int64_t dr = (nr + nth - 1)/nth;
4255
4256 // row range for this thread
4257 const int64_t ir0 = dr*ith;
4258 const int64_t ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
4259
4260 // block-tiling attempt
4261 const int64_t blck_0 = MAX(GGML_VEC_MAD_UNROLL, 32)((32) > (32) ? (32) : (32));
4262 const int64_t blck_1 = 16;
4263
4264 // dps == dst per src0, used for group query attention
4265 const int64_t dps2 = ne2 / ne02;
4266 const int64_t dps3 = ne3 / ne03;
4267
4268 for (int64_t bir = ir0; bir < ir1; bir += blck_1) {
4269 const int64_t bir1 = MIN(bir + blck_1, ir1)((bir + blck_1) < (ir1) ? (bir + blck_1) : (ir1));
4270 for (int64_t bi01 = 0; bi01 < ne01; bi01 += blck_0) {
4271 const int64_t bne01 = MIN(bi01 + blck_0, ne01)((bi01 + blck_0) < (ne01) ? (bi01 + blck_0) : (ne01));
4272 for (int64_t ir = bir; ir < bir1; ++ir) {
4273 // dst indices
4274 const int64_t i3 = ir/(ne2*ne1);
4275 const int64_t i2 = (ir - i3*ne2*ne1)/ne1;
4276 const int64_t i1 = (ir - i3*ne2*ne1 - i2*ne1);
4277
4278 const int64_t i02 = i2 / dps2;
4279 const int64_t i03 = i3 / dps3;
4280
4281 //const int64_t i10 = i1;
4282 const int64_t i12 = i2;
4283 const int64_t i13 = i3;
4284
4285#if GGML_VEC_MAD_UNROLL32 > 2
4286 const int64_t bne01_unroll = bne01 - (bne01 % GGML_VEC_MAD_UNROLL32);
4287 for (int64_t i01 = bi01; i01 < bne01_unroll; i01 += GGML_VEC_MAD_UNROLL32) {
4288 const int64_t i11 = i01;
4289
4290 float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));
4291 float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));
4292 float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
4293
4294 ggml_vec_mad_f32_unroll(ne0, nb01, nb11, d, s0, s1);
4295 }
4296 for (int64_t i01 = bne01_unroll; i01 < bne01; ++i01) {
4297 const int64_t i11 = i01;
4298
4299 float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));
4300 float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));
4301 float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
4302
4303 ggml_vec_mad_f32(ne0, d, s0, *s1);
4304 }
4305#else
4306 for (int64_t i01 = bi01; i01 < bne01; ++i01) {
4307 const int64_t i11 = i01;
4308
4309 float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));
4310 float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));
4311 float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
4312
4313 ggml_vec_mad_f32(ne0, d, s0, *s1);
4314 }
4315#endif
4316 }
4317 }
4318 }
4319}
4320
4321static void ggml_compute_forward_out_prod_q_f32(
4322 const ggml_compute_params * params,
4323 ggml_tensor * dst) {
4324
4325 const ggml_tensor * src0 = dst->src[0];
4326 const ggml_tensor * src1 = dst->src[1];
4327
4328 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);;
4329
4330 const int ith = params->ith;
4331 const int nth = params->nth;
4332
4333 const ggml_type type = src0->type;
4334 ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;
4335
4336 GGML_ASSERT(ne02 == ne12)if (!(ne02 == ne12)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4336, "GGML_ASSERT(%s) failed", "ne02 == ne12");
4337 GGML_ASSERT(ne03 == ne13)if (!(ne03 == ne13)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4337, "GGML_ASSERT(%s) failed", "ne03 == ne13");
4338 GGML_ASSERT(ne2 == ne12)if (!(ne2 == ne12)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4338, "GGML_ASSERT(%s) failed", "ne2 == ne12");
4339 GGML_ASSERT(ne3 == ne13)if (!(ne3 == ne13)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4339, "GGML_ASSERT(%s) failed", "ne3 == ne13");
4340
4341 // we don't support permuted src0 dim0
4342 GGML_ASSERT(nb00 == ggml_type_size(type))if (!(nb00 == ggml_type_size(type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4342, "GGML_ASSERT(%s) failed", "nb00 == ggml_type_size(type)"
);
4343
4344 // dst dim0 cannot be transposed or permuted
4345 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4345, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
4346 // GGML_ASSERT(nb0 <= nb1);
4347 // GGML_ASSERT(nb1 <= nb2);
4348 // GGML_ASSERT(nb2 <= nb3);
4349
4350 GGML_ASSERT(ne0 == ne00)if (!(ne0 == ne00)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4350, "GGML_ASSERT(%s) failed", "ne0 == ne00");
4351 GGML_ASSERT(ne1 == ne10)if (!(ne1 == ne10)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4351, "GGML_ASSERT(%s) failed", "ne1 == ne10");
4352 GGML_ASSERT(ne2 == ne02)if (!(ne2 == ne02)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4352, "GGML_ASSERT(%s) failed", "ne2 == ne02");
4353 GGML_ASSERT(ne3 == ne03)if (!(ne3 == ne03)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4353, "GGML_ASSERT(%s) failed", "ne3 == ne03");
4354
4355 // nb01 >= nb00 - src0 is not transposed
4356 // compute by src0 rows
4357
4358 if (ith == 0) {
4359 ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0);
4360 }
4361 ggml_barrier(params->threadpool);
4362
4363 // parallelize by last three dimensions
4364
4365 // total rows in dst
4366 const int64_t nr = ne1*ne2*ne3;
4367
4368 // rows per thread
4369 const int64_t dr = (nr + nth - 1)/nth;
4370
4371 // row range for this thread
4372 const int64_t ir0 = dr*ith;
4373 const int64_t ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
4374
4375 // dst[:,:,:,:] = 0
4376 // for i2,i3:
4377 // for i1:
4378 // for i01:
4379 // for i0:
4380 // dst[i0,i1,i2,i3] += src0[i0,i01,i2,i3] * src1[i1,i01,i2,i3]
4381
4382 float * wdata = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32) * ith;
4383
4384 for (int64_t ir = ir0; ir < ir1; ++ir) {
4385 // dst indices
4386 const int64_t i3 = ir/(ne2*ne1);
4387 const int64_t i2 = (ir - i3*ne2*ne1)/ne1;
4388 const int64_t i1 = (ir - i3*ne2*ne1 - i2*ne1);
4389
4390 const int64_t i02 = i2;
4391 const int64_t i03 = i3;
4392
4393 //const int64_t i10 = i1;
4394 const int64_t i12 = i2;
4395 const int64_t i13 = i3;
4396
4397 for (int64_t i01 = 0; i01 < ne01; ++i01) {
4398 const int64_t i11 = i01;
4399
4400 float * s0 = (float *) ((char *) src0->data + ( i01*nb01 + i02*nb02 + i03*nb03));
4401 float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));
4402 float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
4403
4404 dequantize_row_q(s0, wdata, ne0);
4405 ggml_vec_mad_f32(ne0, d, wdata, *s1);
4406 }
4407 }
4408}
4409
4410void ggml_compute_forward_out_prod(
4411 const ggml_compute_params * params,
4412 ggml_tensor * dst) {
4413
4414 const ggml_tensor * src0 = dst->src[0];
4415
4416 switch (src0->type) {
4417 case GGML_TYPE_Q1_0:
4418 case GGML_TYPE_Q4_0:
4419 case GGML_TYPE_Q4_1:
4420 case GGML_TYPE_Q5_0:
4421 case GGML_TYPE_Q5_1:
4422 case GGML_TYPE_Q8_0:
4423 case GGML_TYPE_MXFP4:
4424 case GGML_TYPE_NVFP4:
4425 case GGML_TYPE_Q2_K:
4426 case GGML_TYPE_Q3_K:
4427 case GGML_TYPE_Q4_K:
4428 case GGML_TYPE_Q5_K:
4429 case GGML_TYPE_Q6_K:
4430 case GGML_TYPE_TQ1_0:
4431 case GGML_TYPE_TQ2_0:
4432 case GGML_TYPE_IQ2_XXS:
4433 case GGML_TYPE_IQ2_XS:
4434 case GGML_TYPE_IQ3_XXS:
4435 case GGML_TYPE_IQ1_S:
4436 case GGML_TYPE_IQ1_M:
4437 case GGML_TYPE_IQ4_NL:
4438 case GGML_TYPE_IQ4_XS:
4439 case GGML_TYPE_IQ3_S:
4440 case GGML_TYPE_IQ2_S:
4441 {
4442 ggml_compute_forward_out_prod_q_f32(params, dst);
4443 } break;
4444 case GGML_TYPE_F16:
4445 {
4446 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4446, "fatal error"); // todo
4447 // ggml_compute_forward_out_prod_f16_f32(params, dst);
4448 }
4449 case GGML_TYPE_F32:
4450 {
4451 ggml_compute_forward_out_prod_f32(params, dst);
4452 } break;
4453 default:
4454 {
4455 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4455, "fatal error");
4456 }
4457 }
4458}
4459
4460// ggml_compute_forward_scale
4461
4462static void ggml_compute_forward_scale_f32(
4463 const ggml_compute_params * params,
4464 ggml_tensor * dst) {
4465
4466 const ggml_tensor * src0 = dst->src[0];
4467
4468 GGML_ASSERT(ggml_is_contiguous(src0))if (!(ggml_is_contiguous(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4468, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(src0)");
4469 GGML_ASSERT(ggml_is_contiguous(dst))if (!(ggml_is_contiguous(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4469, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dst)");
4470 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4470, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
4471
4472 float s; // scale factor
4473 float b; // bias
4474
4475 memcpy(&s, (float *) dst->op_params + 0, sizeof(float));
4476 memcpy(&b, (float *) dst->op_params + 1, sizeof(float));
4477
4478 const int ith = params->ith;
4479 const int nth = params->nth;
4480
4481 const int nc = src0->ne[0];
4482 const int nr = ggml_nrows(src0);
4483
4484 // rows per thread
4485 const int dr = (nr + nth - 1)/nth;
4486
4487 // row range for this thread
4488 const int ir0 = dr*ith;
4489 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
4490
4491 const size_t nb01 = src0->nb[1];
4492
4493 const size_t nb1 = dst->nb[1];
4494
4495 if (b == 0.0f) {
4496 for (int i1 = ir0; i1 < ir1; i1++) {
4497 if (dst->data != src0->data) {
4498 // src0 is same shape as dst => same indices
4499 // TODO: add x parameter to ggml_vec_scale_f32 and remove this memcpy
4500 memcpy((char *)dst->data + i1*nb1, (char *)src0->data + i1*nb01, nc * sizeof(float));
4501 }
4502 ggml_vec_scale_f32(nc, (float *) ((char *) dst->data + i1*nb1), s);
4503 }
4504 } else {
4505 for (int i1 = ir0; i1 < ir1; i1++) {
4506 ggml_vec_mad1_f32(nc,
4507 (float *) ((char *) dst->data + i1*nb1),
4508 (float *) ((char *) src0->data + i1*nb1),
4509 s, b);
4510 }
4511 }
4512}
4513
4514void ggml_compute_forward_scale(
4515 const ggml_compute_params * params,
4516 ggml_tensor * dst) {
4517
4518 const ggml_tensor * src0 = dst->src[0];
4519
4520 switch (src0->type) {
4521 case GGML_TYPE_F32:
4522 {
4523 ggml_compute_forward_scale_f32(params, dst);
4524 } break;
4525 default:
4526 {
4527 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4527, "fatal error");
4528 }
4529 }
4530}
4531
4532// ggml_compute_forward_set
4533
4534static void ggml_compute_forward_set_f32(
4535 const ggml_compute_params * params,
4536 ggml_tensor * dst) {
4537
4538 const ggml_tensor * src0 = dst->src[0];
4539 const ggml_tensor * src1 = dst->src[1];
4540
4541 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4541, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
4542 GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0))if (!(ggml_is_contiguous(dst) && ggml_is_contiguous(src0
))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4542, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dst) && ggml_is_contiguous(src0)"
);
4543
4544 // view src0 and dst with these strides and data offset inbytes during set
4545 // nb0 is implicitly element_size because src0 and dst are contiguous
4546 size_t nb1 = ((int32_t *) dst->op_params)[0];
4547 size_t nb2 = ((int32_t *) dst->op_params)[1];
4548 size_t nb3 = ((int32_t *) dst->op_params)[2];
4549 size_t offset = ((int32_t *) dst->op_params)[3];
4550 bool inplace = (bool) ((int32_t *) dst->op_params)[4];
4551
4552 if (!inplace) {
4553 if (params->ith == 0) {
4554 // memcpy needs to be synchronized across threads to avoid race conditions.
4555 // => do it in INIT phase
4556 memcpy(
4557 ((char *) dst->data),
4558 ((char *) src0->data),
4559 ggml_nbytes(dst));
4560 }
4561 ggml_barrier(params->threadpool);
4562 }
4563
4564 const int ith = params->ith;
4565 const int nth = params->nth;
4566
4567 const int nr = ggml_nrows(src1);
4568 const int nc = src1->ne[0];
4569
4570 GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne)const int64_t ne10 = (src1) ? (src1)->ne[0] : 0; (void)(ne10
); const int64_t ne11 = (src1) ? (src1)->ne[1] : 0; (void)
(ne11); const int64_t ne12 = (src1) ? (src1)->ne[2] : 0; (
void)(ne12); const int64_t ne13 = (src1) ? (src1)->ne[3] :
0; (void)(ne13);
4571 GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)const size_t nb10 = (src1) ? (src1)->nb[0] : 0; (void)(nb10
); const size_t nb11 = (src1) ? (src1)->nb[1] : 0; (void)(
nb11); const size_t nb12 = (src1) ? (src1)->nb[2] : 0; (void
)(nb12); const size_t nb13 = (src1) ? (src1)->nb[3] : 0; (
void)(nb13);
4572
4573 // src0 and dst as viewed during set
4574 const size_t nb0 = ggml_element_size(src0);
4575
4576 const int im0 = (ne10 == 0 ? 0 : ne10-1);
4577 const int im1 = (ne11 == 0 ? 0 : ne11-1);
4578 const int im2 = (ne12 == 0 ? 0 : ne12-1);
4579 const int im3 = (ne13 == 0 ? 0 : ne13-1);
4580
4581 GGML_ASSERT(offset + im0*nb0 + im1*nb1 + im2*nb2 + im3*nb3 <= ggml_nbytes(dst))if (!(offset + im0*nb0 + im1*nb1 + im2*nb2 + im3*nb3 <= ggml_nbytes
(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4581, "GGML_ASSERT(%s) failed", "offset + im0*nb0 + im1*nb1 + im2*nb2 + im3*nb3 <= ggml_nbytes(dst)"
);
4582
4583 GGML_ASSERT(nb10 == sizeof(float))if (!(nb10 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4583, "GGML_ASSERT(%s) failed", "nb10 == sizeof(float)");
4584
4585 // rows per thread
4586 const int dr = (nr + nth - 1)/nth;
4587
4588 // row range for this thread
4589 const int ir0 = dr*ith;
4590 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
4591
4592 for (int ir = ir0; ir < ir1; ++ir) {
4593 // src0 and dst are viewed with shape of src1 and offset
4594 // => same indices
4595 const int i3 = ir/(ne12*ne11);
4596 const int i2 = (ir - i3*ne12*ne11)/ne11;
4597 const int i1 = (ir - i3*ne12*ne11 - i2*ne11);
4598
4599 ggml_vec_cpy_f32(nc,
4600 (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + offset),
4601 (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));
4602 }
4603}
4604
4605static void ggml_compute_forward_set_i32(
4606 const ggml_compute_params * params,
4607 ggml_tensor * dst) {
4608
4609 const ggml_tensor * src0 = dst->src[0];
4610 const ggml_tensor * src1 = dst->src[1];
4611
4612 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4612, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
4613 GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0))if (!(ggml_is_contiguous(dst) && ggml_is_contiguous(src0
))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4613, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dst) && ggml_is_contiguous(src0)"
);
4614
4615 // view src0 and dst with these strides and data offset inbytes during set
4616 // nb0 is implicitly element_size because src0 and dst are contiguous
4617 size_t nb1 = ((int32_t *) dst->op_params)[0];
4618 size_t nb2 = ((int32_t *) dst->op_params)[1];
4619 size_t nb3 = ((int32_t *) dst->op_params)[2];
4620 size_t offset = ((int32_t *) dst->op_params)[3];
4621 bool inplace = (bool) ((int32_t *) dst->op_params)[4];
4622
4623 if (!inplace) {
4624 if (params->ith == 0) {
4625 // memcpy needs to be synchronized across threads to avoid race conditions.
4626 // => do it in INIT phase
4627 memcpy(
4628 ((char *) dst->data),
4629 ((char *) src0->data),
4630 ggml_nbytes(dst));
4631 }
4632 ggml_barrier(params->threadpool);
4633 }
4634
4635 const int ith = params->ith;
4636 const int nth = params->nth;
4637
4638 const int nr = ggml_nrows(src1);
4639 const int nc = src1->ne[0];
4640
4641 GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne)const int64_t ne10 = (src1) ? (src1)->ne[0] : 0; (void)(ne10
); const int64_t ne11 = (src1) ? (src1)->ne[1] : 0; (void)
(ne11); const int64_t ne12 = (src1) ? (src1)->ne[2] : 0; (
void)(ne12); const int64_t ne13 = (src1) ? (src1)->ne[3] :
0; (void)(ne13);
4642 GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)const size_t nb10 = (src1) ? (src1)->nb[0] : 0; (void)(nb10
); const size_t nb11 = (src1) ? (src1)->nb[1] : 0; (void)(
nb11); const size_t nb12 = (src1) ? (src1)->nb[2] : 0; (void
)(nb12); const size_t nb13 = (src1) ? (src1)->nb[3] : 0; (
void)(nb13);
4643
4644 // src0 and dst as viewed during set
4645 const size_t nb0 = ggml_element_size(src0);
4646
4647 const int im0 = (ne10 == 0 ? 0 : ne10-1);
4648 const int im1 = (ne11 == 0 ? 0 : ne11-1);
4649 const int im2 = (ne12 == 0 ? 0 : ne12-1);
4650 const int im3 = (ne13 == 0 ? 0 : ne13-1);
4651
4652 GGML_ASSERT(offset + im0*nb0 + im1*nb1 + im2*nb2 + im3*nb3 <= ggml_nbytes(dst))if (!(offset + im0*nb0 + im1*nb1 + im2*nb2 + im3*nb3 <= ggml_nbytes
(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4652, "GGML_ASSERT(%s) failed", "offset + im0*nb0 + im1*nb1 + im2*nb2 + im3*nb3 <= ggml_nbytes(dst)"
);
4653
4654 GGML_ASSERT(nb10 == sizeof(int32_t))if (!(nb10 == sizeof(int32_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4654, "GGML_ASSERT(%s) failed", "nb10 == sizeof(int32_t)");
4655
4656 // rows per thread
4657 const int dr = (nr + nth - 1)/nth;
4658
4659 // row range for this thread
4660 const int ir0 = dr*ith;
4661 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
4662
4663 for (int ir = ir0; ir < ir1; ++ir) {
4664 // src0 and dst are viewed with shape of src1 and offset
4665 // => same indices
4666 const int i3 = ir/(ne12*ne11);
4667 const int i2 = (ir - i3*ne12*ne11)/ne11;
4668 const int i1 = (ir - i3*ne12*ne11 - i2*ne11);
4669
4670 ggml_vec_cpy_i32(nc,
4671 (int32_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + offset),
4672 (int32_t *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));
4673 }
4674}
4675
4676void ggml_compute_forward_set(
4677 const ggml_compute_params * params,
4678 ggml_tensor * dst) {
4679
4680 const ggml_tensor * src0 = dst->src[0];
4681
4682 switch (src0->type) {
4683 case GGML_TYPE_F32:
4684 {
4685 ggml_compute_forward_set_f32(params, dst);
4686 } break;
4687 case GGML_TYPE_I32:
4688 {
4689 ggml_compute_forward_set_i32(params, dst);
4690 } break;
4691 case GGML_TYPE_F16:
4692 case GGML_TYPE_BF16:
4693 case GGML_TYPE_Q1_0:
4694 case GGML_TYPE_Q4_0:
4695 case GGML_TYPE_Q4_1:
4696 case GGML_TYPE_Q5_0:
4697 case GGML_TYPE_Q5_1:
4698 case GGML_TYPE_Q8_0:
4699 case GGML_TYPE_Q8_1:
4700 case GGML_TYPE_MXFP4:
4701 case GGML_TYPE_NVFP4:
4702 case GGML_TYPE_Q2_K:
4703 case GGML_TYPE_Q3_K:
4704 case GGML_TYPE_Q4_K:
4705 case GGML_TYPE_Q5_K:
4706 case GGML_TYPE_Q6_K:
4707 case GGML_TYPE_TQ1_0:
4708 case GGML_TYPE_TQ2_0:
4709 case GGML_TYPE_IQ2_XXS:
4710 case GGML_TYPE_IQ2_XS:
4711 case GGML_TYPE_IQ3_XXS:
4712 case GGML_TYPE_IQ1_S:
4713 case GGML_TYPE_IQ1_M:
4714 case GGML_TYPE_IQ4_NL:
4715 case GGML_TYPE_IQ4_XS:
4716 case GGML_TYPE_IQ3_S:
4717 case GGML_TYPE_IQ2_S:
4718 default:
4719 {
4720 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4720, "fatal error");
4721 }
4722 }
4723}
4724
4725// ggml_compute_forward_cpy
4726
4727void ggml_compute_forward_cpy(
4728 const ggml_compute_params * params,
4729 ggml_tensor * dst) {
4730 ggml_compute_forward_dup(params, dst);
4731}
4732
4733// ggml_compute_forward_cont
4734
4735void ggml_compute_forward_cont(
4736 const ggml_compute_params * params,
4737 ggml_tensor * dst) {
4738 ggml_compute_forward_dup(params, dst);
4739}
4740
4741// ggml_compute_forward_get_rows
4742
4743static void ggml_compute_forward_get_rows_q(
4744 const ggml_compute_params * params,
4745 ggml_tensor * dst) {
4746
4747 const ggml_tensor * src0 = dst->src[0];
4748 const ggml_tensor * src1 = dst->src[1];
4749
4750 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
4751
4752 const int64_t nc = ne00;
4753 const int64_t nr = ggml_nelements(src1);
4754
4755 const ggml_type type = src0->type;
4756 ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;
4757
4758 assert(ne0 == nc)(static_cast <bool> (ne0 == nc) ? void (0) : __assert_fail
("ne0 == nc", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
4759 assert(ne02 == ne11)(static_cast <bool> (ne02 == ne11) ? void (0) : __assert_fail
("ne02 == ne11", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
4760 assert(nb00 == ggml_type_size(type))(static_cast <bool> (nb00 == ggml_type_size(type)) ? void
(0) : __assert_fail ("nb00 == ggml_type_size(type)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
4761 assert(ggml_nrows(dst) == nr)(static_cast <bool> (ggml_nrows(dst) == nr) ? void (0) :
__assert_fail ("ggml_nrows(dst) == nr", __builtin_FILE (), __builtin_LINE
(), __extension__ __PRETTY_FUNCTION__));
4762
4763 const int ith = params->ith;
4764 const int nth = params->nth;
4765
4766 // rows per thread
4767 const int dr = (nr + nth - 1)/nth;
4768
4769 // row range for this thread
4770 const int ir0 = dr*ith;
4771 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
4772
4773 for (int64_t i = ir0; i < ir1; ++i) {
4774 const int64_t i12 = i/(ne11*ne10);
4775 const int64_t i11 = (i - i12*ne11*ne10)/ne10;
4776 const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);
4777 const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);
4778
4779 GGML_ASSERT(i01 >= 0 && i01 < ne01)if (!(i01 >= 0 && i01 < ne01)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4779, "GGML_ASSERT(%s) failed", "i01 >= 0 && i01 < ne01"
);
4780
4781 dequantize_row_q(
4782 (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),
4783 (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);
4784 }
4785}
4786
4787static void ggml_compute_forward_get_rows_f16(
4788 const ggml_compute_params * params,
4789 ggml_tensor * dst) {
4790
4791 const ggml_tensor * src0 = dst->src[0];
4792 const ggml_tensor * src1 = dst->src[1];
4793
4794 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
4795
4796 const int64_t nc = ne00;
4797 const int64_t nr = ggml_nelements(src1);
4798
4799 assert(ne0 == nc)(static_cast <bool> (ne0 == nc) ? void (0) : __assert_fail
("ne0 == nc", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
4800 assert(ne02 == ne11)(static_cast <bool> (ne02 == ne11) ? void (0) : __assert_fail
("ne02 == ne11", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
4801 assert(nb00 == sizeof(ggml_fp16_t))(static_cast <bool> (nb00 == sizeof(ggml_fp16_t)) ? void
(0) : __assert_fail ("nb00 == sizeof(ggml_fp16_t)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
4802 assert(ggml_nrows(dst) == nr)(static_cast <bool> (ggml_nrows(dst) == nr) ? void (0) :
__assert_fail ("ggml_nrows(dst) == nr", __builtin_FILE (), __builtin_LINE
(), __extension__ __PRETTY_FUNCTION__));
4803
4804 const int ith = params->ith;
4805 const int nth = params->nth;
4806
4807 // rows per thread
4808 const int dr = (nr + nth - 1)/nth;
4809
4810 // row range for this thread
4811 const int ir0 = dr*ith;
4812 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
4813
4814 for (int64_t i = ir0; i < ir1; ++i) {
4815 const int64_t i12 = i/(ne11*ne10);
4816 const int64_t i11 = (i - i12*ne11*ne10)/ne10;
4817 const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);
4818 const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);
4819
4820 GGML_ASSERT(i01 >= 0 && i01 < ne01)if (!(i01 >= 0 && i01 < ne01)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4820, "GGML_ASSERT(%s) failed", "i01 >= 0 && i01 < ne01"
);
4821
4822 ggml_cpu_fp16_to_fp32(
4823 (const ggml_fp16_t*) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),
4824 (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);
4825 }
4826}
4827
4828static void ggml_compute_forward_get_rows_bf16(
4829 const ggml_compute_params * params,
4830 ggml_tensor * dst) {
4831
4832 const ggml_tensor * src0 = dst->src[0];
4833 const ggml_tensor * src1 = dst->src[1];
4834
4835 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
4836
4837 const int64_t nc = ne00;
4838 const int64_t nr = ggml_nelements(src1);
4839
4840 assert(ne0 == nc)(static_cast <bool> (ne0 == nc) ? void (0) : __assert_fail
("ne0 == nc", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
4841 assert(ne02 == ne11)(static_cast <bool> (ne02 == ne11) ? void (0) : __assert_fail
("ne02 == ne11", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
4842 assert(nb00 == sizeof(ggml_bf16_t))(static_cast <bool> (nb00 == sizeof(ggml_bf16_t)) ? void
(0) : __assert_fail ("nb00 == sizeof(ggml_bf16_t)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
4843 assert(ggml_nrows(dst) == nr)(static_cast <bool> (ggml_nrows(dst) == nr) ? void (0) :
__assert_fail ("ggml_nrows(dst) == nr", __builtin_FILE (), __builtin_LINE
(), __extension__ __PRETTY_FUNCTION__));
4844
4845 const int ith = params->ith;
4846 const int nth = params->nth;
4847
4848 // rows per thread
4849 const int dr = (nr + nth - 1)/nth;
4850
4851 // row range for this thread
4852 const int ir0 = dr*ith;
4853 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
4854
4855 for (int64_t i = ir0; i < ir1; ++i) {
4856 const int64_t i12 = i/(ne11*ne10);
4857 const int64_t i11 = (i - i12*ne11*ne10)/ne10;
4858 const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);
4859 const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);
4860
4861 GGML_ASSERT(i01 >= 0 && i01 < ne01)if (!(i01 >= 0 && i01 < ne01)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4861, "GGML_ASSERT(%s) failed", "i01 >= 0 && i01 < ne01"
);
4862
4863 ggml_cpu_bf16_to_fp32(
4864 (const ggml_bf16_t *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),
4865 (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);
4866 }
4867}
4868
4869static void ggml_compute_forward_get_rows_f32(
4870 const ggml_compute_params * params,
4871 ggml_tensor * dst) {
4872
4873 const ggml_tensor * src0 = dst->src[0];
4874 const ggml_tensor * src1 = dst->src[1];
4875
4876 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
4877
4878 const int64_t nc = ne00;
4879 const int64_t nr = ggml_nelements(src1);
4880
4881 assert(ne0 == nc)(static_cast <bool> (ne0 == nc) ? void (0) : __assert_fail
("ne0 == nc", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
4882 assert(ne02 == ne11)(static_cast <bool> (ne02 == ne11) ? void (0) : __assert_fail
("ne02 == ne11", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
4883 assert(nb00 == sizeof(float))(static_cast <bool> (nb00 == sizeof(float)) ? void (0) :
__assert_fail ("nb00 == sizeof(float)", __builtin_FILE (), __builtin_LINE
(), __extension__ __PRETTY_FUNCTION__));
4884 assert(ggml_nrows(dst) == nr)(static_cast <bool> (ggml_nrows(dst) == nr) ? void (0) :
__assert_fail ("ggml_nrows(dst) == nr", __builtin_FILE (), __builtin_LINE
(), __extension__ __PRETTY_FUNCTION__));
4885
4886 const int ith = params->ith;
4887 const int nth = params->nth;
4888
4889 // rows per thread
4890 const int dr = (nr + nth - 1)/nth;
4891
4892 // row range for this thread
4893 const int ir0 = dr*ith;
4894 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
4895
4896 for (int64_t i = ir0; i < ir1; ++i) {
4897 const int64_t i12 = i/(ne11*ne10);
4898 const int64_t i11 = (i - i12*ne11*ne10)/ne10;
4899 const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);
4900 const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);
4901
4902 GGML_ASSERT(i01 >= 0 && i01 < ne01)if (!(i01 >= 0 && i01 < ne01)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4902, "GGML_ASSERT(%s) failed", "i01 >= 0 && i01 < ne01"
);
4903
4904 ggml_vec_cpy_f32(nc,
4905 (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3),
4906 (float *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03));
4907 }
4908}
4909
4910void ggml_compute_forward_get_rows(
4911 const ggml_compute_params * params,
4912 ggml_tensor * dst) {
4913
4914 const ggml_tensor * src0 = dst->src[0];
4915
4916 switch (src0->type) {
4917 case GGML_TYPE_Q1_0:
4918 case GGML_TYPE_Q4_0:
4919 case GGML_TYPE_Q4_1:
4920 case GGML_TYPE_Q5_0:
4921 case GGML_TYPE_Q5_1:
4922 case GGML_TYPE_Q8_0:
4923 case GGML_TYPE_Q8_1:
4924 case GGML_TYPE_MXFP4:
4925 case GGML_TYPE_NVFP4:
4926 case GGML_TYPE_Q2_K:
4927 case GGML_TYPE_Q3_K:
4928 case GGML_TYPE_Q4_K:
4929 case GGML_TYPE_Q5_K:
4930 case GGML_TYPE_Q6_K:
4931 case GGML_TYPE_TQ1_0:
4932 case GGML_TYPE_TQ2_0:
4933 case GGML_TYPE_IQ2_XXS:
4934 case GGML_TYPE_IQ2_XS:
4935 case GGML_TYPE_IQ3_XXS:
4936 case GGML_TYPE_IQ1_S:
4937 case GGML_TYPE_IQ1_M:
4938 case GGML_TYPE_IQ4_NL:
4939 case GGML_TYPE_IQ4_XS:
4940 case GGML_TYPE_IQ3_S:
4941 case GGML_TYPE_IQ2_S:
4942 {
4943 ggml_compute_forward_get_rows_q(params, dst);
4944 } break;
4945 case GGML_TYPE_F16:
4946 {
4947 ggml_compute_forward_get_rows_f16(params, dst);
4948 } break;
4949 case GGML_TYPE_BF16:
4950 {
4951 ggml_compute_forward_get_rows_bf16(params, dst);
4952 } break;
4953 case GGML_TYPE_F32:
4954 case GGML_TYPE_I32:
4955 {
4956 ggml_compute_forward_get_rows_f32(params, dst);
4957 } break;
4958 default:
4959 {
4960 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 4960, "fatal error");
4961 }
4962 }
4963
4964 //static bool first = true;
4965 //printf("ne0 = %d, ne1 = %d, ne2 = %d\n", dst->ne[0], dst->ne[1], dst->ne[2]);
4966 //if (first) {
4967 // first = false;
4968 //} else {
4969 // for (int k = 0; k < dst->ne[1]; ++k) {
4970 // for (int j = 0; j < dst->ne[0]/16; ++j) {
4971 // for (int i = 0; i < 16; ++i) {
4972 // printf("%8.4f ", ((float *) dst->data)[k*dst->ne[0] + j*16 + i]);
4973 // }
4974 // printf("\n");
4975 // }
4976 // printf("\n");
4977 // }
4978 // printf("\n");
4979 // exit(0);
4980 //}
4981}
4982
4983template<typename idx_t>
4984static void ggml_compute_forward_set_rows_f32(
4985 const ggml_compute_params * params,
4986 ggml_tensor * dst) {
4987
4988 const ggml_tensor * src0 = dst->src[0];
4989 const ggml_tensor * src1 = dst->src[1];
4990
4991 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
4992
4993 const int64_t nc = ne00;
4994 const int64_t nr = ne01;
4995
4996 assert(ne0 == nc)(static_cast <bool> (ne0 == nc) ? void (0) : __assert_fail
("ne0 == nc", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
4997 assert(ne2 == ne02)(static_cast <bool> (ne2 == ne02) ? void (0) : __assert_fail
("ne2 == ne02", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
4998 assert(ne3 == ne03)(static_cast <bool> (ne3 == ne03) ? void (0) : __assert_fail
("ne3 == ne03", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
4999 assert(src0->type == GGML_TYPE_F32)(static_cast <bool> (src0->type == GGML_TYPE_F32) ? void
(0) : __assert_fail ("src0->type == GGML_TYPE_F32", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
5000 assert(ne02 % ne11 == 0)(static_cast <bool> (ne02 % ne11 == 0) ? void (0) : __assert_fail
("ne02 % ne11 == 0", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
5001 assert(ne03 % ne12 == 0)(static_cast <bool> (ne03 % ne12 == 0) ? void (0) : __assert_fail
("ne03 % ne12 == 0", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
5002
5003 const int ith = params->ith;
5004 const int nth = params->nth;
5005
5006 // rows per thread
5007 const int64_t dr = (nr + nth - 1)/nth;
5008
5009 // row range for this thread
5010 const int64_t ir0 = dr*ith;
5011 const int64_t ir1 = std::min(ir0 + dr, nr);
5012
5013 ggml_from_float_t const from_float = ggml_get_type_traits_cpu(dst->type)->from_float;
5014
5015 for (int64_t i03 = 0; i03 < ne03; ++i03) {
5016 for (int64_t i02 = 0; i02 < ne02; ++i02) {
5017 for (int64_t i = ir0; i < ir1; ++i) {
5018 const int64_t i12 = i03%ne12;
5019 const int64_t i11 = i02%ne11;
5020 const int64_t i10 = i;
5021
5022 const int64_t i1 = *(idx_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);
5023
5024 GGML_ASSERT(i1 >= 0 && i1 < ne1)if (!(i1 >= 0 && i1 < ne1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5024, "GGML_ASSERT(%s) failed", "i1 >= 0 && i1 < ne1"
);
5025
5026 from_float(
5027 (const float *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),
5028 ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc);
5029 }
5030 }
5031 }
5032}
5033
5034void ggml_compute_forward_set_rows(
5035 const ggml_compute_params * params,
5036 ggml_tensor * dst) {
5037
5038 const ggml_tensor * src0 = dst->src[0];
5039 const ggml_tensor * src1 = dst->src[1];
5040
5041 switch (src0->type) {
5042 case GGML_TYPE_F32:
5043 {
5044 if (src1->type == GGML_TYPE_I64) {
5045 ggml_compute_forward_set_rows_f32<int64_t>(params, dst);
5046 } else if (src1->type == GGML_TYPE_I32) {
5047 ggml_compute_forward_set_rows_f32<int32_t>(params, dst);
5048 } else {
5049 GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type))ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5049, "src1->type = %d (%s) not supported", src1->type
, ggml_type_name(src1->type));
5050 }
5051 } break;
5052 default:
5053 {
5054 GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type))ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5054, "src0->type = %d (%s) not supported", src0->type
, ggml_type_name(src0->type));
5055 }
5056 }
5057}
5058
5059// ggml_compute_forward_get_rows_back
5060
5061static void ggml_compute_forward_get_rows_back_f32_f16(
5062 const ggml_compute_params * params,
5063 ggml_tensor * dst) {
5064
5065 const ggml_tensor * src0 = dst->src[0];
5066 const ggml_tensor * src1 = dst->src[1];
5067
5068 if (params->ith != 0) {
5069 return;
5070 }
5071
5072 GGML_ASSERT(ggml_is_contiguous(dst))if (!(ggml_is_contiguous(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5072, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dst)");
5073
5074 // ggml_compute_forward_dup_same_cont(params, opt0, dst);
5075
5076 memset(dst->data, 0, ggml_nbytes(dst));
5077
5078 const int nc = src0->ne[0];
5079 const int nr = ggml_nelements(src1);
5080
5081 GGML_ASSERT( dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5081, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
5082 GGML_ASSERT(src0->nb[0] == sizeof(ggml_fp16_t))if (!(src0->nb[0] == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5082, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(ggml_fp16_t)"
);
5083
5084 for (int i = 0; i < nr; ++i) {
5085 const int r = ((int32_t *) src1->data)[i];
5086
5087 for (int j = 0; j < nc; ++j) {
5088 ggml_fp16_t v = ((ggml_fp16_t *) ((char *) src0->data + i*src0->nb[1]))[j];
5089 ((float *) ((char *) dst->data + r*dst->nb[1]))[j] += GGML_CPU_FP16_TO_FP32(v)ggml_lookup_fp16_to_fp32(v);
5090 }
5091 }
5092}
5093
5094static void ggml_compute_forward_get_rows_back_f32(
5095 const ggml_compute_params * params,
5096 ggml_tensor * dst) {
5097
5098 const ggml_tensor * src0 = dst->src[0];
5099 const ggml_tensor * src1 = dst->src[1];
5100
5101 if (params->ith != 0) {
5102 return;
5103 }
5104
5105 GGML_ASSERT(ggml_is_contiguous(dst))if (!(ggml_is_contiguous(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5105, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dst)");
5106
5107 // ggml_compute_forward_dup_same_cont(params, opt0, dst);
5108
5109 memset(dst->data, 0, ggml_nbytes(dst));
5110
5111 const int nc = src0->ne[0];
5112 const int nr = ggml_nelements(src1);
5113
5114 GGML_ASSERT( dst->ne[0] == nc)if (!(dst->ne[0] == nc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5114, "GGML_ASSERT(%s) failed", "dst->ne[0] == nc");
5115 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5115, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
5116
5117 for (int i = 0; i < nr; ++i) {
5118 const int r = ((int32_t *) src1->data)[i];
5119
5120 ggml_vec_add_f32(nc,
5121 (float *) ((char *) dst->data + r*dst->nb[1]),
5122 (float *) ((char *) dst->data + r*dst->nb[1]),
5123 (float *) ((char *) src0->data + i*src0->nb[1]));
5124 }
5125}
5126
5127void ggml_compute_forward_get_rows_back(
5128 const ggml_compute_params * params,
5129 ggml_tensor * dst) {
5130
5131 const ggml_tensor * src0 = dst->src[0];
5132
5133 switch (src0->type) {
5134 case GGML_TYPE_F16:
5135 {
5136 ggml_compute_forward_get_rows_back_f32_f16(params, dst);
5137 } break;
5138 case GGML_TYPE_F32:
5139 {
5140 ggml_compute_forward_get_rows_back_f32(params, dst);
5141 } break;
5142 default:
5143 {
5144 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5144, "fatal error");
5145 }
5146 }
5147
5148 //static bool first = true;
5149 //printf("ne0 = %d, ne1 = %d, ne2 = %d\n", dst->ne[0], dst->ne[1], dst->ne[2]);
5150 //if (first) {
5151 // first = false;
5152 //} else {
5153 // for (int k = 0; k < dst->ne[1]; ++k) {
5154 // for (int j = 0; j < dst->ne[0]/16; ++j) {
5155 // for (int i = 0; i < 16; ++i) {
5156 // printf("%8.4f ", ((float *) dst->data)[k*dst->ne[0] + j*16 + i]);
5157 // }
5158 // printf("\n");
5159 // }
5160 // printf("\n");
5161 // }
5162 // printf("\n");
5163 // exit(0);
5164 //}
5165}
5166
5167// ggml_compute_forward_diag
5168
5169static void ggml_compute_forward_diag_f32(
5170 const ggml_compute_params * params,
5171 ggml_tensor * dst) {
5172
5173 const ggml_tensor * src0 = dst->src[0];
5174
5175 if (params->ith != 0) {
5176 return;
5177 }
5178
5179 // TODO: handle transposed/permuted matrices
5180
5181 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
5182
5183 GGML_ASSERT(ne00 == ne0)if (!(ne00 == ne0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5183, "GGML_ASSERT(%s) failed", "ne00 == ne0");
5184 GGML_ASSERT(ne00 == ne1)if (!(ne00 == ne1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5184, "GGML_ASSERT(%s) failed", "ne00 == ne1");
5185 GGML_ASSERT(ne01 == 1)if (!(ne01 == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5185, "GGML_ASSERT(%s) failed", "ne01 == 1");
5186 GGML_ASSERT(ne02 == ne2)if (!(ne02 == ne2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5186, "GGML_ASSERT(%s) failed", "ne02 == ne2");
5187 GGML_ASSERT(ne03 == ne3)if (!(ne03 == ne3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5187, "GGML_ASSERT(%s) failed", "ne03 == ne3");
5188
5189 GGML_ASSERT(nb00 == sizeof(float))if (!(nb00 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5189, "GGML_ASSERT(%s) failed", "nb00 == sizeof(float)");
5190 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5190, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
5191
5192 for (int i3 = 0; i3 < ne3; i3++) {
5193 for (int i2 = 0; i2 < ne2; i2++) {
5194 for (int i1 = 0; i1 < ne1; i1++) {
5195 float * d = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
5196 float * s = (float *)((char *) src0->data + i3*nb03 + i2*nb02);
5197 for (int i0 = 0; i0 < i1; i0++) {
5198 d[i0] = 0;
5199 }
5200 d[i1] = s[i1];
5201 for (int i0 = i1+1; i0 < ne0; i0++) {
5202 d[i0] = 0;
5203 }
5204 }
5205 }
5206 }
5207}
5208
5209void ggml_compute_forward_diag(
5210 const ggml_compute_params * params,
5211 ggml_tensor * dst) {
5212
5213 const ggml_tensor * src0 = dst->src[0];
5214
5215 switch (src0->type) {
5216 case GGML_TYPE_F32:
5217 {
5218 ggml_compute_forward_diag_f32(params, dst);
5219 } break;
5220 default:
5221 {
5222 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5222, "fatal error");
5223 }
5224 }
5225}
5226
5227// ggml_compute_forward_diag_mask_inf
5228
5229static void ggml_compute_forward_diag_mask_f32(
5230 const ggml_compute_params * params,
5231 ggml_tensor * dst,
5232 const float value) {
5233
5234 const ggml_tensor * src0 = dst->src[0];
5235
5236 const int ith = params->ith;
5237 const int nth = params->nth;
5238
5239 const int n_past = ((int32_t *) dst->op_params)[0];
5240 const bool inplace = src0->data == dst->data;
5241
5242 GGML_ASSERT(n_past >= 0)if (!(n_past >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5242, "GGML_ASSERT(%s) failed", "n_past >= 0");
5243
5244 if (!inplace) {
5245 if (ith == 0) {
5246 // memcpy needs to be synchronized across threads to avoid race conditions.
5247 // => do it in INIT phase
5248 GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0))if (!(ggml_nelements(dst) == ggml_nelements(src0))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5248, "GGML_ASSERT(%s) failed", "ggml_nelements(dst) == ggml_nelements(src0)"
);
5249 GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0))if (!(ggml_is_contiguous(dst) && ggml_is_contiguous(src0
))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5249, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dst) && ggml_is_contiguous(src0)"
);
5250 memcpy(
5251 ((char *) dst->data),
5252 ((char *) src0->data),
5253 ggml_nbytes(dst));
5254 }
5255 ggml_barrier(params->threadpool);
5256 }
5257
5258 // TODO: handle transposed/permuted matrices
5259
5260 const int n = ggml_nrows(src0);
5261 const int nc = src0->ne[0];
5262 const int nr = src0->ne[1];
5263 const int nz = n/nr;
5264
5265 GGML_ASSERT( dst->nb[0] == sizeof(float))if (!(dst->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5265, "GGML_ASSERT(%s) failed", "dst->nb[0] == sizeof(float)"
);
5266 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5266, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
5267
5268 for (int k = 0; k < nz; k++) {
5269 for (int j = ith; j < nr; j += nth) {
5270 for (int i = n_past; i < nc; i++) {
5271 if (i > n_past + j) {
5272 *(float *)((char *) dst->data + k*dst->nb[2] + j*dst->nb[1] + i*dst->nb[0]) = value;
5273 }
5274 }
5275 }
5276 }
5277}
5278
5279void ggml_compute_forward_diag_mask_inf(
5280 const ggml_compute_params * params,
5281 ggml_tensor * dst) {
5282
5283 const ggml_tensor * src0 = dst->src[0];
5284
5285 switch (src0->type) {
5286 case GGML_TYPE_F32:
5287 {
5288 ggml_compute_forward_diag_mask_f32(params, dst, -INFINITY(__builtin_inff()));
5289 } break;
5290 default:
5291 {
5292 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5292, "fatal error");
5293 }
5294 }
5295}
5296
5297void ggml_compute_forward_diag_mask_zero(
5298 const ggml_compute_params * params,
5299 ggml_tensor * dst) {
5300
5301 const ggml_tensor * src0 = dst->src[0];
5302
5303 switch (src0->type) {
5304 case GGML_TYPE_F32:
5305 {
5306 ggml_compute_forward_diag_mask_f32(params, dst, 0);
5307 } break;
5308 default:
5309 {
5310 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5310, "fatal error");
5311 }
5312 }
5313}
5314
5315// ggml_compute_forward_soft_max
5316
5317static void ggml_compute_forward_soft_max_f32(
5318 const ggml_compute_params * params,
5319 ggml_tensor * dst) {
5320
5321 const ggml_tensor * src0 = dst->src[0];
5322 const ggml_tensor * src1 = dst->src[1];
5323 const ggml_tensor * src2 = dst->src[2];
5324
5325 assert(ggml_is_contiguous(dst))(static_cast <bool> (ggml_is_contiguous(dst)) ? void (0
) : __assert_fail ("ggml_is_contiguous(dst)", __builtin_FILE (
), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
5326 assert(ggml_are_same_shape(src0, dst))(static_cast <bool> (ggml_are_same_shape(src0, dst)) ? void
(0) : __assert_fail ("ggml_are_same_shape(src0, dst)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
5327
5328 float scale = 1.0f;
5329 float max_bias = 0.0f;
5330
5331 memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));
5332 memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));
5333
5334 const int ith = params->ith;
5335 const int nth = params->nth;
5336
5337 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
5338
5339 const int64_t nb11 = src1 ? src1->nb[1] : 1;
5340 const int64_t nb12 = src1 ? src1->nb[2] : 1;
5341 const int64_t nb13 = src1 ? src1->nb[3] : 1;
5342
5343 const int64_t ne12 = src1 ? src1->ne[2] : 1;
5344 const int64_t ne13 = src1 ? src1->ne[3] : 1;
5345
5346 // TODO: is this supposed to be ceil instead of floor?
5347 // https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L370
5348 const uint32_t n_head = ne02;
5349 const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));
5350
5351 const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
5352 const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
5353
5354 float * wp = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;
5355
5356 const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);
5357
5358 // sinks
5359 const float * sk = src2 ? (float *)((char *) src2->data) : nullptr;
5360
5361 for (int64_t i03 = 0; i03 < ne03; i03++) {
5362 for (int64_t i02 = 0; i02 < ne02; i02++) {
5363 for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
5364 const int64_t i11 = i01;
5365 const int64_t i12 = i02%ne12;
5366 const int64_t i13 = i03%ne13;
5367
5368 // ALiBi
5369 const uint32_t h = i02; // head
5370 const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;
5371
5372 float * sp = (float *)((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
5373 float * dp = (float *)((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
5374
5375 // broadcast the mask across rows
5376 ggml_fp16_t * mp_f16 = src1 ? (ggml_fp16_t *)((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13) : NULL__null;
5377 float * mp_f32 = src1 ? (float *)((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13) : NULL__null;
5378
5379 ggml_vec_cpy_f32 (ne00, wp, sp);
5380 ggml_vec_scale_f32(ne00, wp, scale);
5381 if (mp_f32) {
5382 if (use_f16) {
5383 for (int i = 0; i < ne00; ++i) {
5384 wp[i] += slope*GGML_CPU_FP16_TO_FP32(mp_f16[i])ggml_lookup_fp16_to_fp32(mp_f16[i]);
5385 }
5386 } else {
5387 for (int i = 0; i < ne00; ++i) {
5388 wp[i] += slope*mp_f32[i];
5389 }
5390 }
5391 }
5392
5393#ifndef NDEBUG
5394 for (int i = 0; i < ne00; ++i) {
5395 //printf("p[%d] = %f\n", i, p[i]);
5396 assert(!isnan(wp[i]))(static_cast <bool> (!isnan(wp[i])) ? void (0) : __assert_fail
("!isnan(wp[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
5397 }
5398#endif // NDEBUG
5399
5400 float max = -INFINITY(__builtin_inff());
5401 ggml_vec_max_f32(ne00, &max, wp);
5402
5403 // if we have sinks, make a correction as if they were included in the softmax
5404 if (sk) {
5405 max = MAX(max, sk[i02])((max) > (sk[i02]) ? (max) : (sk[i02]));
5406 }
5407
5408 ggml_float sum = ggml_vec_soft_max_f32(ne00, dp, wp, max);
5409 assert(sum > 0.0)(static_cast <bool> (sum > 0.0) ? void (0) : __assert_fail
("sum > 0.0", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
5410
5411 if (sk) {
5412 sum += (ggml_float) expf(sk[i02] - max);
5413 }
5414
5415 sum = 1.0/sum;
5416 ggml_vec_scale_f32(ne00, dp, sum);
5417
5418#ifndef NDEBUG
5419 for (int i = 0; i < ne00; ++i) {
5420 assert(!isnan(dp[i]))(static_cast <bool> (!isnan(dp[i])) ? void (0) : __assert_fail
("!isnan(dp[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
5421 assert(!isinf(dp[i]))(static_cast <bool> (!isinf(dp[i])) ? void (0) : __assert_fail
("!isinf(dp[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
5422 }
5423#endif // NDEBUG
5424 }
5425 }
5426 }
5427}
5428
5429void ggml_compute_forward_soft_max(
5430 const ggml_compute_params * params,
5431 ggml_tensor * dst) {
5432
5433 const ggml_tensor * src0 = dst->src[0];
5434
5435 switch (src0->type) {
5436 case GGML_TYPE_F32:
5437 {
5438 ggml_compute_forward_soft_max_f32(params, dst);
5439 } break;
5440 default:
5441 {
5442 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5442, "fatal error");
5443 }
5444 }
5445}
5446
5447
5448// ggml_compute_forward_soft_max_ext_back
5449
5450static void ggml_compute_forward_soft_max_ext_back_f32(
5451 const ggml_compute_params * params,
5452 ggml_tensor * dst) {
5453
5454 const ggml_tensor * src0 = dst->src[0];
5455 const ggml_tensor * src1 = dst->src[1];
5456
5457 GGML_ASSERT(ggml_is_contiguous(src0))if (!(ggml_is_contiguous(src0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5457, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(src0)");
5458 GGML_ASSERT(ggml_is_contiguous(src1))if (!(ggml_is_contiguous(src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5458, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(src1)");
5459 GGML_ASSERT(ggml_is_contiguous(dst))if (!(ggml_is_contiguous(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5459, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dst)");
5460 GGML_ASSERT(ggml_are_same_shape(src0, dst))if (!(ggml_are_same_shape(src0, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5460, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, dst)"
);
5461 GGML_ASSERT(ggml_are_same_shape(src1, dst))if (!(ggml_are_same_shape(src1, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5461, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src1, dst)"
);
5462
5463 float scale = 1.0f;
5464 float max_bias = 0.0f;
5465
5466 memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float));
5467 memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));
5468
5469 GGML_ASSERT(max_bias == 0.0f)if (!(max_bias == 0.0f)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5469, "GGML_ASSERT(%s) failed", "max_bias == 0.0f");
5470
5471 // TODO: handle transposed/permuted matrices
5472
5473 const int ith = params->ith;
5474 const int nth = params->nth;
5475
5476 const int nc = src0->ne[0];
5477 const int nr = ggml_nrows(src0);
5478
5479 // rows per thread
5480 const int dr = (nr + nth - 1)/nth;
5481
5482 // row range for this thread
5483 const int ir0 = dr*ith;
5484 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
5485
5486 for (int i1 = ir0; i1 < ir1; i1++) {
5487 float *dy = (float *)((char *) src0->data + i1*src0->nb[1]);
5488 float *y = (float *)((char *) src1->data + i1*src1->nb[1]);
5489 float *dx = (float *)((char *) dst->data + i1*dst->nb[1]);
5490
5491#ifndef NDEBUG
5492 for (int i = 0; i < nc; ++i) {
5493 //printf("p[%d] = %f\n", i, p[i]);
5494 assert(!isnan(dy[i]))(static_cast <bool> (!isnan(dy[i])) ? void (0) : __assert_fail
("!isnan(dy[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
5495 assert(!isnan(y[i]))(static_cast <bool> (!isnan(y[i])) ? void (0) : __assert_fail
("!isnan(y[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
5496 }
5497#endif // NDEBUG
5498 // Jii = yi - yi*yi
5499 // Jij = -yi*yj
5500 // J = diag(y)-y.T*y
5501 // dx = J * dy
5502 // dxk = sum_i(Jki * dyi)
5503 // dxk = sum_i(-yk*yi * dyi) - (-yk*yk)*dyk + (yk - yk*yk)*dyk
5504 // dxk = sum_i(-yk*yi * dyi) + yk*yk*dyk + yk*dyk - yk*yk*dyk
5505 // dxk = sum_i(-yk*yi * dyi) + yk*dyk
5506 // dxk = -yk * sum_i(yi * dyi) + yk*dyk
5507 // dxk = -yk * dot(y, dy) + yk*dyk
5508 // dxk = yk * (- dot(y, dy) + dyk)
5509 // dxk = yk * (dyk - dot(y, dy))
5510 //
5511 // post-order:
5512 // dot_y_dy := dot(y, dy)
5513 // dx := dy
5514 // dx := dx - dot_y_dy
5515 // dx := dx * y
5516
5517 // linear runtime, no additional memory
5518 float dot_y_dy = 0;
5519 ggml_vec_dot_f32 (nc, &dot_y_dy, 0, y, 0, dy, 0, 1);
5520 ggml_vec_cpy_f32 (nc, dx, dy);
5521 ggml_vec_acc1_f32 (nc, dx, -dot_y_dy);
5522 ggml_vec_mul_f32 (nc, dx, dx, y);
5523 ggml_vec_scale_f32(nc, dx, scale);
5524
5525#ifndef NDEBUG
5526 for (int i = 0; i < nc; ++i) {
5527 assert(!isnan(dx[i]))(static_cast <bool> (!isnan(dx[i])) ? void (0) : __assert_fail
("!isnan(dx[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
5528 assert(!isinf(dx[i]))(static_cast <bool> (!isinf(dx[i])) ? void (0) : __assert_fail
("!isinf(dx[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
5529 }
5530#endif // NDEBUG
5531 }
5532}
5533
5534void ggml_compute_forward_soft_max_ext_back(
5535 const ggml_compute_params * params,
5536 ggml_tensor * dst) {
5537
5538 const ggml_tensor * src0 = dst->src[0];
5539
5540 switch (src0->type) {
5541 case GGML_TYPE_F32:
5542 {
5543 ggml_compute_forward_soft_max_ext_back_f32(params, dst);
5544 } break;
5545 default:
5546 {
5547 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5547, "fatal error");
5548 }
5549 }
5550}
5551
5552// ggml_compute_forward_clamp
5553
5554static void ggml_compute_forward_clamp_f32(
5555 const ggml_compute_params * params,
5556 ggml_tensor * dst) {
5557
5558 const ggml_tensor * src0 = dst->src[0];
5559
5560 float min;
5561 float max;
5562 memcpy(&min, (float *) dst->op_params + 0, sizeof(float));
5563 memcpy(&max, (float *) dst->op_params + 1, sizeof(float));
5564
5565 const int ith = params->ith;
5566 const int nth = params->nth;
5567
5568 const int n = ggml_nrows(src0);
5569 const int nc = src0->ne[0];
5570
5571 const size_t nb00 = src0->nb[0];
5572 const size_t nb01 = src0->nb[1];
5573
5574 const size_t nb0 = dst->nb[0];
5575 const size_t nb1 = dst->nb[1];
5576
5577 GGML_ASSERT( nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5577, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
5578 GGML_ASSERT(nb00 == sizeof(float))if (!(nb00 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5578, "GGML_ASSERT(%s) failed", "nb00 == sizeof(float)");
5579
5580 for (int j = ith; j < n; j += nth) {
5581 float * dst_ptr = (float *) ((char *) dst->data + j*nb1);
5582 float * src0_ptr = (float *) ((char *) src0->data + j*nb01);
5583
5584 for (int i = 0; i < nc; i++) {
5585 dst_ptr[i] = MAX(MIN(src0_ptr[i], max), min)((((src0_ptr[i]) < (max) ? (src0_ptr[i]) : (max))) > (min
) ? (((src0_ptr[i]) < (max) ? (src0_ptr[i]) : (max))) : (min
));
5586 }
5587 }
5588}
5589
5590static void ggml_compute_forward_clamp_f16(
5591 const ggml_compute_params * params,
5592 ggml_tensor * dst) {
5593
5594 const ggml_tensor * src0 = dst->src[0];
5595
5596 float min;
5597 float max;
5598 memcpy(&min, (float *) dst->op_params + 0, sizeof(float));
5599 memcpy(&max, (float *) dst->op_params + 1, sizeof(float));
5600
5601 const int ith = params->ith;
5602 const int nth = params->nth;
5603
5604 const int n = ggml_nrows(src0);
5605 const int nc = src0->ne[0];
5606
5607 const size_t nb00 = src0->nb[0];
5608 const size_t nb01 = src0->nb[1];
5609
5610 const size_t nb0 = dst->nb[0];
5611 const size_t nb1 = dst->nb[1];
5612
5613 GGML_ASSERT( nb0 == sizeof(ggml_fp16_t))if (!(nb0 == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5613, "GGML_ASSERT(%s) failed", "nb0 == sizeof(ggml_fp16_t)"
);
5614 GGML_ASSERT(nb00 == sizeof(ggml_fp16_t))if (!(nb00 == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5614, "GGML_ASSERT(%s) failed", "nb00 == sizeof(ggml_fp16_t)"
);
5615
5616 for (int j = ith; j < n; j += nth) {
5617 ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + j*nb1);
5618 ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + j*nb01);
5619
5620 for (int i = 0; i < nc; i++) {
5621 float v = GGML_CPU_FP16_TO_FP32(src0_ptr[i])ggml_lookup_fp16_to_fp32(src0_ptr[i]);
5622 dst_ptr[i] = GGML_CPU_FP32_TO_FP16(MAX(MIN(v, max), min))ggml_compute_fp32_to_fp16(((((v) < (max) ? (v) : (max))) >
(min) ? (((v) < (max) ? (v) : (max))) : (min)));
5623 }
5624 }
5625}
5626
5627void ggml_compute_forward_clamp(
5628 const ggml_compute_params * params,
5629 ggml_tensor * dst) {
5630
5631 const ggml_tensor * src0 = dst->src[0];
5632
5633 switch (src0->type) {
5634 case GGML_TYPE_F32:
5635 {
5636 ggml_compute_forward_clamp_f32(params, dst);
5637 } break;
5638 case GGML_TYPE_F16:
5639 {
5640 ggml_compute_forward_clamp_f16(params, dst);
5641 } break;
5642 case GGML_TYPE_BF16:
5643 case GGML_TYPE_Q1_0:
5644 case GGML_TYPE_Q4_0:
5645 case GGML_TYPE_Q4_1:
5646 case GGML_TYPE_Q5_0:
5647 case GGML_TYPE_Q5_1:
5648 case GGML_TYPE_Q8_0:
5649 case GGML_TYPE_Q8_1:
5650 case GGML_TYPE_MXFP4:
5651 case GGML_TYPE_NVFP4:
5652 case GGML_TYPE_Q2_K:
5653 case GGML_TYPE_Q3_K:
5654 case GGML_TYPE_Q4_K:
5655 case GGML_TYPE_Q5_K:
5656 case GGML_TYPE_Q6_K:
5657 case GGML_TYPE_TQ1_0:
5658 case GGML_TYPE_TQ2_0:
5659 case GGML_TYPE_IQ2_XXS:
5660 case GGML_TYPE_IQ2_XS:
5661 case GGML_TYPE_IQ3_XXS:
5662 case GGML_TYPE_IQ1_S:
5663 case GGML_TYPE_IQ1_M:
5664 case GGML_TYPE_IQ4_NL:
5665 case GGML_TYPE_IQ4_XS:
5666 case GGML_TYPE_IQ3_S:
5667 case GGML_TYPE_IQ2_S:
5668 case GGML_TYPE_Q8_K:
5669 case GGML_TYPE_I8:
5670 case GGML_TYPE_I16:
5671 case GGML_TYPE_I32:
5672 case GGML_TYPE_I64:
5673 case GGML_TYPE_F64:
5674 case GGML_TYPE_COUNT:
5675 {
5676 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5676, "fatal error");
5677 }
5678 }
5679}
5680
5681// ggml_compute_forward_rope
5682
5683static float rope_yarn_ramp(const float low, const float high, const int i0) {
5684 const float y = (i0 / 2 - low) / MAX(0.001f, high - low)((0.001f) > (high - low) ? (0.001f) : (high - low));
5685 return 1 - MIN(1, MAX(0, y))((1) < (((0) > (y) ? (0) : (y))) ? (1) : (((0) > (y)
? (0) : (y))));
5686}
5687
5688// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn
5689// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.
5690static void rope_yarn(
5691 float theta_extrap, float freq_scale, float corr_dims[2], int64_t i0, float ext_factor, float mscale,
5692 float * cos_theta, float * sin_theta) {
5693 // Get n-d rotational scaling corrected for extrapolation
5694 float theta_interp = freq_scale * theta_extrap;
5695 float theta = theta_interp;
5696 if (ext_factor != 0.0f) {
5697 float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor;
5698 theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;
5699
5700 // Get n-d magnitude scaling corrected for interpolation
5701 mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);
5702 }
5703 *cos_theta = cosf(theta) * mscale;
5704 *sin_theta = sinf(theta) * mscale;
5705}
5706
5707static void ggml_rope_cache_init(
5708 float theta_base, float freq_scale, const float * freq_factors, float corr_dims[2], int64_t ne0, float ext_factor, float mscale,
5709 float * cache, float sin_sign, float theta_scale) {
5710 // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py
5711 float theta = theta_base;
5712 for (int64_t i0 = 0; i0 < ne0; i0 += 2) {
5713 const float ff = freq_factors ? freq_factors[i0/2] : 1.0f;
5714 rope_yarn(
5715 theta/ff, freq_scale, corr_dims, i0, ext_factor, mscale, &cache[i0 + 0], &cache[i0 + 1]
5716 );
5717 cache[i0 + 1] *= sin_sign;
5718
5719 theta *= theta_scale;
5720 }
5721}
5722
5723static void ggml_mrope_cache_init(
5724 float theta_base_t, float theta_base_h, float theta_base_w, float theta_base_e, int sections[4], bool is_imrope, bool indep_sects,
5725 float freq_scale, const float * freq_factors, float corr_dims[2], int64_t ne0, float ext_factor, float mscale,
5726 float * cache, float sin_sign, float theta_scale) {
5727 // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py
5728 float theta_t = theta_base_t;
5729 float theta_h = theta_base_h;
5730 float theta_w = theta_base_w;
5731 float theta_e = theta_base_e; // extra position id for vision encoder
5732 int sect_dims = sections[0] + sections[1] + sections[2] + sections[3];
5733 int sec_w = sections[1] + sections[0];
5734 int sec_e = sections[2] + sec_w;
5735 GGML_ASSERT(sect_dims <= ne0)if (!(sect_dims <= ne0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5735, "GGML_ASSERT(%s) failed", "sect_dims <= ne0");
5736
5737 for (int64_t i0 = 0; i0 < ne0; i0 += 2) {
5738 const float ff = freq_factors ? freq_factors[i0/2] : 1.0f;
5739
5740 int sector = (i0 / 2) % sect_dims;
5741 if (indep_sects) {
5742 // compute theta independently for each dim sections
5743 // (i.e. reset corresponding theta when `i0` go from one section to another)
5744 if (sector == 0) {
5745 theta_t = theta_base_t;
5746 }
5747 else if (sector == sections[0]) {
5748 theta_h = theta_base_h;;
5749 }
5750 else if (sector == sec_w) {
5751 theta_w = theta_base_w;
5752 }
5753 else if (sector == sec_e) {
5754 theta_e = theta_base_e;
5755 }
5756 }
5757
5758 float theta = theta_t;
5759 if (is_imrope) { // qwen3vl apply interleaved mrope
5760 if (sector % 3 == 1 && sector < 3 * sections[1]) {
5761 theta = theta_h;
5762 } else if (sector % 3 == 2 && sector < 3 * sections[2]) {
5763 theta = theta_w;
5764 } else if (sector % 3 == 0 && sector < 3 * sections[0]) {
5765 theta = theta_t;
5766 } else {
5767 theta = theta_e;
5768 }
5769 } else {
5770 if (sector >= sections[0] && sector < sec_w) {
5771 theta = theta_h;
5772 }
5773 else if (sector >= sec_w && sector < sec_w + sections[2]) {
5774 theta = theta_w;
5775 }
5776 else if (sector >= sec_w + sections[2]) {
5777 theta = theta_e;
5778 }
5779 }
5780
5781 rope_yarn(
5782 theta/ff, freq_scale, corr_dims, i0, ext_factor, mscale, &cache[i0 + 0], &cache[i0 + 1]
5783 );
5784 cache[i0 + 1] *= sin_sign;
5785
5786 theta_t *= theta_scale;
5787 theta_w *= theta_scale;
5788 theta_h *= theta_scale;
5789 theta_e *= theta_scale;
5790 }
5791}
5792
5793
5794template<typename T>
5795static void rotate_pairs(const int64_t n, const int64_t n_offset, const float * cache, const T * src_data, T * dst_data, const int scale = 2) {
5796 for (int64_t i0 = 0; i0 < n; i0 += 2) {
5797 const int64_t ic = i0/scale; // hack for GGML_ROPE_TYPE_NORMAL, where we need ic = i0; for all other cases, ic = i0/2
5798
5799 const float cos_theta = cache[i0 + 0];
5800 const float sin_theta = cache[i0 + 1];
5801
5802 const T * const src = src_data + ic;
5803 T * dst = dst_data + ic;
5804
5805 const float x0 = type_conversion_table<T>::to_f32(src[0]);
5806 const float x1 = type_conversion_table<T>::to_f32(src[n_offset]);
5807
5808 dst[0] = type_conversion_table<T>::from_f32(x0*cos_theta - x1*sin_theta);
5809 dst[n_offset] = type_conversion_table<T>::from_f32(x0*sin_theta + x1*cos_theta);
5810 }
5811}
5812
5813template<typename T> //float or ggml_fp16_t
5814static void ggml_compute_forward_rope_flt(
5815 const ggml_compute_params * params,
5816 ggml_tensor * dst,
5817 const bool forward) {
5818
5819 const ggml_tensor * src0 = dst->src[0];
5820 const ggml_tensor * src1 = dst->src[1];
5821 const ggml_tensor * src2 = dst->src[2];
5822
5823 GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)if (!(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5823, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16"
);
5824 GGML_ASSERT(src1->type == GGML_TYPE_I32)if (!(src1->type == GGML_TYPE_I32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5824, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_I32"
);
5825
5826 float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
5827 int sections[4];
5828
5829 //const int n_past = ((int32_t *) dst->op_params)[0];
5830 const int n_dims = ((int32_t *) dst->op_params)[1];
5831 const int mode = ((int32_t *) dst->op_params)[2];
5832 //const int n_ctx = ((int32_t *) dst->op_params)[3];
5833 const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
5834
5835 memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
5836 memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
5837 memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
5838 memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
5839 memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
5840 memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
5841 memcpy(&sections, (int32_t *) dst->op_params + 11, sizeof(int)*4);
5842
5843 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
5844
5845 //printf("ne0: %d, ne1: %d, ne2: %d, ne3: %d\n", ne0, ne1, ne2, ne3);
5846 //printf("n_past = %d, ne2 = %d\n", n_past, ne2);
5847
5848 GGML_ASSERT(nb0 == nb00)if (!(nb0 == nb00)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5848, "GGML_ASSERT(%s) failed", "nb0 == nb00");
5849 GGML_ASSERT(nb0 == sizeof(T))if (!(nb0 == sizeof(T))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5849, "GGML_ASSERT(%s) failed", "nb0 == sizeof(T)");
5850
5851 const int ith = params->ith;
5852 const int nth = params->nth;
5853
5854 const int nr = ggml_nrows(dst);
5855
5856 GGML_ASSERT(n_dims <= ne0)if (!(n_dims <= ne0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5856, "GGML_ASSERT(%s) failed", "n_dims <= ne0");
5857 GGML_ASSERT(n_dims % 2 == 0)if (!(n_dims % 2 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5857, "GGML_ASSERT(%s) failed", "n_dims % 2 == 0");
5858
5859 // rows per thread
5860 const int dr = (nr + nth - 1)/nth;
5861
5862 // row range for this thread
5863 const int ir0 = dr*ith;
5864 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
5865
5866 // row index used to determine which thread to use
5867 int ir = 0;
5868
5869 const float theta_scale = powf(freq_base, -2.0f/n_dims);
5870
5871 float corr_dims[2];
5872 ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
5873
5874 const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE40; // qwen3vl apply interleaved mrope
5875 const bool mrope_used = mode & GGML_ROPE_TYPE_MROPE8; // ggml_rope_multi, note: also true for vision (24 & 8 == true) and for imrope
5876 const bool is_vision = mode == GGML_ROPE_TYPE_VISION24;
5877
5878 if (mrope_used) {
5879 GGML_ASSERT(sections[0] > 0 || sections[1] > 0 || sections[2] > 0)if (!(sections[0] > 0 || sections[1] > 0 || sections[2]
> 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5879, "GGML_ASSERT(%s) failed", "sections[0] > 0 || sections[1] > 0 || sections[2] > 0"
);
5880 }
5881
5882 if (is_vision) {
5883 GGML_ASSERT(n_dims == ne0/2)if (!(n_dims == ne0/2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5883, "GGML_ASSERT(%s) failed", "n_dims == ne0/2");
5884 }
5885
5886 const float * freq_factors = NULL__null;
5887 if (src2 != NULL__null) {
5888 GGML_ASSERT(src2->type == GGML_TYPE_F32)if (!(src2->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5888, "GGML_ASSERT(%s) failed", "src2->type == GGML_TYPE_F32"
);
5889 GGML_ASSERT(src2->ne[0] >= n_dims / 2)if (!(src2->ne[0] >= n_dims / 2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5889, "GGML_ASSERT(%s) failed", "src2->ne[0] >= n_dims / 2"
);
5890 freq_factors = (const float *) src2->data;
5891 }
5892
5893 // backward process uses inverse rotation by cos and sin.
5894 // cos and sin build a rotation matrix, where the inverse is the transpose.
5895 // this essentially just switches the sign of sin.
5896 const float sin_sign = forward ? 1.0f : -1.0f;
5897
5898 const int32_t * pos = (const int32_t *) src1->data;
5899
5900 int64_t last_i2 = -1;
5901
5902 for (int64_t i3 = 0; i3 < ne3; i3++) { // batch
5903 for (int64_t i2 = 0; i2 < ne2; i2++) { // seq-len
5904 for (int64_t i1 = 0; i1 < ne1; i1++) { // attn-heads
5905 if (ir++ < ir0) continue; // skip rows mapped to other threads
5906 if (ir > ir1) break;
5907
5908 float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith;
5909 if (last_i2 != i2) {
5910 if (!mrope_used) {
5911 const int64_t p = pos[i2];
5912 ggml_rope_cache_init(p, freq_scale, freq_factors, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);
5913 }
5914 else {
5915 const int64_t p_t = pos[i2];
5916 const int64_t p_h = pos[i2 + ne2];
5917 const int64_t p_w = pos[i2 + ne2 * 2];
5918 const int64_t p_e = pos[i2 + ne2 * 3];
5919 ggml_mrope_cache_init(
5920 p_t, p_h, p_w, p_e, sections, is_imrope, is_vision,
5921 freq_scale, freq_factors, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);
5922 }
5923
5924 last_i2 = i2;
5925 }
5926
5927 T * src = (T *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
5928 T * dst_data = (T *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
5929
5930 switch (mode) {
5931 case GGML_ROPE_TYPE_NORMAL0:
5932 rotate_pairs<T>(n_dims, 1, cache, src, dst_data, 1);
5933 break;
5934 case GGML_ROPE_TYPE_NEOX2:
5935 case GGML_ROPE_TYPE_MROPE8:
5936 case GGML_ROPE_TYPE_IMROPE40:
5937 rotate_pairs<T>(n_dims, n_dims/2, cache, src, dst_data);
5938 break;
5939 case GGML_ROPE_TYPE_VISION24:
5940 rotate_pairs<T>(ne0, n_dims, cache, src, dst_data);
5941 break;
5942 default:
5943 GGML_ABORT("rope type not supported")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5943, "rope type not supported");
5944 }
5945
5946 if (!is_vision) {
5947 // fill the remain channels with data from src tensor
5948 for (int64_t i0 = n_dims; i0 < ne0; i0 += 2) {
5949 const T * const src = (T *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
5950 T * dst_data = (T *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
5951
5952 dst_data[0] = src[0];
5953 dst_data[1] = src[1];
5954 }
5955 }
5956 } //attn-heads
5957 }
5958 }
5959}
5960
5961void ggml_compute_forward_rope(
5962 const ggml_compute_params * params,
5963 ggml_tensor * dst) {
5964
5965 const ggml_tensor * src0 = dst->src[0];
5966
5967 switch (src0->type) {
5968 case GGML_TYPE_F16:
5969 {
5970 ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, true);
5971 } break;
5972 case GGML_TYPE_F32:
5973 {
5974 ggml_compute_forward_rope_flt<float>(params, dst, true);
5975 } break;
5976 default:
5977 {
5978 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 5978, "fatal error");
5979 }
5980 }
5981}
5982
5983// ggml_compute_forward_rope_back
5984
5985void ggml_compute_forward_rope_back(
5986 const ggml_compute_params * params,
5987 ggml_tensor * dst) {
5988
5989 const ggml_tensor * src0 = dst->src[0];
5990
5991 switch (src0->type) {
5992 case GGML_TYPE_F16:
5993 {
5994 ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, false);
5995 } break;
5996 case GGML_TYPE_F32:
5997 {
5998 ggml_compute_forward_rope_flt<float>(params, dst, false);
5999 } break;
6000 default:
6001 {
6002 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6002, "fatal error");
6003 }
6004 }
6005}
6006
6007// ggml_compute_forward_conv_transpose_1d
6008
6009static void ggml_compute_forward_conv_transpose_1d_f16_f32(
6010 const ggml_compute_params * params,
6011 ggml_tensor * dst) {
6012
6013 const ggml_tensor * src0 = dst->src[0];
6014 const ggml_tensor * src1 = dst->src[1];
6015
6016 GGML_ASSERT(src0->type == GGML_TYPE_F16)if (!(src0->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6016, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F16"
);
6017 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6017, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
6018 GGML_ASSERT( dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6018, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
6019
6020 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
6021
6022 const int ith = params->ith;
6023 const int nth = params->nth;
6024
6025 const int nk = ne00*ne01*ne02;
6026
6027 GGML_ASSERT(nb00 == sizeof(ggml_fp16_t))if (!(nb00 == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6027, "GGML_ASSERT(%s) failed", "nb00 == sizeof(ggml_fp16_t)"
);
6028 GGML_ASSERT(nb10 == sizeof(float))if (!(nb10 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6028, "GGML_ASSERT(%s) failed", "nb10 == sizeof(float)");
6029
6030 if (ith == 0) {
6031 memset(params->wdata, 0, params->wsize);
6032
6033 // permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)
6034 {
6035 ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;
6036
6037 for (int64_t i02 = 0; i02 < ne02; i02++) {
6038 for (int64_t i01 = 0; i01 < ne01; i01++) {
6039 const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i02*nb02 + i01*nb01);
6040 ggml_fp16_t * dst_data = wdata + i01*ne00*ne02;
6041 for (int64_t i00 = 0; i00 < ne00; i00++) {
6042 dst_data[i00*ne02 + i02] = src[i00];
6043 }
6044 }
6045 }
6046 }
6047
6048 // permute source data (src1) from (L x Cin) to (Cin x L)
6049 {
6050 ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk;
6051 ggml_fp16_t * dst_data = wdata;
6052
6053 for (int64_t i11 = 0; i11 < ne11; i11++) {
6054 const float * const src = (float *)((char *) src1->data + i11*nb11);
6055 for (int64_t i10 = 0; i10 < ne10; i10++) {
6056 dst_data[i10*ne11 + i11] = GGML_CPU_FP32_TO_FP16(src[i10])ggml_compute_fp32_to_fp16(src[i10]);
6057 }
6058 }
6059 }
6060
6061 // need to zero dst since we are accumulating into it
6062 memset(dst->data, 0, ggml_nbytes(dst));
6063 }
6064 ggml_barrier(params->threadpool);
6065
6066 const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
6067
6068 // total rows in dst
6069 const int nr = ne1;
6070
6071 // rows per thread
6072 const int dr = (nr + nth - 1)/nth;
6073
6074 // row range for this thread
6075 const int ir0 = dr*ith;
6076 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
6077
6078 ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;
6079 ggml_fp16_t * const wdata_src = wdata + nk;
6080
6081 for (int i1 = ir0; i1 < ir1; i1++) {
6082 float * dst_data = (float *)((char *) dst->data + i1*nb1);
6083 ggml_fp16_t * wdata_kernel = wdata + i1*ne02*ne00;
6084 for (int i10 = 0; i10 < ne10; i10++) {
6085 const int i1n = i10*ne11;
6086 for (int i00 = 0; i00 < ne00; i00++) {
6087 float v = 0;
6088 ggml_vec_dot_f16(ne02, &v, 0,
6089 (ggml_fp16_t *) wdata_src + i1n, 0,
6090 (ggml_fp16_t *) wdata_kernel + i00*ne02, 0, 1);
6091 dst_data[i10*s0 + i00] += v;
6092 }
6093 }
6094 }
6095}
6096
6097static void ggml_compute_forward_conv_transpose_1d_f32(
6098 const ggml_compute_params * params,
6099 ggml_tensor * dst) {
6100
6101 const ggml_tensor * src0 = dst->src[0];
6102 const ggml_tensor * src1 = dst->src[1];
6103
6104 GGML_ASSERT(src0->type == GGML_TYPE_F32)if (!(src0->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6104, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F32"
);
6105 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6105, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
6106 GGML_ASSERT( dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6106, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
6107
6108 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
6109
6110 const int ith = params->ith;
6111 const int nth = params->nth;
6112
6113 const int nk = ne00*ne01*ne02;
6114
6115 GGML_ASSERT(nb00 == sizeof(float))if (!(nb00 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6115, "GGML_ASSERT(%s) failed", "nb00 == sizeof(float)");
6116 GGML_ASSERT(nb10 == sizeof(float))if (!(nb10 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6116, "GGML_ASSERT(%s) failed", "nb10 == sizeof(float)");
6117
6118 if (ith == 0) {
6119 memset(params->wdata, 0, params->wsize);
6120
6121 // prepare kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)
6122 {
6123 float * const wdata = (float *) params->wdata + 0;
6124
6125 for (int64_t i02 = 0; i02 < ne02; i02++) {
6126 for (int64_t i01 = 0; i01 < ne01; i01++) {
6127 const float * const src = (float *)((char *) src0->data + i02*nb02 + i01*nb01);
6128 float * dst_data = wdata + i01*ne00*ne02;
6129 for (int64_t i00 = 0; i00 < ne00; i00++) {
6130 dst_data[i00*ne02 + i02] = src[i00];
6131 }
6132 }
6133 }
6134 }
6135
6136 // prepare source data (src1)
6137 {
6138 float * const wdata = (float *) params->wdata + nk;
6139 float * dst_data = wdata;
6140
6141 for (int64_t i11 = 0; i11 < ne11; i11++) {
6142 const float * const src = (float *)((char *) src1->data + i11*nb11);
6143 for (int64_t i10 = 0; i10 < ne10; i10++) {
6144 dst_data[i10*ne11 + i11] = src[i10];
6145 }
6146 }
6147 }
6148
6149 // need to zero dst since we are accumulating into it
6150 memset(dst->data, 0, ggml_nbytes(dst));
6151 }
6152 ggml_barrier(params->threadpool);
6153
6154 const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
6155
6156 // total rows in dst
6157 const int nr = ne1;
6158
6159 // rows per thread
6160 const int dr = (nr + nth - 1)/nth;
6161
6162 // row range for this thread
6163 const int ir0 = dr*ith;
6164 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
6165
6166 float * const wdata = (float *) params->wdata + 0;
6167 float * const wdata_src = wdata + nk;
6168
6169 for (int i1 = ir0; i1 < ir1; i1++) {
6170 float * dst_data = (float *)((char *) dst->data + i1*nb1);
6171 float * wdata_kernel = wdata + i1*ne02*ne00;
6172 for (int i10 = 0; i10 < ne10; i10++) {
6173 const int i1n = i10*ne11;
6174 for (int i00 = 0; i00 < ne00; i00++) {
6175 float v = 0;
6176 ggml_vec_dot_f32(ne02, &v, 0,
6177 wdata_src + i1n, 0,
6178 wdata_kernel + i00*ne02, 0, 1);
6179 dst_data[i10*s0 + i00] += v;
6180 }
6181 }
6182 }
6183}
6184
6185void ggml_compute_forward_conv_transpose_1d(
6186 const ggml_compute_params * params,
6187 ggml_tensor * dst) {
6188
6189 const ggml_tensor * src0 = dst->src[0];
6190
6191 switch (src0->type) {
6192 case GGML_TYPE_F16:
6193 {
6194 ggml_compute_forward_conv_transpose_1d_f16_f32(params, dst);
6195 } break;
6196 case GGML_TYPE_F32:
6197 {
6198 ggml_compute_forward_conv_transpose_1d_f32(params, dst);
6199 } break;
6200 default:
6201 {
6202 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6202, "fatal error");
6203 }
6204 }
6205}
6206
6207// ggml_compute_forward_im2col_f32
6208// src0: kernel [OC, IC, KH, KW]
6209// src1: image [N, IC, IH, IW]
6210// dst: result [N, OH, OW, IC*KH*KW]
6211static void ggml_compute_forward_im2col_f32(
6212 const ggml_compute_params * params,
6213 ggml_tensor * dst) {
6214
6215 const ggml_tensor * src0 = dst->src[0];
6216 const ggml_tensor * src1 = dst->src[1];
6217
6218 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6218, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
6219 GGML_ASSERT( dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6219, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
6220
6221 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);;
6222
6223 const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
6224 const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
6225 const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
6226 const int32_t p1 = ((const int32_t *)(dst->op_params))[3];
6227 const int32_t d0 = ((const int32_t *)(dst->op_params))[4];
6228 const int32_t d1 = ((const int32_t *)(dst->op_params))[5];
6229 const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;
6230
6231 const int ith = params->ith;
6232 const int nth = params->nth;
6233
6234 const int64_t N = is_2D ? ne13 : ne12;
6235 const int64_t IC = is_2D ? ne12 : ne11;
6236 const int64_t IH = is_2D ? ne11 : 1;
6237 const int64_t IW = ne10;
6238
6239 const int64_t KH = is_2D ? ne01 : 1;
6240 const int64_t KW = ne00;
6241
6242 const int64_t OH = is_2D ? ne2 : 1;
6243 const int64_t OW = ne1;
6244
6245 int ofs0 = is_2D ? nb13 : nb12;
6246 int ofs1 = is_2D ? nb12 : nb11;
6247
6248 GGML_ASSERT(nb10 == sizeof(float))if (!(nb10 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6248, "GGML_ASSERT(%s) failed", "nb10 == sizeof(float)");
6249
6250 // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
6251 {
6252 float * const wdata = (float *) dst->data;
6253
6254 for (int64_t in = 0; in < N; in++) {
6255 for (int64_t ioh = 0; ioh < OH; ioh++) { // 1
6256 for (int64_t iow = 0; iow < OW; iow++) {
6257 for (int64_t iic = ith; iic < IC; iic += nth) {
6258
6259 // micro kernel
6260 float * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
6261 const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]
6262
6263 for (int64_t ikh = 0; ikh < KH; ikh++) { // 1
6264 for (int64_t ikw = 0; ikw < KW; ikw++) {
6265 const int64_t iiw = iow*s0 + ikw*d0 - p0;
6266 const int64_t iih = ioh*s1 + ikh*d1 - p1;
6267
6268 if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
6269 dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;
6270 } else {
6271 dst_data[iic*(KH*KW) + ikh*KW + ikw] = (src_data[iih*IW + iiw]);
6272 }
6273 }
6274 }
6275 }
6276 }
6277 }
6278 }
6279 }
6280}
6281
6282
6283// ggml_compute_forward_im2col_f16
6284// src0: kernel [OC, IC, KH, KW]
6285// src1: image [N, IC, IH, IW]
6286// dst: result [N, OH, OW, IC*KH*KW]
6287static void ggml_compute_forward_im2col_f16(
6288 const ggml_compute_params * params,
6289 ggml_tensor * dst) {
6290
6291 const ggml_tensor * src0 = dst->src[0];
6292 const ggml_tensor * src1 = dst->src[1];
6293
6294 GGML_ASSERT(src0->type == GGML_TYPE_F16)if (!(src0->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6294, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F16"
);
6295 GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6295, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32"
);
6296 GGML_ASSERT( dst->type == GGML_TYPE_F16)if (!(dst->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6296, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F16"
);
6297
6298 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);;
6299
6300 const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
6301 const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
6302 const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
6303 const int32_t p1 = ((const int32_t *)(dst->op_params))[3];
6304 const int32_t d0 = ((const int32_t *)(dst->op_params))[4];
6305 const int32_t d1 = ((const int32_t *)(dst->op_params))[5];
6306 const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;
6307
6308 const int ith = params->ith;
6309 const int nth = params->nth;
6310
6311 const int64_t N = is_2D ? ne13 : ne12;
6312 const int64_t IC = is_2D ? ne12 : ne11;
6313 const int64_t IH = is_2D ? ne11 : 1;
6314 const int64_t IW = ne10;
6315
6316 const int64_t KH = is_2D ? ne01 : 1;
6317 const int64_t KW = ne00;
6318
6319 const int64_t OH = is_2D ? ne2 : 1;
6320 const int64_t OW = ne1;
6321
6322 int ofs0 = is_2D ? nb13 : nb12;
6323 int ofs1 = is_2D ? nb12 : nb11;
6324
6325 GGML_ASSERT(nb00 == sizeof(ggml_fp16_t))if (!(nb00 == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6325, "GGML_ASSERT(%s) failed", "nb00 == sizeof(ggml_fp16_t)"
);
6326 GGML_ASSERT(nb10 == ggml_type_size(src1->type))if (!(nb10 == ggml_type_size(src1->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6326, "GGML_ASSERT(%s) failed", "nb10 == ggml_type_size(src1->type)"
);
6327
6328 // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
6329 {
6330 ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;
6331
6332 for (int64_t in = 0; in < N; in++) {
6333 for (int64_t ioh = 0; ioh < OH; ioh++) { // 1
6334 for (int64_t iow = 0; iow < OW; iow++) {
6335 for (int64_t iic = ith; iic < IC; iic += nth) {
6336
6337 // micro kernel
6338 ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
6339 const float * const src_data_f32 = src1->type == GGML_TYPE_F32
6340 ? (const float *)((const char *) src1->data + in*ofs0 + iic*ofs1)
6341 : nullptr; // [IH, IW]
6342 const ggml_fp16_t * const src_data_f16 = src1->type == GGML_TYPE_F16
6343 ? (const ggml_fp16_t *)((const char *) src1->data + in*ofs0 + iic*ofs1)
6344 : nullptr; // [IH, IW]
6345
6346 for (int64_t ikh = 0; ikh < KH; ikh++) { // 1
6347 for (int64_t ikw = 0; ikw < KW; ikw++) {
6348 const int64_t iiw = iow*s0 + ikw*d0 - p0;
6349 const int64_t iih = ioh*s1 + ikh*d1 - p1;
6350
6351 if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
6352 dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;
6353 } else {
6354 if (src_data_f32 != nullptr) {
6355 dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_CPU_FP32_TO_FP16(src_data_f32[iih*IW + iiw])ggml_compute_fp32_to_fp16(src_data_f32[iih*IW + iiw]);
6356 } else {
6357 dst_data[iic*(KH*KW) + ikh*KW + ikw] = src_data_f16[iih*IW + iiw];
6358 }
6359 }
6360 }
6361 }
6362 }
6363 }
6364 }
6365 }
6366 }
6367}
6368
6369void ggml_compute_forward_im2col(
6370 const ggml_compute_params * params,
6371 ggml_tensor * dst) {
6372 switch (dst->type) {
6373 case GGML_TYPE_F16:
6374 {
6375 ggml_compute_forward_im2col_f16(params, dst);
6376 } break;
6377 case GGML_TYPE_F32:
6378 {
6379 ggml_compute_forward_im2col_f32(params, dst);
6380 } break;
6381 default:
6382 {
6383 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6383, "fatal error");
6384 }
6385 }
6386}
6387
6388// ggml_compute_forward_im2col_back_f32
6389
6390void ggml_compute_forward_im2col_back_f32(
6391 const ggml_compute_params * params,
6392 ggml_tensor * dst) {
6393
6394 const ggml_tensor * src0 = dst->src[0]; // gradients of forward pass output
6395 const ggml_tensor * src1 = dst->src[1]; // convolution kernel
6396
6397 GGML_ASSERT(src0->type == GGML_TYPE_F32)if (!(src0->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6397, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F32"
);
6398 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6398, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
6399 GGML_ASSERT( dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6399, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
6400
6401 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);;
6402
6403 const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
6404 const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
6405 const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
6406 const int32_t p1 = ((const int32_t *)(dst->op_params))[3];
6407 const int32_t d0 = ((const int32_t *)(dst->op_params))[4];
6408 const int32_t d1 = ((const int32_t *)(dst->op_params))[5];
6409 const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;
6410
6411 const int ith = params->ith;
6412 const int nth = params->nth;
6413
6414 const int64_t N = is_2D ? ne3 : ne2;
6415 const int64_t IC = is_2D ? ne2 : ne1;
6416 const int64_t IH = is_2D ? ne1 : 1;
6417 const int64_t IW = ne0;
6418
6419 const int64_t KH = is_2D ? ne11 : 1;
6420 const int64_t KW = ne10;
6421
6422 const int64_t OH = is_2D ? ne02 : 1;
6423 const int64_t OW = ne01;
6424
6425 int ofs0 = is_2D ? nb3 : nb2;
6426 int ofs1 = is_2D ? nb2 : nb1;
6427
6428 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6428, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
6429
6430 // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
6431 {
6432 float * const wdata = (float *) dst->data;
6433
6434 for (int64_t in = 0; in < N; in++) {
6435 for (int64_t iic = ith; iic < IC; iic += nth) {
6436 for (int64_t iih = 0; iih < IH; iih++) {
6437 for (int64_t iiw = 0; iiw < IW; iiw++) {
6438
6439 // micro kernel
6440 float grad = 0.0f;
6441 for (int64_t ikh = 0; ikh < KH; ikh++) {
6442 for (int64_t ikw = 0; ikw < KW; ikw++) {
6443 // For s0 > 1 some values were skipped over in the forward pass.
6444 // These values have tmpw % s0 != 0 and need to be skipped in the backwards pass as well.
6445 const int64_t tmpw = (iiw + p0 - ikw*d0);
6446 if (tmpw % s0 != 0) {
6447 continue;
6448 }
6449 const int64_t iow = tmpw / s0;
6450
6451 // Equivalent logic as above except for s1.
6452 int64_t ioh;
6453 if (is_2D) {
6454 const int64_t tmph = iih + p1 - ikh*d1;
6455
6456 if (tmph % s1 != 0) {
6457 continue;
6458 }
6459
6460 ioh = tmph / s1;
6461 } else {
6462 ioh = 0;
6463 }
6464
6465 if (iow < 0 || iow >= OW || ioh < 0 || ioh >= OH) {
6466 continue;
6467 }
6468
6469 const float * const grad_in = (const float *) src0->data
6470 + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
6471 grad += grad_in[iic*(KH*KW) + ikh*KW + ikw];
6472 }
6473 }
6474 float * dst_data = (float *)((char *) wdata + (in*ofs0 + iic*ofs1)); // [IH, IW]
6475 dst_data[iih*IW + iiw] = grad;
6476 }
6477 }
6478 }
6479 }
6480 }
6481}
6482
6483
6484// ggml_compute_forward_im2col_3d_f16
6485// src0: kernel [OC*IC, KD, KH, KW]
6486// src1: image [N*IC, ID, IH, IW]
6487// dst: result [N*OD, OH, OW, IC * KD * KH * KW]
6488static void ggml_compute_forward_im2col_3d_f16(
6489 const ggml_compute_params * params,
6490 ggml_tensor * dst) {
6491
6492 const ggml_tensor * src0 = dst->src[0];
6493 const ggml_tensor * src1 = dst->src[1];
6494
6495 GGML_ASSERT(src0->type == GGML_TYPE_F16)if (!(src0->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6495, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F16"
);
6496 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6496, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
6497 GGML_ASSERT( dst->type == GGML_TYPE_F16)if (!(dst->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6497, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F16"
);
6498
6499 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);;
6500
6501 const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
6502 const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
6503 const int32_t s2 = ((const int32_t *)(dst->op_params))[2];
6504 const int32_t p0 = ((const int32_t *)(dst->op_params))[3];
6505 const int32_t p1 = ((const int32_t *)(dst->op_params))[4];
6506 const int32_t p2 = ((const int32_t *)(dst->op_params))[5];
6507 const int32_t d0 = ((const int32_t *)(dst->op_params))[6];
6508 const int32_t d1 = ((const int32_t *)(dst->op_params))[7];
6509 const int32_t d2 = ((const int32_t *)(dst->op_params))[8];
6510 const int32_t IC = ((const int32_t *)(dst->op_params))[9];
6511
6512
6513 const int ith = params->ith;
6514 const int nth = params->nth;
6515
6516 const int64_t N = ne13 / IC;
6517 const int64_t ID = ne12;
6518 const int64_t IH = ne11;
6519 const int64_t IW = ne10;
6520
6521 const int64_t OC = ne03 / IC;
6522 GGML_UNUSED(OC)(void)(OC);
6523 const int64_t KD = ne02;
6524 const int64_t KH = ne01;
6525 const int64_t KW = ne00;
6526
6527 const int64_t OD = ne3 / N;
6528 const int64_t OH = ne2;
6529 const int64_t OW = ne1;
6530 const int64_t OH_OW = OH*OW;
6531 const int64_t KD_KH_KW = KD*KH*KW;
6532 const int64_t KH_KW = KH*KW;
6533 const int64_t IC_KD_KH_KW = IC*KD*KH*KW;
6534
6535 GGML_ASSERT(nb10 == sizeof(float))if (!(nb10 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6535, "GGML_ASSERT(%s) failed", "nb10 == sizeof(float)");
6536
6537 // im2col: [N*IC, ID, IH, IW] => [N*OD, OH, OW, IC * KD * KH * KW]
6538 {
6539 ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;
6540
6541 for (int64_t in = 0; in < N; in++) {
6542 for (int64_t iod = 0; iod < OD; iod++) {
6543 for (int64_t ioh = 0; ioh < OH; ioh++) {
6544 for (int64_t iow = 0; iow < OW; iow++) {
6545 for (int64_t iic = ith; iic < IC; iic += nth) {
6546
6547 // micro kernel
6548 ggml_fp16_t * dst_data = wdata + (in*OD*OH_OW + iod*OH_OW + ioh*OW + iow)*IC_KD_KH_KW; // [IC, KD, KH, KW]
6549 const float * const src_data = (const float *) ((const char *)src1->data + (in*IC + iic)*nb13); // [ID, IH, IW]
6550
6551 for (int64_t ikd = 0; ikd < KD; ikd++) {
6552 for (int64_t ikh = 0; ikh < KH; ikh++) {
6553 for (int64_t ikw = 0; ikw < KW; ikw++) {
6554 const int64_t iiw = iow*s0 + ikw*d0 - p0;
6555 const int64_t iih = ioh*s1 + ikh*d1 - p1;
6556 const int64_t iid = iod*s2 + ikd*d2 - p2;
6557
6558 if (iid < 0 || iid >= ID || iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
6559 dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = 0;
6560 } else {
6561 const float * const s = (const float *) ((const char *)src_data + iid*nb12 + iih*nb11 + iiw*nb10); // [ID, IH, IW]
6562 dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = GGML_CPU_FP32_TO_FP16(*s)ggml_compute_fp32_to_fp16(*s);
6563 }
6564 }
6565 }
6566 }
6567 }
6568 }
6569 }
6570 }
6571 }
6572 }
6573}
6574
6575// ggml_compute_forward_im2col_3d_f32
6576// src0: kernel [OC*IC, KD, KH, KW]
6577// src1: image [N*IC, ID, IH, IW]
6578// dst: result [N*OD, OH, OW, IC * KD * KH * KW]
6579static void ggml_compute_forward_im2col_3d_f32(
6580 const ggml_compute_params * params,
6581 ggml_tensor * dst) {
6582
6583 const ggml_tensor * src0 = dst->src[0];
6584 const ggml_tensor * src1 = dst->src[1];
6585
6586 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6586, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
6587 GGML_ASSERT( dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6587, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
6588
6589 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);;
6590
6591 const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
6592 const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
6593 const int32_t s2 = ((const int32_t *)(dst->op_params))[2];
6594 const int32_t p0 = ((const int32_t *)(dst->op_params))[3];
6595 const int32_t p1 = ((const int32_t *)(dst->op_params))[4];
6596 const int32_t p2 = ((const int32_t *)(dst->op_params))[5];
6597 const int32_t d0 = ((const int32_t *)(dst->op_params))[6];
6598 const int32_t d1 = ((const int32_t *)(dst->op_params))[7];
6599 const int32_t d2 = ((const int32_t *)(dst->op_params))[8];
6600 const int32_t IC = ((const int32_t *)(dst->op_params))[9];
6601
6602
6603 const int ith = params->ith;
6604 const int nth = params->nth;
6605
6606 const int64_t N = ne13 / IC;
6607 const int64_t ID = ne12;
6608 const int64_t IH = ne11;
6609 const int64_t IW = ne10;
6610
6611 const int64_t OC = ne03 / IC;
6612 GGML_UNUSED(OC)(void)(OC);
6613 const int64_t KD = ne02;
6614 const int64_t KH = ne01;
6615 const int64_t KW = ne00;
6616
6617 const int64_t OD = ne3 / N;
6618 const int64_t OH = ne2;
6619 const int64_t OW = ne1;
6620
6621 const int64_t OH_OW = OH*OW;
6622 const int64_t KD_KH_KW = KD*KH*KW;
6623 const int64_t KH_KW = KH*KW;
6624 const int64_t IC_KD_KH_KW = IC*KD*KH*KW;
6625
6626 GGML_ASSERT(nb10 == sizeof(float))if (!(nb10 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6626, "GGML_ASSERT(%s) failed", "nb10 == sizeof(float)");
6627
6628 // im2col: [N*IC, ID, IH, IW] => [N*OD, OH, OW, IC * KD * KH * KW]
6629 {
6630 float * const wdata = (float *) dst->data;
6631
6632 for (int64_t in = 0; in < N; in++) {
6633 for (int64_t iod = 0; iod < OD; iod++) {
6634 for (int64_t ioh = 0; ioh < OH; ioh++) {
6635 for (int64_t iow = 0; iow < OW; iow++) {
6636 for (int64_t iic = ith; iic < IC; iic += nth) {
6637
6638 // micro kernel
6639 float * dst_data = wdata + (in*OD*OH_OW + iod*OH_OW + ioh*OW + iow)*IC_KD_KH_KW; // [IC, KD, KH, KW]
6640 const float * const src_data = (const float *) ((const char *)src1->data + (in*IC + iic)*nb13); // [ID, IH, IW]
6641
6642 for (int64_t ikd = 0; ikd < KD; ikd++) {
6643 for (int64_t ikh = 0; ikh < KH; ikh++) {
6644 for (int64_t ikw = 0; ikw < KW; ikw++) {
6645 const int64_t iiw = iow*s0 + ikw*d0 - p0;
6646 const int64_t iih = ioh*s1 + ikh*d1 - p1;
6647 const int64_t iid = iod*s2 + ikd*d2 - p2;
6648
6649 if (iid < 0 || iid >= ID || iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) {
6650 dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = 0;
6651 } else {
6652 const float * const s = (const float *) ((const char *)src_data + iid*nb12 + iih*nb11 + iiw*nb10); // [ID, IH, IW]
6653 dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = *s;
6654 }
6655 }
6656 }
6657 }
6658 }
6659 }
6660 }
6661 }
6662 }
6663 }
6664}
6665
6666
6667void ggml_compute_forward_im2col_3d(
6668 const ggml_compute_params * params,
6669 ggml_tensor * dst) {
6670 switch (dst->type) {
6671 case GGML_TYPE_F16:
6672 {
6673 ggml_compute_forward_im2col_3d_f16(params, dst);
6674 } break;
6675 case GGML_TYPE_F32:
6676 {
6677 ggml_compute_forward_im2col_3d_f32(params, dst);
6678 } break;
6679 default:
6680 {
6681 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6681, "fatal error");
6682 }
6683 }
6684}
6685
6686static void ggml_call_mul_mat(ggml_type type, const ggml_compute_params * params, int64_t m, int64_t n, int64_t k,
6687 void * a, void * b, float * c) {
6688 const ggml_type_traits * traits = ggml_get_type_traits(type);
6689 struct ggml_tensor src1 = {};
6690 src1.type = type;
6691 src1.ne[0] = k;
6692 src1.ne[1] = m;
6693 src1.ne[2] = 1;
6694 src1.ne[3] = 1;
6695 src1.nb[0] = traits->type_size;
6696 src1.nb[1] = k * traits->type_size;
6697 src1.nb[2] = src1.nb[1];
6698 src1.nb[3] = src1.nb[2];
6699 src1.data = a;
6700
6701 struct ggml_tensor src0 = {};
6702 src0.type = type;
6703 src0.ne[0] = k;
6704 src0.ne[1] = n;
6705 src0.ne[2] = 1;
6706 src0.ne[3] = 1;
6707 src0.nb[0] = traits->type_size;
6708 src0.nb[1] = k * traits->type_size;
6709 src0.nb[2] = src0.nb[1];
6710 src0.nb[3] = src0.nb[2];
6711 src0.data = b;
6712
6713 struct ggml_tensor dst = {};
6714 dst.ne[0] = n;
6715 dst.ne[1] = m;
6716 dst.ne[2] = 1;
6717 dst.ne[3] = 1;
6718 dst.nb[0] = sizeof(float);
6719 dst.nb[1] = n * sizeof(float);
6720 dst.nb[2] = dst.nb[1];
6721 dst.nb[3] = dst.nb[2];
6722 dst.data = c;
6723 dst.src[0] = &src0;
6724 dst.src[1] = &src1;
6725
6726 ggml_compute_forward_mul_mat(params, &dst);
6727}
6728
6729static inline int64_t ggml_wrap_around(int64_t coord, int64_t size) {
6730 return (coord + size) % size; // adding size avoids negative number weirdness
6731}
6732
6733// ggml_compute_forward_col2im_1d
6734//
6735// Scatter-add columns [K*OC, T_in] -> signal [T_out, OC]
6736// where T_out = (T_in - 1)*s + K - 2*p. Gather approach: each output reads ceil(K/s) inputs.
6737// Parallelized over the time axis so the split stays balanced whatever OC is.
6738// Supports F32, F16, BF16 input/output (same type), F32 accumulator.
6739
6740template <typename elem_t>
6741static void ggml_compute_forward_col2im_1d_impl(
6742 const ggml_compute_params * params,
6743 ggml_tensor * dst) {
6744
6745 const ggml_tensor * src = dst->src[0]; // [K*OC, T_in]
6746
6747 GGML_ASSERT(ggml_is_contiguous(src))if (!(ggml_is_contiguous(src))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6747, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(src)");
6748 GGML_ASSERT(ggml_is_contiguous(dst))if (!(ggml_is_contiguous(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6748, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dst)");
6749
6750 const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
6751 const int32_t OC = ((const int32_t *)(dst->op_params))[1];
6752 const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
6753
6754 const int64_t K_OC = src->ne[0];
6755 const int64_t T_in = src->ne[1];
6756 const int64_t K = K_OC / OC;
6757 const int64_t T_out = dst->ne[0];
6758
6759 const elem_t * col_data = (const elem_t *) src->data;
6760 elem_t * dst_data = (elem_t *) dst->data;
6761
6762 const int ith = params->ith;
6763 const int nth = params->nth;
6764
6765 // Parallelize over the time axis: the split stays balanced whatever OC is,
6766 // down to OC = 1 for mono audio, and threads read disjoint column bands
6767 const int64_t dr = (T_out + nth - 1) / nth;
6768 const int64_t it0 = dr * ith;
6769 const int64_t it1 = it0 + dr < T_out ? it0 + dr : T_out;
6770
6771 for (int64_t oc = 0; oc < OC; oc++) {
6772 for (int64_t t_out = it0; t_out < it1; t_out++) {
6773 const int64_t t_abs = t_out + p0; // absolute position in uncropped signal
6774 // Gather: find all (t_in, k) where t_in * s + k == t_abs, 0 <= k < K
6775 int64_t t_in_min = (t_abs - K + 1 + s0 - 1) / s0; // ceil((t_abs-K+1)/s)
6776 if (t_in_min < 0) t_in_min = 0;
6777 int64_t t_in_max = t_abs / s0;
6778 if (t_in_max >= T_in) t_in_max = T_in - 1;
6779
6780 float sum = 0.0f;
6781 for (int64_t t_in = t_in_min; t_in <= t_in_max; t_in++) {
6782 int64_t k = t_abs - t_in * s0;
6783 if (k >= 0 && k < K) {
6784 // col layout: [K*OC, T_in], element (oc*K+k, t_in)
6785 sum += type_conversion_table<elem_t>::to_f32(col_data[(oc * K + k) + t_in * K_OC]);
6786 }
6787 }
6788 // dst layout: [T_out, OC], element (t_out, oc)
6789 dst_data[t_out + oc * T_out] = type_conversion_table<elem_t>::from_f32(sum);
6790 }
6791 }
6792}
6793
6794void ggml_compute_forward_col2im_1d(
6795 const ggml_compute_params * params,
6796 ggml_tensor * dst) {
6797 switch (dst->src[0]->type) {
6798 case GGML_TYPE_F32: ggml_compute_forward_col2im_1d_impl<float> (params, dst); break;
6799 case GGML_TYPE_F16: ggml_compute_forward_col2im_1d_impl<ggml_fp16_t>(params, dst); break;
6800 case GGML_TYPE_BF16: ggml_compute_forward_col2im_1d_impl<ggml_bf16_t>(params, dst); break;
6801 default: GGML_ABORT("col2im_1d: unsupported type %d", dst->src[0]->type)ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6801, "col2im_1d: unsupported type %d", dst->src[0]->
type);
6802 }
6803}
6804
6805// ggml_compute_forward_conv_2d
6806
6807
6808static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params,
6809 const ggml_tensor * kernel, // [KW, KH, IC, OC]
6810 const ggml_tensor * src, // [W, H, C, N]
6811 ggml_tensor * dst, // [OW, OH, OC, N]
6812 ggml_type kernel_type) {
6813
6814 GGML_ASSERT(ggml_is_contiguous(kernel))if (!(ggml_is_contiguous(kernel))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6814, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(kernel)"
);
6815 GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32)if (!(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6815, "GGML_ASSERT(%s) failed", "kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32"
);
6816 GGML_ASSERT(kernel->type == kernel_type)if (!(kernel->type == kernel_type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6816, "GGML_ASSERT(%s) failed", "kernel->type == kernel_type"
);
6817
6818 const ggml_type_traits * traits = ggml_get_type_traits(kernel_type);
6819
6820 const int32_t stride_x = dst->op_params[0];
6821 const int32_t stride_y = dst->op_params[1];
6822 const int32_t pad_x = dst->op_params[2];
6823 const int32_t pad_y = dst->op_params[3];
6824 const int32_t dilation_x = dst->op_params[4];
6825 const int32_t dilation_y = dst->op_params[5];
6826
6827 const int64_t c_in = src->ne[2];
6828 const int64_t c_out = kernel->ne[3];
6829 GGML_ASSERT(c_in == kernel->ne[2])if (!(c_in == kernel->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6829, "GGML_ASSERT(%s) failed", "c_in == kernel->ne[2]");
6830
6831 const int64_t src_w = src->ne[0];
6832 const int64_t src_h = src->ne[1];
6833 const int64_t knl_w = kernel->ne[0];
6834 const int64_t knl_h = kernel->ne[1];
6835 const int64_t dst_w = dst->ne[0];
6836 const int64_t dst_h = dst->ne[1];
6837
6838 const float * src_data = (float *) src->data;
6839 void * knl_data = kernel->data;
6840 float * dst_data = (float *) dst->data;
6841
6842 const int64_t knl_n = knl_w * knl_h * c_in;
6843 const int64_t patch_total = dst->ne[3] * dst_w * dst_h;
6844
6845 const int64_t space_per_patch = knl_n * traits->type_size + c_out * sizeof(float);
6846 const int64_t batch_size = params->wsize / space_per_patch;
6847 const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size;
6848 const int64_t batch_n = (patch_total + patches_per_batch - 1) / patches_per_batch;
6849
6850 GGML_ASSERT(patches_per_batch > 0 && batch_size >= 1)if (!(patches_per_batch > 0 && batch_size >= 1)
) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6850, "GGML_ASSERT(%s) failed", "patches_per_batch > 0 && batch_size >= 1"
);
6851
6852 void * tmp = params->wdata;
6853
6854 for (int64_t batch_i = 0; batch_i < batch_n; ++batch_i) {
6855
6856 const int64_t patch_start_batch = batch_i * patches_per_batch;
6857 const int64_t patch_end_batch = std::min(patch_start_batch + patches_per_batch,
6858 patch_total);
6859 const int64_t patch_n = patch_end_batch - patch_start_batch;
6860
6861 const int64_t patch_per_thread = (patch_n + params->nth - 1) / params->nth;
6862 const int64_t patch_start = patch_start_batch + params->ith * patch_per_thread;
6863 const int64_t patch_end = std::min(patch_start + patch_per_thread, patch_end_batch);
6864
6865 //im2col for a patch
6866 for (int64_t p = patch_start; p < patch_end; ++p) {
6867 const int64_t batch_n = p / (dst_w * dst_h);
6868 const int64_t src_x = (p / dst_w) % dst_h;
6869 const int64_t src_y = p % dst_w;
6870
6871 const float * src_base = (const float *)((const char *)src_data + batch_n * src->nb[3]);
6872 char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n * traits->type_size;
6873
6874 for (int64_t ic = 0; ic < c_in; ++ic) {
6875 for (int64_t ky = 0; ky < knl_h; ++ky) {
6876 for (int64_t kx = 0; kx < knl_w; ++kx) {
6877 const int64_t sy = src_x * stride_y + ky * dilation_y - pad_y;
6878 const int64_t sx = src_y * stride_x + kx * dilation_x - pad_x;
6879
6880 int64_t dst_idx = ic * (knl_h * knl_w) + ky * knl_w + kx;
6881
6882 float src_val;
6883 if (sy < 0 || sy >= src_h || sx < 0 || sx >= src_w) {
6884 src_val = 0.0f;
6885 } else {
6886 const float * src_ptr = (const float *)((const char *)src_base + sx * src->nb[0] + sy * src->nb[1] + ic * src->nb[2]);
6887 src_val = *src_ptr;
6888 }
6889
6890 char * element_ptr = dst_row + dst_idx * traits->type_size;
6891 if (kernel_type == GGML_TYPE_F32) {
6892 *(float *) element_ptr = src_val;
6893 } else if (kernel_type == GGML_TYPE_F16) {
6894 *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val)ggml_compute_fp32_to_fp16(src_val);
6895 }
6896 }
6897 }
6898 }
6899 } // patches handled by this thread
6900
6901 ggml_barrier(params->threadpool);
6902
6903 float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n * traits->type_size);
6904
6905 GGML_ASSERT(gemm_output + patch_n * c_out <= (float*)tmp + params->wsize)if (!(gemm_output + patch_n * c_out <= (float*)tmp + params
->wsize)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6905, "GGML_ASSERT(%s) failed", "gemm_output + patch_n * c_out <= (float*)tmp + params->wsize"
);
6906
6907 // GEMM: patches[patch_n, knl_n] × kernel[knl_n, c_out] = output[patch_n, c_out]
6908 ggml_call_mul_mat(kernel_type, params, patch_n, c_out, knl_n, tmp, knl_data, gemm_output);
6909
6910 ggml_barrier(params->threadpool);
6911
6912
6913 //permute back [OC, N, OH, OW] to [N, OC, OH, OW]
6914 const int64_t permute_per_thread = (patch_n + params->nth - 1) / params->nth;
6915 const int64_t permute_start = params->ith * permute_per_thread;
6916 const int64_t permute_end = std::min(permute_start + permute_per_thread, patch_n);
6917
6918 for (int64_t i = permute_start; i < permute_end; ++i) {
6919 const int64_t p = patch_start_batch + i;
6920 const int64_t batch_n = p / (dst_w * dst_h);
6921 const int64_t dst_y = (p / dst_w) % dst_h;
6922 const int64_t dst_x = p % dst_w;
6923
6924 for (int64_t oc = 0; oc < c_out; ++oc) {
6925 const float value = gemm_output[i * c_out + oc];
6926 float * dst_ptr = (float *)((char *)dst_data + dst_x * dst->nb[0] + dst_y * dst->nb[1] + oc * dst->nb[2] + batch_n * dst->nb[3]);
6927 *dst_ptr = value;
6928 }
6929 }
6930 }
6931}
6932
6933void ggml_compute_forward_conv_2d(
6934 const ggml_compute_params * params,
6935 ggml_tensor * dst) {
6936
6937 const ggml_tensor * src0 = dst->src[0];
6938 const ggml_tensor * src1 = dst->src[1];
6939
6940 ggml_compute_forward_conv_2d_impl(params, src0, src1, dst, src0->type);
6941}
6942
6943// ggml_compute_forward_conv_3d
6944
6945static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params,
6946 const ggml_tensor * kernel,
6947 const ggml_tensor * src,
6948 ggml_tensor * dst,
6949 ggml_type kernel_type) {
6950
6951 GGML_ASSERT(ggml_is_contiguous(kernel))if (!(ggml_is_contiguous(kernel))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6951, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(kernel)"
);
6952 GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32)if (!(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6952, "GGML_ASSERT(%s) failed", "kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32"
);
6953 GGML_ASSERT(kernel->type == kernel_type)if (!(kernel->type == kernel_type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6953, "GGML_ASSERT(%s) failed", "kernel->type == kernel_type"
);
6954
6955 const ggml_type_traits * traits = ggml_get_type_traits(kernel_type);
6956
6957 const int32_t s0 = dst->op_params[0];
6958 const int32_t s1 = dst->op_params[1];
6959 const int32_t s2 = dst->op_params[2];
6960 const int32_t p0 = dst->op_params[3];
6961 const int32_t p1 = dst->op_params[4];
6962 const int32_t p2 = dst->op_params[5];
6963 const int32_t d0 = dst->op_params[6];
6964 const int32_t d1 = dst->op_params[7];
6965 const int32_t d2 = dst->op_params[8];
6966 const int32_t c = dst->op_params[9];
6967 const int32_t n = dst->op_params[10];
6968 const int32_t oc = dst->op_params[11];
6969
6970 const int64_t src_w = src->ne[0];
6971 const int64_t src_h = src->ne[1];
6972 const int64_t src_d = src->ne[2];
6973 const int64_t knl_w = kernel->ne[0];
6974 const int64_t knl_h = kernel->ne[1];
6975 const int64_t knl_d = kernel->ne[2];
6976 const int64_t dst_w = dst->ne[0];
6977 const int64_t dst_h = dst->ne[1];
6978 const int64_t dst_d = dst->ne[2];
6979
6980 const float * src_data = (float *) src->data;
6981 void * knl_data = kernel->data;
6982 float * dst_data = (float *) dst->data;
6983
6984 const int64_t knl_n_per_channel = knl_w * knl_h * knl_d;
6985 const int64_t knl_n_total = knl_n_per_channel * c;
6986 const int64_t patch_total = n * dst_w * dst_h * dst_d;
6987
6988 const int64_t space_per_patch = knl_n_total * traits->type_size + oc * sizeof(float);
6989 const int64_t batch_size = params->wsize / space_per_patch;
6990 const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size;
6991 const int64_t batch_n = (patch_total + patches_per_batch - 1) / patches_per_batch;
6992
6993 GGML_ASSERT(patches_per_batch > 0 && batch_size >= 1)if (!(patches_per_batch > 0 && batch_size >= 1)
) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 6993, "GGML_ASSERT(%s) failed", "patches_per_batch > 0 && batch_size >= 1"
);
6994
6995 void * tmp = params->wdata;
6996
6997 for (int64_t batch_i = 0; batch_i < batch_n; ++batch_i) {
6998 const int64_t patch_start_batch = batch_i * patches_per_batch;
6999 const int64_t patch_end_batch = std::min(patch_start_batch + patches_per_batch, patch_total);
7000 const int64_t patch_n_in_batch = patch_end_batch - patch_start_batch;
7001
7002 const int64_t patch_per_thread = (patch_n_in_batch + params->nth - 1) / params->nth;
7003 const int64_t patch_start = patch_start_batch + params->ith * patch_per_thread;
7004 const int64_t patch_end = std::min(patch_start + patch_per_thread, patch_end_batch);
7005
7006 for (int64_t p = patch_start; p < patch_end; ++p) {
7007 const int64_t p_in_batch = p % (dst_w * dst_h * dst_d);
7008 const int64_t p_in_depth = p_in_batch % (dst_w * dst_h);
7009 const int64_t batch_idx = p / (dst_w * dst_h * dst_d);
7010 const int64_t dst_z = p_in_batch / (dst_w * dst_h);
7011 const int64_t dst_y = p_in_depth / dst_w;
7012 const int64_t dst_x = p_in_depth % dst_w;
7013
7014 char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n_total * traits->type_size;
7015
7016 for (int64_t ic = 0; ic < c; ++ic) {
7017 for (int64_t kz = 0; kz < knl_d; ++kz) {
7018 for (int64_t ky = 0; ky < knl_h; ++ky) {
7019 for (int64_t kx = 0; kx < knl_w; ++kx) {
7020 const int64_t sz = dst_z * s2 + kz * d2 - p2;
7021 const int64_t sy = dst_y * s1 + ky * d1 - p1;
7022 const int64_t sx = dst_x * s0 + kx * d0 - p0;
7023
7024 int64_t dst_idx = ic * knl_n_per_channel + kz * (knl_h * knl_w) + ky * knl_w + kx;
7025
7026 float src_val;
7027 if (sz < 0 || sz >= src_d || sy < 0 || sy >= src_h || sx < 0 || sx >= src_w) {
7028 src_val = 0.0f;
7029 } else {
7030 const int64_t cn_idx = batch_idx * c + ic;
7031 const float * src_ptr = (const float *)((const char *)src_data + sx*src->nb[0] + sy*src->nb[1] + sz*src->nb[2] + cn_idx*src->nb[3]);
7032 src_val = *src_ptr;
7033 }
7034
7035 char * element_ptr = dst_row + dst_idx * traits->type_size;
7036 if (kernel_type == GGML_TYPE_F32) {
7037 *(float *)element_ptr = src_val;
7038 } else if (kernel_type == GGML_TYPE_F16) {
7039 *(ggml_fp16_t *)element_ptr = GGML_CPU_FP32_TO_FP16(src_val)ggml_compute_fp32_to_fp16(src_val);
7040 }
7041 }
7042 }
7043 }
7044 }
7045 }
7046
7047 ggml_barrier(params->threadpool);
7048
7049 float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n_total * traits->type_size);
7050 ggml_call_mul_mat(kernel_type, params, patch_n_in_batch, oc, knl_n_total, tmp, knl_data, gemm_output);
7051
7052 ggml_barrier(params->threadpool);
7053
7054 const int64_t permute_per_thread = (patch_n_in_batch + params->nth - 1) / params->nth;
7055 const int64_t permute_start = params->ith * permute_per_thread;
7056 const int64_t permute_end = std::min(permute_start + permute_per_thread, patch_n_in_batch);
7057
7058 for (int64_t i = permute_start; i < permute_end; ++i) {
7059 const int64_t p = patch_start_batch + i;
7060 const int64_t p_in_batch = p % (dst_w * dst_h * dst_d);
7061 const int64_t p_in_depth = p_in_batch % (dst_w * dst_h);
7062 const int64_t batch_idx = p / (dst_w * dst_h * dst_d);
7063 const int64_t dst_z = p_in_batch / (dst_w * dst_h);
7064 const int64_t dst_y = p_in_depth / dst_w;
7065 const int64_t dst_x = p_in_depth % dst_w;
7066
7067 for (int64_t ioc = 0; ioc < oc; ++ioc) {
7068 const float value = gemm_output[i * oc + ioc];
7069 const int64_t ocn_idx = batch_idx * oc + ioc;
7070 float * dst_ptr = (float *)((char *)dst_data + dst_x*dst->nb[0] + dst_y*dst->nb[1] + dst_z*dst->nb[2] + ocn_idx*dst->nb[3]);
7071 *dst_ptr = value;
7072 }
7073 }
7074 }
7075}
7076
7077void ggml_compute_forward_conv_3d(
7078 const ggml_compute_params * params,
7079 ggml_tensor * dst) {
7080 const ggml_tensor * src0 = dst->src[0];
7081 const ggml_tensor * src1 = dst->src[1];
7082 ggml_compute_forward_conv_3d_impl(params, src0, src1, dst, src0->type);
7083}
7084
7085template <typename kernel_t>
7086static void ggml_compute_forward_conv_transpose_2d_impl(
7087 const ggml_compute_params * params,
7088 ggml_tensor * dst) {
7089
7090 const ggml_tensor * src0 = dst->src[0];
7091 const ggml_tensor * src1 = dst->src[1];
7092
7093 GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32)if (!(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7093, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32"
);
7094 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7094, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
7095 GGML_ASSERT( dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7095, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
7096
7097 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
7098
7099 const int ith = params->ith;
7100 const int nth = params->nth;
7101
7102 const int nk = ne00*ne01*ne02*ne03;
7103
7104 GGML_ASSERT(nb00 == ggml_type_size(src0->type))if (!(nb00 == ggml_type_size(src0->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7104, "GGML_ASSERT(%s) failed", "nb00 == ggml_type_size(src0->type)"
);
7105 GGML_ASSERT(nb10 == sizeof(float))if (!(nb10 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7105, "GGML_ASSERT(%s) failed", "nb10 == sizeof(float)");
7106
7107 if (ith == 0) {
7108 memset(params->wdata, 0, params->wsize);
7109
7110 // permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout)
7111 {
7112 kernel_t * const wdata = (kernel_t *) params->wdata + 0;
7113
7114 for (int64_t i03 = 0; i03 < ne03; i03++) {
7115 for (int64_t i02 = 0; i02 < ne02; i02++) {
7116 const kernel_t * const src = (kernel_t *)((char *) src0->data + i03*nb03 + i02*nb02);
7117 kernel_t * dst_data = wdata + i02*ne01*ne00*ne03;
7118 for (int64_t i01 = 0; i01 < ne01; i01++) {
7119 for (int64_t i00 = 0; i00 < ne00; i00++) {
7120 dst_data[i01*ne00*ne03 + i00*ne03 + i03] = src[i01 * ne00 + i00];
7121 }
7122 }
7123 }
7124 }
7125 }
7126
7127 // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh)
7128 {
7129 kernel_t * const wdata = (kernel_t *) params->wdata + nk;
7130 for (int i12 = 0; i12 < ne12; i12++) {
7131 for (int i11 = 0; i11 < ne11; i11++) {
7132 const float * const src = (float *)((char *) src1->data + i12*nb12 + i11*nb11);
7133 kernel_t * dst_data = wdata + i11*ne10*ne12;
7134 for (int i10 = 0; i10 < ne10; i10++) {
7135 if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
7136 dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10])ggml_compute_fp32_to_fp16(src[i10]);
7137 } else {
7138 dst_data[i10*ne12 + i12] = src[i10];
7139 }
7140 }
7141 }
7142 }
7143 }
7144
7145 memset(dst->data, 0, ggml_nbytes(dst));
7146 }
7147 ggml_barrier(params->threadpool);
7148
7149 const int32_t stride = ggml_get_op_params_i32(dst, 0);
7150
7151 // total patches in dst
7152 const int np = ne2;
7153
7154 // patches per thread
7155 const int dp = (np + nth - 1)/nth;
7156
7157 // patch range for this thread
7158 const int ip0 = dp*ith;
7159 const int ip1 = MIN(ip0 + dp, np)((ip0 + dp) < (np) ? (ip0 + dp) : (np));
7160
7161 kernel_t * const wdata = (kernel_t *) params->wdata + 0;
7162 kernel_t * const wdata_src = wdata + nk;
7163
7164 for (int i2 = ip0; i2 < ip1; i2++) { // Cout
7165 float * dst_data = (float *)((char *) dst->data + i2*nb2);
7166 kernel_t * wdata_kernel = wdata + i2*ne01*ne00*ne03;
7167 for (int i11 = 0; i11 < ne11; i11++) {
7168 for (int i10 = 0; i10 < ne10; i10++) {
7169 const int i1n = i11*ne10*ne12 + i10*ne12;
7170 for (int i01 = 0; i01 < ne01; i01++) {
7171 for (int i00 = 0; i00 < ne00; i00++) {
7172 float v = 0;
7173 if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
7174 ggml_vec_dot_f16(ne03, &v, 0,
7175 wdata_src + i1n, 0,
7176 wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
7177 } else {
7178 ggml_vec_dot_f32(ne03, &v, 0,
7179 wdata_src + i1n, 0,
7180 wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
7181 }
7182 dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;
7183 }
7184 }
7185 }
7186 }
7187 }
7188}
7189
7190void ggml_compute_forward_conv_transpose_2d(
7191 const ggml_compute_params * params,
7192 ggml_tensor * dst) {
7193
7194 const ggml_tensor * src0 = dst->src[0];
7195
7196 switch (src0->type) {
7197 case GGML_TYPE_F16:
7198 {
7199 ggml_compute_forward_conv_transpose_2d_impl<ggml_fp16_t>(params, dst);
7200 } break;
7201 case GGML_TYPE_F32:
7202 {
7203 ggml_compute_forward_conv_transpose_2d_impl<float>(params, dst);
7204 } break;
7205 default:
7206 {
7207 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7207, "fatal error");
7208 }
7209 }
7210}
7211
7212// ggml_compute_forward_conv_2d_dw
7213
7214struct ggml_conv_2d_dw_params {
7215 int64_t channels;
7216 int64_t batch;
7217 int64_t src_w;
7218 int64_t src_h;
7219 int64_t dst_w;
7220 int64_t dst_h;
7221 int64_t knl_w;
7222 int64_t knl_h;
7223 int stride_x;
7224 int stride_y;
7225 int pad_x;
7226 int pad_y;
7227 int dilation_x;
7228 int dilation_y;
7229};
7230
7231static void ggml_compute_forward_conv_2d_dw_cwhn(
7232 const ggml_compute_params * params,
7233 const ggml_tensor * src,
7234 const ggml_tensor * kernel,
7235 ggml_tensor * dst,
7236 const ggml_conv_2d_dw_params & p) {
7237
7238 const int64_t c = p.channels;
7239 const float * knl_data = (const float *)kernel->data;
7240
7241 const int64_t rows_total = p.dst_h * p.batch;
7242 const int64_t rows_per_thread = (rows_total + params->nth - 1) / params->nth;
7243 const int64_t row_start = params->ith * rows_per_thread;
7244 const int64_t row_end = MIN(row_start + rows_per_thread, rows_total)((row_start + rows_per_thread) < (rows_total) ? (row_start
+ rows_per_thread) : (rows_total));
7245
7246#ifdef GGML_SIMD
7247 #if defined(__ARM_FEATURE_SVE)
7248 const int64_t pkg_size = svcntw();
7249 #else
7250 const int64_t pkg_size = GGML_F32_EPR8;
7251 #endif
7252 const int64_t pkg_count = c / pkg_size;
7253 const int64_t c_pkg_end = pkg_count * pkg_size;
7254#else
7255 const int64_t c_pkg_end = 0;
7256#endif
7257
7258 for (int64_t row = row_start; row < row_end; ++row) {
7259 const int64_t dst_y = row % p.dst_h;
7260 const float * src_data = (const float *)src->data + (row / p.dst_h) * p.src_w * p.src_h * c;
7261 for (int64_t dst_x = 0; dst_x < p.dst_w; ++dst_x) {
7262 float * dst_data = (float *)dst->data + (row * p.dst_w + dst_x) * c;
7263 const int64_t src_y_base = dst_y * p.stride_y - p.pad_y;
7264 const int64_t src_x_base = dst_x * p.stride_x - p.pad_x;
7265
7266#ifdef GGML_SIMD
7267 // Vectorized loop
7268 for (int64_t c_i = 0; c_i < c_pkg_end; c_i += pkg_size) {
7269 GGML_F32_VEC__m256 sum = GGML_F32_VEC_ZERO_mm256_setzero_ps();
7270 for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) {
7271 const int64_t src_y = src_y_base + knl_y * p.dilation_y;
7272 if (src_y < 0 || src_y >= p.src_h) {
7273 continue;
7274 }
7275 for (int64_t knl_x = 0; knl_x < p.knl_w; ++knl_x) {
7276 const int64_t src_x = src_x_base + knl_x * p.dilation_x;
7277 if (src_x < 0 || src_x >= p.src_w) {
7278 continue;
7279 }
7280 GGML_F32_VEC__m256 k = GGML_F32_VEC_LOAD_mm256_loadu_ps(knl_data + (knl_y * p.knl_w + knl_x) * c + c_i);
7281 GGML_F32_VEC__m256 s = GGML_F32_VEC_LOAD_mm256_loadu_ps(src_data + (src_y * p.src_w + src_x) * c + c_i);
7282 sum = GGML_F32_VEC_FMA(sum, k, s)_mm256_fmadd_ps(k, s, sum);
7283 }
7284 }
7285 GGML_F32_VEC_STORE_mm256_storeu_ps(dst_data + c_i, sum);
7286 }
7287#endif
7288 // Scalar loop
7289 for (int64_t c_i = c_pkg_end; c_i < c; ++c_i) {
7290 float sum = 0.0f;
7291 for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) {
7292 const int64_t src_y = src_y_base + knl_y * p.dilation_y;
7293 if (src_y < 0 || src_y >= p.src_h) {
7294 continue;
7295 }
7296 for (int64_t knl_x = 0; knl_x < p.knl_w; ++knl_x) {
7297 const int64_t src_x = src_x_base + knl_x * p.dilation_x;
7298 if (src_x < 0 || src_x >= p.src_w) {
7299 continue;
7300 }
7301 sum += knl_data[(knl_y * p.knl_w + knl_x) * c + c_i]
7302 * src_data[(src_y * p.src_w + src_x) * c + c_i];
7303 }
7304 }
7305 dst_data[c_i] = sum;
7306 }
7307 }
7308 }
7309}
7310
7311static void ggml_compute_forward_conv_2d_dw_whcn(
7312 const ggml_compute_params * params,
7313 const ggml_tensor * src,
7314 const ggml_tensor * kernel,
7315 ggml_tensor * dst,
7316 const ggml_conv_2d_dw_params & p) {
7317
7318 const int64_t n = p.channels * p.batch;
7319 const int64_t per_thread = (n + params->nth - 1) / params->nth;
7320 const int64_t start = params->ith * per_thread;
7321 const int64_t end = MIN(start + per_thread, n)((start + per_thread) < (n) ? (start + per_thread) : (n));
7322
7323 for (int64_t i = start; i < end; ++i) {
7324 const float * knl_data = (const float *)kernel->data + (i % p.channels) * p.knl_w * p.knl_h;
7325 const float * src_data = (const float *)src->data + i * p.src_w * p.src_h;
7326 float * dst_data = (float *)dst->data + i * p.dst_w * p.dst_h;
7327
7328 for (int64_t dst_y = 0; dst_y < p.dst_h; ++dst_y) {
7329 for (int64_t dst_x = 0; dst_x < p.dst_w; ++dst_x) {
7330
7331 float sum = 0.0f;
7332 for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) {
7333 const int64_t src_y = dst_y * p.stride_y + knl_y * p.dilation_y - p.pad_y;
7334 if (src_y < 0 || src_y >= p.src_h) {
7335 continue;
7336 }
7337 for (int64_t knl_x = 0; knl_x < p.knl_w; ++knl_x) {
7338 const int64_t src_x = dst_x * p.stride_x + knl_x * p.dilation_x - p.pad_x;
7339 if (src_x < 0 || src_x >= p.src_w) {
7340 continue;
7341 }
7342 sum += knl_data[knl_y * p.knl_w + knl_x]
7343 * src_data[src_y * p.src_w + src_x];
7344 }
7345 }
7346 dst_data[dst_y * p.dst_w + dst_x] = sum;
7347 }
7348 }
7349 }
7350}
7351
7352void ggml_compute_forward_conv_2d_dw(
7353 const ggml_compute_params * params,
7354 ggml_tensor * dst) {
7355
7356 const ggml_tensor * kernel = dst->src[0];
7357 const ggml_tensor * src = dst->src[1];
7358 ggml_conv_2d_dw_params p;
7359 p.channels = src->ne[2];
7360 p.batch = src->ne[3];
7361 p.src_w = src->ne[0];
7362 p.src_h = src->ne[1];
7363 p.dst_w = dst->ne[0];
7364 p.dst_h = dst->ne[1];
7365 p.knl_w = kernel->ne[0];
7366 p.knl_h = kernel->ne[1];
7367 p.stride_x = dst->op_params[0];
7368 p.stride_y = dst->op_params[1];
7369 p.pad_x = dst->op_params[2];
7370 p.pad_y = dst->op_params[3];
7371 p.dilation_x = dst->op_params[4];
7372 p.dilation_y = dst->op_params[5];
7373
7374 GGML_ASSERT(kernel->ne[3] == p.channels)if (!(kernel->ne[3] == p.channels)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7374, "GGML_ASSERT(%s) failed", "kernel->ne[3] == p.channels"
);
7375 GGML_ASSERT(dst->ne[3] == p.batch)if (!(dst->ne[3] == p.batch)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7375, "GGML_ASSERT(%s) failed", "dst->ne[3] == p.batch");
7376
7377 if (ggml_is_contiguous(src)) {
7378 ggml_compute_forward_conv_2d_dw_whcn(params, src, kernel, dst, p);
7379 } else if (ggml_is_contiguous_channels(src)) {
7380 // kernel should also have channels most contiguous in memory
7381 GGML_ASSERT(kernel->nb[0] >= kernel->nb[2] && kernel->nb[1] >= kernel->nb[0])if (!(kernel->nb[0] >= kernel->nb[2] && kernel
->nb[1] >= kernel->nb[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7381, "GGML_ASSERT(%s) failed", "kernel->nb[0] >= kernel->nb[2] && kernel->nb[1] >= kernel->nb[0]"
);
7382 ggml_compute_forward_conv_2d_dw_cwhn(params, src, kernel, dst, p);
7383 } else {
7384 GGML_ABORT("non-contiguous memory layout not supported")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7384, "non-contiguous memory layout not supported");
7385 }
7386}
7387
7388// ggml_compute_forward_pool_1d_ksp
7389static void ggml_compute_forward_pool_1d_ksp(
7390 const ggml_compute_params * params,
7391 const ggml_op_pool op,
7392 const int k,
7393 const int s,
7394 const int p,
7395 ggml_tensor * dst) {
7396
7397 const ggml_tensor * src = dst->src[0];
7398
7399 assert(src->type == GGML_TYPE_F32 || src->type == GGML_TYPE_F16)(static_cast <bool> (src->type == GGML_TYPE_F32 || src
->type == GGML_TYPE_F16) ? void (0) : __assert_fail ("src->type == GGML_TYPE_F32 || src->type == GGML_TYPE_F16"
, __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
7400
7401 if (params->ith != 0) {
7402 return;
7403 }
7404
7405 const int64_t IW = src->ne[0];
7406 const int64_t OW = dst->ne[0];
7407
7408 const int64_t nr = ggml_nrows(src);
7409
7410 for (int64_t ir = 0; ir < nr; ++ir) {
7411 const char * srow_bytes = (const char *) src->data + ir * src->nb[1];
7412 float * drow = (float *) (( char *) dst->data + ir * dst->nb[1]);
7413
7414 for (int64_t ow = 0; ow < OW; ++ow) {
7415 float res = 0;
7416 switch (op) {
7417 case GGML_OP_POOL_AVG: res = 0.0f; break;
7418 case GGML_OP_POOL_MAX: res = -FLT_MAX3.40282347e+38F; break;
7419 case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7419, "fatal error");
7420 }
7421
7422 int count = 0;
7423 const int base = (int) ow * s - p;
7424
7425 for (int ki = 0; ki < k; ++ki) {
7426 const int j = base + ki;
7427 if (j < 0 || j >= (int) IW) {
7428 continue;
7429 }
7430
7431 float v;
7432 if (src->type == GGML_TYPE_F32) {
7433 v = ((const float *) srow_bytes)[j];
7434 } else {
7435 v = GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t *) srow_bytes)[j])ggml_lookup_fp16_to_fp32(((const ggml_fp16_t *) srow_bytes)[j
]);
7436 }
7437
7438 switch (op) {
7439 case GGML_OP_POOL_AVG: res += v; break;
7440 case GGML_OP_POOL_MAX: res = std::max(v, res); break;
7441 case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7441, "fatal error");
7442 }
7443
7444 ++count;
7445 }
7446
7447 switch (op) {
7448 case GGML_OP_POOL_AVG: res = (count > 0) ? (res / count) : 0.0f; break;
7449 case GGML_OP_POOL_MAX: break;
7450 case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7450, "fatal error");
7451 }
7452
7453 drow[ow] = res;
7454 }
7455 }
7456}
7457
7458// ggml_compute_forward_pool_1d
7459
7460void ggml_compute_forward_pool_1d(
7461 const ggml_compute_params * params,
7462 ggml_tensor * dst) {
7463
7464 const int32_t * opts = (const int32_t *)dst->op_params;
7465 ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);
7466 const int k0 = opts[1];
7467 const int s0 = opts[2];
7468 const int p0 = opts[3];
7469
7470 ggml_compute_forward_pool_1d_ksp(params, op, k0, s0, p0, dst);
7471}
7472
7473// ggml_compute_forward_pool_2d
7474
7475void ggml_compute_forward_pool_2d(
7476 const ggml_compute_params * params,
7477 ggml_tensor * dst) {
7478
7479 const ggml_tensor * src = dst->src[0];
7480
7481 assert(src->type == GGML_TYPE_F32 || src->type == GGML_TYPE_F16)(static_cast <bool> (src->type == GGML_TYPE_F32 || src
->type == GGML_TYPE_F16) ? void (0) : __assert_fail ("src->type == GGML_TYPE_F32 || src->type == GGML_TYPE_F16"
, __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
7482
7483 if (params->ith != 0) {
7484 return;
7485 }
7486
7487 const int32_t * opts = (const int32_t *)dst->op_params;
7488
7489 ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);
7490 const int k0 = opts[1];
7491 const int k1 = opts[2];
7492 const int s0 = opts[3];
7493 const int s1 = opts[4];
7494 const int p0 = opts[5];
7495 const int p1 = opts[6];
7496 const char * cdata = (const char*)src->data;
7497 const char * const data_end = cdata + ggml_nbytes(src);
7498
7499 const int64_t px = dst->ne[0];
7500 const int64_t py = dst->ne[1];
7501 const int64_t pa = px * py;
7502
7503 float * dplane = (float *)dst->data;
7504
7505 const int ka = k0 * k1;
7506 const int offset0 = -p0;
7507 const int offset1 = -p1;
7508
7509 while (cdata < data_end) {
7510 for (int oy = 0; oy < py; ++oy) {
7511 float * const drow = dplane + oy * px;
7512 float * const out = drow;
7513
7514 for (int ox = 0; ox < px; ++ox) {
7515 float res = 0;
7516 switch (op) {
7517 case GGML_OP_POOL_AVG: res = 0; break;
7518 case GGML_OP_POOL_MAX: res = -FLT_MAX3.40282347e+38F; break;
7519 case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7519, "fatal error");
7520 }
7521
7522 const int ix = offset0 + ox * s0;
7523 const int iy = offset1 + oy * s1;
7524
7525 for (int ky = 0; ky < k1; ++ky) {
7526 if (iy + ky < 0 || iy + ky >= src->ne[1]) {
7527 continue;
7528 }
7529
7530 const void * srow = (const void *)(cdata + src->nb[1] * (iy + ky));
7531 for (int kx = 0; kx < k0; ++kx) {
7532 int j = ix + kx;
7533 if (j < 0 || j >= src->ne[0]) {
7534 continue;
7535 }
7536
7537 const float srow_j = (src->type == GGML_TYPE_F32) ? ((const float*)srow)[j] : GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t*)srow)[j])ggml_lookup_fp16_to_fp32(((const ggml_fp16_t*)srow)[j]);
7538 switch (op) {
7539 case GGML_OP_POOL_AVG: res += srow_j; break;
7540 case GGML_OP_POOL_MAX: res = std::max(srow_j, res); break;
7541 case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7541, "fatal error");
7542 }
7543 }
7544 }
7545 switch (op) {
7546 case GGML_OP_POOL_AVG: res /= ka; break;
7547 case GGML_OP_POOL_MAX: break;
7548 case GGML_OP_POOL_COUNT: GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7548, "fatal error");
7549 }
7550
7551 out[ox] = res;
7552 }
7553 }
7554
7555 cdata += src->nb[2];
7556 dplane += pa;
7557 }
7558}
7559
7560// ggml_compute_forward_pool_2d_back
7561
7562void ggml_compute_forward_pool_2d_back(
7563 const ggml_compute_params * params,
7564 ggml_tensor * dst) {
7565
7566 const ggml_tensor * src = dst->src[0];
7567 const ggml_tensor * dstf = dst->src[1]; // forward tensor of dst
7568
7569 assert(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)(static_cast <bool> (dst->type == GGML_TYPE_F32 || dst
->type == GGML_TYPE_F16) ? void (0) : __assert_fail ("dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16"
, __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
7570
7571 if (params->ith != 0) {
7572 return;
7573 }
7574
7575 const int32_t * opts = (const int32_t *)dst->op_params;
7576 ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);
7577 const int k0 = opts[1];
7578 const int k1 = opts[2];
7579 const int s0 = opts[3];
7580 const int s1 = opts[4];
7581 const int p0 = opts[5];
7582 const int p1 = opts[6];
7583
7584 char * cdata = (char *) dst->data;
7585 const char * cdataf = (const char *) dstf->data;
7586 const char * const data_end = cdata + ggml_nbytes(dst);
7587
7588 GGML_ASSERT(params->ith == 0)if (!(params->ith == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7588, "GGML_ASSERT(%s) failed", "params->ith == 0");
7589 memset(cdata, 0, ggml_nbytes(dst));
7590
7591 const int64_t px = src->ne[0];
7592 const int64_t py = src->ne[1];
7593 const int64_t pa = px * py;
7594
7595 const float * splane = (const float *) src->data;
7596
7597 const int ka = k0 * k1;
7598 const int offset0 = -p0;
7599 const int offset1 = -p1;
7600
7601 while (cdata < data_end) {
7602 for (int oy = 0; oy < py; ++oy) {
7603 const float * const srow = splane + oy * px;
7604 for (int ox = 0; ox < px; ++ox) {
7605 const float grad0 = srow[ox];
7606
7607 const int ix = offset0 + ox * s0;
7608 const int iy = offset1 + oy * s1;
7609
7610 if (op == GGML_OP_POOL_MAX) {
7611 float maxval = -FLT_MAX3.40282347e+38F;
7612 int kxmax = -1;
7613 int kymax = -1;
7614
7615 for (int ky = 0; ky < k1; ++ky) {
7616 if (iy + ky < 0 || iy + ky >= dst->ne[1]) {
7617 continue;
7618 }
7619 const void * drowf = (const void *)(cdataf + dst->nb[1] * (iy + ky));
7620 for (int kx = 0; kx < k0; ++kx) {
7621 int j = ix + kx;
7622 if (j < 0 || j >= dst->ne[0]) {
7623 continue;
7624 }
7625
7626 const float val = dst->type == GGML_TYPE_F32 ?
7627 ((const float *) drowf)[j] : GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t *) drowf)[j])ggml_lookup_fp16_to_fp32(((const ggml_fp16_t *) drowf)[j]);
7628 if (val <= maxval) {
7629 continue;
7630 }
7631
7632 maxval = val;
7633 kxmax = kx;
7634 kymax = ky;
7635 }
7636 }
7637
7638 if (kxmax == -1 || kymax == -1) {
7639 continue;
7640 }
7641
7642 void * drow = (void *)(cdata + dst->nb[1] * (iy + kymax));
7643 const int j = ix + kxmax;
7644 if (dst->type == GGML_TYPE_F32) {
7645 ((float *) drow)[j] += grad0;
7646 } else {
7647 ((ggml_fp16_t *) drow)[j] = GGML_CPU_FP32_TO_FP16(grad0 + GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t *) drow)[j]))ggml_compute_fp32_to_fp16(grad0 + ggml_lookup_fp16_to_fp32(((
const ggml_fp16_t *) drow)[j]));
7648 }
7649 } else if (op == GGML_OP_POOL_AVG) {
7650 const float grad = grad0 / ka;
7651
7652 for (int ky = 0; ky < k1; ++ky) {
7653 if (iy + ky < 0 || iy + ky >= dst->ne[1]) {
7654 continue;
7655 }
7656 void * drow = (void *)(cdata + dst->nb[1] * (iy + ky));
7657 for (int kx = 0; kx < k0; ++kx) {
7658 int j = ix + kx;
7659 if (j < 0 || j >= dst->ne[0]) {
7660 continue;
7661 }
7662
7663 if (dst->type == GGML_TYPE_F32) {
7664 ((float *) drow)[j] += grad;
7665 } else {
7666 ((ggml_fp16_t *) drow)[j] += GGML_CPU_FP32_TO_FP16(grad)ggml_compute_fp32_to_fp16(grad);
7667 }
7668 }
7669 }
7670 } else {
7671 GGML_ASSERT(false)if (!(false)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7671, "GGML_ASSERT(%s) failed", "false");
7672 }
7673 }
7674 }
7675
7676 cdata += dst->nb[2];
7677 cdataf += dst->nb[2];
7678 splane += pa;
7679 }
7680}
7681
7682// ggml_compute_forward_upscale
7683
7684static void ggml_compute_forward_upscale_f32(
7685 const ggml_compute_params * params,
7686 ggml_tensor * dst) {
7687
7688 const ggml_tensor * src0 = dst->src[0];
7689
7690 GGML_ASSERT(src0->type == GGML_TYPE_F32)if (!(src0->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7690, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F32"
);
7691
7692 const int ith = params->ith;
7693 const int nth = params->nth;
7694
7695 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
7696
7697 float sf0 = (float)ne0/src0->ne[0];
7698 float sf1 = (float)ne1/src0->ne[1];
7699 float sf2 = (float)ne2/src0->ne[2];
7700 float sf3 = (float)ne3/src0->ne[3];
7701 float pixel_offset = 0.5f;
7702
7703 const int32_t mode_flags = ggml_get_op_params_i32(dst, 0);
7704 const ggml_scale_mode mode = (ggml_scale_mode) (mode_flags & 0xFF);
7705
7706 if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) {
7707 pixel_offset = 0.0f;
7708 sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0;
7709 sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1;
7710 }
7711
7712 if (mode == GGML_SCALE_MODE_NEAREST) {
7713 for (int64_t i3 = 0; i3 < ne3; i3++) {
7714 const int64_t i03 = i3 / sf3;
7715 for (int64_t i2 = ith; i2 < ne2; i2 += nth) {
7716 const int64_t i02 = i2 / sf2;
7717 for (int64_t i1 = 0; i1 < ne1; i1++) {
7718 const int64_t i01 = i1 / sf1;
7719 for (int64_t i0 = 0; i0 < ne0; i0++) {
7720 const int64_t i00 = i0 / sf0;
7721
7722 const float * x = (float *)((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
7723 float * y = (float *)((char *) dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
7724
7725 *y = *x;
7726 }
7727 }
7728 }
7729 }
7730 } else if (mode == GGML_SCALE_MODE_BILINEAR && (mode_flags & GGML_SCALE_FLAG_ANTIALIAS)) {
7731 // Similar to F.interpolate(..., mode="bilinear", align_corners=False, antialias=True)
7732 // https://github.com/pytorch/pytorch/blob/8871ff29b743948d1225389d5b7068f37b22750b/aten/src/ATen/native/cpu/UpSampleKernel.cpp
7733 auto triangle_filter = [](float x) -> float {
7734 return std::max(1.0f - fabsf(x), 0.0f);
7735 };
7736
7737 // support and invscale, minimum 1 pixel for bilinear
7738 const float support1 = std::max(1.0f, 1.0f / sf1);
7739 const float invscale1 = 1.0f / support1;
7740 const float support0 = std::max(1.0f, 1.0f / sf0);
7741 const float invscale0 = 1.0f / support0;
7742
7743 for (int64_t i3 = 0; i3 < ne3; i3++) {
7744 const int64_t i03 = i3 / sf3;
7745 for (int64_t i2 = ith; i2 < ne2; i2 += nth) {
7746 const int64_t i02 = i2 / sf2;
7747 for (int64_t i1 = 0; i1 < ne1; i1++) {
7748 const float y = ((float) i1 + pixel_offset) / sf1;
7749 for (int64_t i0 = 0; i0 < ne0; i0++) {
7750 const float x = ((float) i0 + pixel_offset) / sf0;
7751
7752 // the range of source pixels that contribute
7753 const int64_t x_min = std::max<int64_t>(x - support0 + pixel_offset, 0);
7754 const int64_t x_max = std::min<int64_t>(x + support0 + pixel_offset, ne00);
7755 const int64_t y_min = std::max<int64_t>(y - support1 + pixel_offset, 0);
7756 const int64_t y_max = std::min<int64_t>(y + support1 + pixel_offset, ne01);
7757
7758 // bilinear filter with antialiasing
7759 float val = 0.0f;
7760 float total_weight = 0.0f;
7761
7762 for (int64_t sy = y_min; sy < y_max; sy++) {
7763 const float weight_y = triangle_filter((sy - y + pixel_offset) * invscale1);
7764
7765 for (int64_t sx = x_min; sx < x_max; sx++) {
7766 const float weight_x = triangle_filter((sx - x + pixel_offset) * invscale0);
7767 const float weight = weight_x * weight_y;
7768
7769 if (weight <= 0.0f) {
7770 continue;
7771 }
7772
7773 const float pixel = *(const float *)((const char *)src0->data + sx*nb00 + sy*nb01 + i02*nb02 + i03*nb03);
7774 val += pixel * weight;
7775 total_weight += weight;
7776 }
7777 }
7778
7779 if (total_weight > 0.0f) {
7780 val /= total_weight;
7781 }
7782
7783 float * dst_ptr = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
7784 *dst_ptr = val;
7785 }
7786 }
7787 }
7788 }
7789 } else if (mode == GGML_SCALE_MODE_BILINEAR) {
7790 for (int64_t i3 = 0; i3 < ne3; i3++) {
7791 const int64_t i03 = i3 / sf3;
7792 for (int64_t i2 = ith; i2 < ne2; i2 += nth) {
7793 const int64_t i02 = i2 / sf2;
7794 for (int64_t i1 = 0; i1 < ne1; i1++) {
7795 const float y = ((float)i1 + pixel_offset) / sf1 - pixel_offset;
7796 int64_t y0 = (int64_t)floorf(y);
7797 int64_t y1 = y0 + 1;
7798
7799 y0 = std::max(int64_t(0), std::min(y0, ne01 - 1));
7800 y1 = std::max(int64_t(0), std::min(y1, ne01 - 1));
7801
7802 float dy = y - (float)y0;
7803 dy = std::max(0.0f, std::min(dy, 1.0f));
7804
7805 for (int64_t i0 = 0; i0 < ne0; i0++) {
7806 const float x = ((float)i0 + pixel_offset) / sf0 - pixel_offset;
7807 int64_t x0 = (int64_t)floorf(x);
7808 int64_t x1 = x0 + 1;
7809
7810 x0 = std::max(int64_t(0), std::min(x0, ne00 - 1));
7811 x1 = std::max(int64_t(0), std::min(x1, ne00 - 1));
7812
7813 float dx = x - (float)x0;
7814 dx = std::max(0.0f, std::min(dx, 1.0f));
7815
7816 // fetch the four surrounding pixel values and interpolate
7817 const float a = *(const float *)((const char *)src0->data + x0*nb00 + y0*nb01 + i02*nb02 + i03*nb03);
7818 const float b = *(const float *)((const char *)src0->data + x1*nb00 + y0*nb01 + i02*nb02 + i03*nb03);
7819 const float c = *(const float *)((const char *)src0->data + x0*nb00 + y1*nb01 + i02*nb02 + i03*nb03);
7820 const float d = *(const float *)((const char *)src0->data + x1*nb00 + y1*nb01 + i02*nb02 + i03*nb03);
7821
7822 const float val = a*(1 - dx)*(1 - dy) + b*dx*(1 - dy) + c*(1 - dx)*dy + d*dx*dy;
7823
7824 float * y_dst = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
7825 *y_dst = val;
7826 }
7827 }
7828 }
7829 }
7830 } else if (mode == GGML_SCALE_MODE_BICUBIC) {
7831 // https://en.wikipedia.org/wiki/Bicubic_interpolation#Bicubic_convolution_algorithm
7832 const float a = -0.75f; // use alpha = -0.75 (same as PyTorch)
7833 auto weight1 = [a](float x) { return ((a + 2) * x - (a + 3)) * x * x + 1; };
7834 auto weight2 = [a](float x) { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; };
7835 auto bicubic = [=](float p0, float p1, float p2, float p3, float x) {
7836 const float w0 = weight2(x + 1);
7837 const float w1 = weight1(x + 0);
7838 const float w2 = weight1(1 - x);
7839 const float w3 = weight2(2 - x);
7840 return p0*w0 + p1*w1 + p2*w2 + p3*w3;
7841 };
7842
7843 for (int64_t i3 = 0; i3 < ne3; i3++) {
7844 const int64_t i03 = i3 / sf3;
7845 for (int64_t i2 = ith; i2 < ne2; i2 += nth) {
7846 const int64_t i02 = i2 / sf2;
7847 for (int64_t i1 = 0; i1 < ne1; i1++) {
7848 const float y = ((float)i1 + pixel_offset) / sf1 - pixel_offset;
7849 const int64_t y0 = (int64_t)floorf(y);
7850 const float dy = y - (float)y0;
7851
7852 for (int64_t i0 = 0; i0 < ne0; i0++) {
7853 const float x = ((float)i0 + pixel_offset) / sf0 - pixel_offset;
7854 const int64_t x0 = (int64_t)floorf(x);
7855 const float dx = x - (float)x0;
7856
7857 auto p = [=](int64_t x_off, int64_t y_off) -> float {
7858 int64_t i00 = std::max(int64_t(0), std::min(x0 + x_off, ne00 - 1));
7859 int64_t i01 = std::max(int64_t(0), std::min(y0 + y_off, ne01 - 1));
7860 return *(const float *)((const char *)src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
7861 };
7862
7863 const float val = bicubic(
7864 bicubic(p(-1,-1), p(0,-1), p(1,-1), p(2,-1), dx),
7865 bicubic(p(-1, 0), p(0, 0), p(1, 0), p(2, 0), dx),
7866 bicubic(p(-1, 1), p(0, 1), p(1, 1), p(2, 1), dx),
7867 bicubic(p(-1, 2), p(0, 2), p(1, 2), p(2, 2), dx), dy);
7868
7869 float * y_dst = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
7870 *y_dst = val;
7871 }
7872 }
7873 }
7874 }
7875 } else {
7876 GGML_ABORT("unsupported upscale mode")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7876, "unsupported upscale mode");
7877 }
7878}
7879
7880void ggml_compute_forward_upscale(
7881 const ggml_compute_params * params,
7882 ggml_tensor * dst) {
7883
7884 const ggml_tensor * src0 = dst->src[0];
7885
7886 switch (src0->type) {
7887 case GGML_TYPE_F32:
7888 {
7889 ggml_compute_forward_upscale_f32(params, dst);
7890 } break;
7891 default:
7892 {
7893 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7893, "fatal error");
7894 }
7895 }
7896}
7897
7898
7899// ggml_compute_forward_pad
7900
7901template<bool circular_t>
7902static void ggml_compute_forward_pad_f32(
7903 const ggml_compute_params * params,
7904 ggml_tensor * dst) {
7905
7906 const ggml_tensor * src0 = dst->src[0];
7907
7908 assert(dst->nb[0] == sizeof(float))(static_cast <bool> (dst->nb[0] == sizeof(float)) ? void
(0) : __assert_fail ("dst->nb[0] == sizeof(float)", __builtin_FILE
(), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
7909
7910 const int ith = params->ith;
7911 const int nth = params->nth;
7912
7913 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
7914
7915 float * dst_ptr = (float *) dst->data;
7916 const int32_t lp0 = ggml_get_op_params_i32(dst, 0);
7917 const int32_t rp0 = ggml_get_op_params_i32(dst, 1);
7918 const int32_t lp1 = ggml_get_op_params_i32(dst, 2);
7919 const int32_t rp1 = ggml_get_op_params_i32(dst, 3);
7920 const int32_t lp2 = ggml_get_op_params_i32(dst, 4);
7921 const int32_t rp2 = ggml_get_op_params_i32(dst, 5);
7922 const int32_t lp3 = ggml_get_op_params_i32(dst, 6);
7923 const int32_t rp3 = ggml_get_op_params_i32(dst, 7);
7924
7925 // TODO: optimize
7926
7927 for (int64_t i2 = 0; i2 < ne2; ++i2) {
7928 for (int64_t i1 = ith; i1 < ne1; i1 += nth) {
7929 for (int64_t i0 = 0; i0 < ne0; ++i0) {
7930 for (int64_t i3 = 0; i3 < ne3; ++i3) {
7931 // circular means wrap around on a torus, so x and y loop around
7932 if constexpr (circular_t) {
7933 const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0;
7934 const int64_t src_i0 = ggml_wrap_around(i0 - lp0, ne00);
7935 const int64_t src_i1 = ggml_wrap_around(i1 - lp1, ne01);
7936 const int64_t src_i2 = ggml_wrap_around(i2 - lp2, ne02);
7937 const int64_t src_i3 = ggml_wrap_around(i3 - lp3, ne03);
7938
7939 const int64_t src_idx =
7940 src_i3*nb03 +
7941 src_i2*nb02 +
7942 src_i1*nb01 +
7943 src_i0*nb00;
7944
7945 const float * src_ptr = (const float *)((char *) src0->data + src_idx);
7946 dst_ptr[dst_idx] = *src_ptr;
7947 } else {
7948 const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0;
7949 if ((i0 >= lp0 && i0 < ne0 - rp0) \
7950 && (i1 >= lp1 && i1 < ne1 - rp1) \
7951 && (i2 >= lp2 && i2 < ne2 - rp2) \
7952 && (i3 >= lp3 && i3 < ne3 - rp3)) {
7953 const int64_t src_idx = (i3 - lp3)*nb03 + (i2 - lp2)*nb02 + (i1 - lp1)*nb01 + (i0 - lp0)*nb00;
7954 const float * src_ptr = (const float *)((char *) src0->data + src_idx);
7955 dst_ptr[dst_idx] = *src_ptr;
7956 } else {
7957 dst_ptr[dst_idx] = 0;
7958 }
7959 }
7960 }
7961 }
7962 }
7963 }
7964}
7965
7966
7967void ggml_compute_forward_pad(
7968 const ggml_compute_params * params,
7969 ggml_tensor * dst) {
7970 const ggml_tensor * src0 = dst->src[0];
7971 const bool circular = (bool) ggml_get_op_params_i32(dst, 8);
7972 switch (src0->type) {
7973 case GGML_TYPE_F32:
7974 {
7975 if (circular) {
7976 ggml_compute_forward_pad_f32<true>(params, dst);
7977 } else {
7978 ggml_compute_forward_pad_f32<false>(params, dst);
7979 }
7980 } break;
7981 default:
7982 {
7983 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7983, "fatal error");
7984 }
7985 }
7986}
7987
7988// ggml_compute_forward_pad_reflect_1d
7989
7990void ggml_compute_forward_pad_reflect_1d(
7991 const ggml_compute_params * params,
7992 ggml_tensor * dst) {
7993
7994 const ggml_tensor * src0 = dst->src[0];
7995
7996 GGML_ASSERT(src0->type == GGML_TYPE_F32)if (!(src0->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7996, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F32"
);
7997 GGML_ASSERT( dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 7997, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
7998
7999 const int ith = params->ith;
8000 const int nth = params->nth;
8001
8002 const int32_t * opts = (const int32_t *) dst->op_params;
8003 const int p0 = opts[0];
8004 const int p1 = opts[1];
8005
8006 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
8007
8008 for (int64_t i3 = 0; i3 < ne3; i3++) {
8009 for (int64_t i2 = 0; i2 < ne2; i2++) {
8010 for (int64_t i1 = ith; i1 < ne1; i1 += nth) {
8011 float * left = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + p0*nb0);
8012 float * right = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + (ne0-p1-1)*nb0);
8013
8014 ggml_vec_cpy_f32(ne00, left, (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));
8015
8016 for (int i0 = 1; i0 <= p0; i0++) { left[-i0] = left[i0]; }
8017 for (int i0 = 1; i0 <= p1; i0++) { right[i0] = right[-i0]; }
8018 }
8019 }
8020 }
8021}
8022
8023// ggml_compute_forward_roll
8024
8025static int64_t ggml_wrap_index(int64_t i, int64_t ne) {
8026 if (i < 0) {
8027 return i + ne;
8028 } else if (i >= ne) {
8029 return i - ne;
8030 }
8031 return i;
8032}
8033
8034static void ggml_compute_forward_roll_f32(
8035 const ggml_compute_params * params,
8036 ggml_tensor * dst) {
8037
8038 const ggml_tensor * src0 = dst->src[0];
8039 const float * src_data = (const float *) src0->data;
8040 float * dst_data = (float *) dst->data;
8041
8042 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
8043
8044 const int s0 = ggml_get_op_params_i32(dst, 0);
8045 const int s1 = ggml_get_op_params_i32(dst, 1);
8046 const int s2 = ggml_get_op_params_i32(dst, 2);
8047 const int s3 = ggml_get_op_params_i32(dst, 3);
8048
8049 const int64_t total = ne1 * ne2 * ne3;
8050 const int64_t per_thread = (total + params->nth) / params->nth;
8051 const int64_t start = params->ith * per_thread;
8052 const int64_t end = std::min(start + per_thread, total);
8053
8054 for (int64_t i = start; i < end; ++i) {
8055 const int64_t i1 = i % ne1;
8056 const int64_t i2 = (i / ne1) % ne2;
8057 const int64_t i3 = i / (ne2 * ne1);
8058 float * dst_row = dst_data + (i3*nb3 + i2*nb2 + i1*nb1) / sizeof(float);
8059
8060 const int64_t i01 = ggml_wrap_index(i1 - s1, ne01);
8061 const int64_t i02 = ggml_wrap_index(i2 - s2, ne02);
8062 const int64_t i03 = ggml_wrap_index(i3 - s3, ne03);
8063 const float * src_row = src_data + (i03*nb03 + i02*nb02 + i01*nb01) / sizeof(float);
8064
8065 const int64_t s = ggml_wrap_index(-s0, ne00);
8066 const int64_t n = ne00 - s;
8067 ggml_vec_cpy_f32(n, dst_row, src_row + s);
8068 ggml_vec_cpy_f32(s, dst_row + n, src_row);
8069 }
8070}
8071
8072void ggml_compute_forward_roll(
8073 const ggml_compute_params * params,
8074 ggml_tensor * dst) {
8075
8076 const ggml_tensor * src0 = dst->src[0];
8077
8078 switch (src0->type) {
8079 case GGML_TYPE_F32:
8080 {
8081 ggml_compute_forward_roll_f32(params, dst);
8082 } break;
8083 default:
8084 {
8085 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8085, "fatal error");
8086 }
8087 }
8088}
8089
8090// ggml_compute_forward_arange
8091
8092static void ggml_compute_forward_arange_f32(
8093 const ggml_compute_params * params,
8094 ggml_tensor * dst) {
8095
8096 GGML_ASSERT(dst->nb[0] == sizeof(float))if (!(dst->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8096, "GGML_ASSERT(%s) failed", "dst->nb[0] == sizeof(float)"
);
8097
8098 const int ith = params->ith;
8099 const int nth = params->nth;
8100
8101 const float start = ggml_get_op_params_f32(dst, 0);
8102 const float stop = ggml_get_op_params_f32(dst, 1);
8103 const float step = ggml_get_op_params_f32(dst, 2);
8104
8105 const int64_t steps = (int64_t) ceilf((stop - start) / step);
8106
8107 GGML_ASSERT(ggml_nelements(dst) == steps)if (!(ggml_nelements(dst) == steps)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8107, "GGML_ASSERT(%s) failed", "ggml_nelements(dst) == steps"
);
8108
8109 for (int64_t i = ith; i < steps; i+= nth) {
8110 float value = start + step * i;
8111 ((float *)dst->data)[i] = value;
8112 }
8113}
8114
8115void ggml_compute_forward_arange(
8116 const ggml_compute_params * params,
8117 ggml_tensor * dst) {
8118 switch (dst->type) {
8119 case GGML_TYPE_F32:
8120 {
8121 ggml_compute_forward_arange_f32(params, dst);
8122 } break;
8123 default:
8124 {
8125 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8125, "fatal error");
8126 }
8127 }
8128}
8129
8130static void ggml_compute_forward_timestep_embedding_f32(
8131 const ggml_compute_params * params,
8132 ggml_tensor * dst) {
8133
8134 const ggml_tensor * src0 = dst->src[0];
8135
8136 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8136, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
8137
8138 const int ith = params->ith;
8139 const int nth = params->nth;
8140
8141 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
8142
8143 const int dim = ggml_get_op_params_i32(dst, 0);
8144 const int max_period = ggml_get_op_params_i32(dst, 1);
8145
8146 int half = dim / 2;
8147
8148 for (int64_t i = 0; i < ne00; i++) {
8149 float * embed_data = (float *)((char *) dst->data + i*nb1);
8150 for (int64_t j = ith; j < half; j += nth) {
8151 float timestep = ((float *)src0->data)[i];
8152 float freq = (float)expf(-logf(max_period) * j / half);
8153 float arg = timestep * freq;
8154 embed_data[j] = cosf(arg);
8155 embed_data[j + half] = sinf(arg);
8156 }
8157 if (dim % 2 != 0 && ith == 0) {
8158 embed_data[2 * half] = 0.f;
8159 }
8160 }
8161}
8162
8163void ggml_compute_forward_timestep_embedding(
8164 const ggml_compute_params * params,
8165 ggml_tensor * dst) {
8166
8167 const ggml_tensor * src0 = dst->src[0];
8168
8169 switch (src0->type) {
8170 case GGML_TYPE_F32:
8171 {
8172 ggml_compute_forward_timestep_embedding_f32(params, dst);
8173 } break;
8174 default:
8175 {
8176 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8176, "fatal error");
8177 }
8178 }
8179}
8180
8181// ggml_compute_forward_argsort
8182
8183template<enum ggml_sort_order order>
8184struct cmp_argsort {
8185 const float * data;
8186 bool operator()(int32_t a, int32_t b) const {
8187 if constexpr (order == GGML_SORT_ORDER_ASC) {
8188 return data[a] < data[b];
8189 } else {
8190 return data[a] > data[b];
8191 }
8192 }
8193};
8194
8195static void ggml_compute_forward_argsort_f32(
8196 const ggml_compute_params * params,
8197 ggml_tensor * dst) {
8198
8199 const ggml_tensor * src0 = dst->src[0];
8200
8201 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
8202
8203 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8203, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
8204
8205 const int ith = params->ith;
8206 const int nth = params->nth;
8207
8208 const int64_t nr = ggml_nrows(src0);
8209
8210 ggml_sort_order order = (ggml_sort_order) ggml_get_op_params_i32(dst, 0);
8211
8212 for (int64_t i = ith; i < nr; i += nth) {
8213 const float * src_data = (float *)((char *) src0->data + i*nb01);
8214
8215 int32_t * dst_data = (int32_t *)((char *) dst->data + i*nb1);
8216
8217 for (int64_t j = 0; j < ne0; j++) {
8218 dst_data[j] = j;
8219 }
8220
8221 switch (order) {
8222 case GGML_SORT_ORDER_ASC:
8223 std::sort(dst_data, dst_data + ne0, cmp_argsort<GGML_SORT_ORDER_ASC>{src_data});
8224 break;
8225
8226 case GGML_SORT_ORDER_DESC:
8227 std::sort(dst_data, dst_data + ne0, cmp_argsort<GGML_SORT_ORDER_DESC>{src_data});
8228 break;
8229
8230 default:
8231 GGML_ABORT("invalid sort order")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8231, "invalid sort order");
8232 }
8233 }
8234}
8235
8236void ggml_compute_forward_argsort(
8237 const ggml_compute_params * params,
8238 ggml_tensor * dst) {
8239
8240 const ggml_tensor * src0 = dst->src[0];
8241
8242 switch (src0->type) {
8243 case GGML_TYPE_F32:
8244 {
8245 ggml_compute_forward_argsort_f32(params, dst);
8246 } break;
8247 default:
8248 {
8249 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8249, "fatal error");
8250 }
8251 }
8252}
8253
8254// ggml_compute_forward_top_k
8255
8256struct cmp_top_k {
8257 const float * data;
8258 bool operator()(int32_t a, int32_t b) const {
8259 return data[a] > data[b];
8260 }
8261};
8262
8263static void ggml_compute_forward_top_k_f32(
8264 const ggml_compute_params * params,
8265 ggml_tensor * dst) {
8266
8267 const ggml_tensor * src0 = dst->src[0];
8268
8269 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
8270
8271 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8271, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
8272
8273 const int ith = params->ith;
8274 const int nth = params->nth;
8275
8276 const int64_t nr = ggml_nrows(src0);
8277
8278 const int top_k = ne0;
8279
8280 int32_t * tmp = (int32_t *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;
8281
8282 for (int64_t i = ith; i < nr; i += nth) {
8283 const float * src_data = (float *)((char *) src0->data + i*nb01);
8284
8285 for (int64_t j = 0; j < ne00; j++) {
8286 tmp[j] = j;
8287 }
8288
8289 std::partial_sort(tmp, tmp + top_k, tmp + ne00, cmp_top_k{src_data});
8290
8291 int32_t * dst_data = (int32_t *)((char *) dst->data + i*nb1);
8292
8293 std::copy(tmp, tmp + top_k, dst_data);
8294
8295 // emphasize that the order is not important
8296 if (top_k > 1) {
8297 std::swap(dst_data[0], dst_data[1]);
8298 }
8299 }
8300}
8301
8302void ggml_compute_forward_top_k(
8303 const ggml_compute_params * params,
8304 ggml_tensor * dst) {
8305
8306 const ggml_tensor * src0 = dst->src[0];
8307
8308 switch (src0->type) {
8309 case GGML_TYPE_F32:
8310 {
8311 ggml_compute_forward_top_k_f32(params, dst);
8312 } break;
8313 default:
8314 {
8315 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8315, "fatal error");
8316 }
8317 }
8318}
8319
8320static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
8321 const ggml_compute_params * params,
8322 ggml_tensor * dst,
8323 int ir0, int ir1,
8324 int64_t ic_start, int64_t ic_end,
8325 float * partials, int64_t partial_stride) {
8326
8327 const bool write_partials = (partials != nullptr);
8328 const ggml_tensor * q = dst->src[0];
8329 const ggml_tensor * k = dst->src[1];
8330 const ggml_tensor * v = dst->src[2];
8331 const ggml_tensor * mask = dst->src[3];
8332 const ggml_tensor * sinks = dst->src[4];
8333
8334 GGML_TENSOR_LOCALS(int64_t, neq, q, ne)const int64_t neq0 = (q) ? (q)->ne[0] : 0; (void)(neq0); const
int64_t neq1 = (q) ? (q)->ne[1] : 0; (void)(neq1); const int64_t
neq2 = (q) ? (q)->ne[2] : 0; (void)(neq2); const int64_t neq3
= (q) ? (q)->ne[3] : 0; (void)(neq3);
8335 GGML_TENSOR_LOCALS(size_t, nbq, q, nb)const size_t nbq0 = (q) ? (q)->nb[0] : 0; (void)(nbq0); const
size_t nbq1 = (q) ? (q)->nb[1] : 0; (void)(nbq1); const size_t
nbq2 = (q) ? (q)->nb[2] : 0; (void)(nbq2); const size_t nbq3
= (q) ? (q)->nb[3] : 0; (void)(nbq3);
8336 GGML_TENSOR_LOCALS(int64_t, nek, k, ne)const int64_t nek0 = (k) ? (k)->ne[0] : 0; (void)(nek0); const
int64_t nek1 = (k) ? (k)->ne[1] : 0; (void)(nek1); const int64_t
nek2 = (k) ? (k)->ne[2] : 0; (void)(nek2); const int64_t nek3
= (k) ? (k)->ne[3] : 0; (void)(nek3);
8337 GGML_TENSOR_LOCALS(size_t, nbk, k, nb)const size_t nbk0 = (k) ? (k)->nb[0] : 0; (void)(nbk0); const
size_t nbk1 = (k) ? (k)->nb[1] : 0; (void)(nbk1); const size_t
nbk2 = (k) ? (k)->nb[2] : 0; (void)(nbk2); const size_t nbk3
= (k) ? (k)->nb[3] : 0; (void)(nbk3);
8338 GGML_TENSOR_LOCALS(int64_t, nev, v, ne)const int64_t nev0 = (v) ? (v)->ne[0] : 0; (void)(nev0); const
int64_t nev1 = (v) ? (v)->ne[1] : 0; (void)(nev1); const int64_t
nev2 = (v) ? (v)->ne[2] : 0; (void)(nev2); const int64_t nev3
= (v) ? (v)->ne[3] : 0; (void)(nev3);
8339 GGML_TENSOR_LOCALS(size_t, nbv, v, nb)const size_t nbv0 = (v) ? (v)->nb[0] : 0; (void)(nbv0); const
size_t nbv1 = (v) ? (v)->nb[1] : 0; (void)(nbv1); const size_t
nbv2 = (v) ? (v)->nb[2] : 0; (void)(nbv2); const size_t nbv3
= (v) ? (v)->nb[3] : 0; (void)(nbv3);
8340 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
8341 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
8342
8343 const int64_t DK = nek0;
8344 const int64_t DV = nev0;
8345 const int64_t N = neq1;
8346
8347 GGML_ASSERT(ne0 == DV)if (!(ne0 == DV)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8347, "GGML_ASSERT(%s) failed", "ne0 == DV");
8348 GGML_ASSERT(ne2 == N)if (!(ne2 == N)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8348, "GGML_ASSERT(%s) failed", "ne2 == N");
8349
8350 // input tensor rows must be contiguous
8351 GGML_ASSERT(nbq0 == ggml_type_size(q->type))if (!(nbq0 == ggml_type_size(q->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8351, "GGML_ASSERT(%s) failed", "nbq0 == ggml_type_size(q->type)"
);
8352 GGML_ASSERT(nbk0 == ggml_type_size(k->type))if (!(nbk0 == ggml_type_size(k->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8352, "GGML_ASSERT(%s) failed", "nbk0 == ggml_type_size(k->type)"
);
8353 GGML_ASSERT(nbv0 == ggml_type_size(v->type))if (!(nbv0 == ggml_type_size(v->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8353, "GGML_ASSERT(%s) failed", "nbv0 == ggml_type_size(v->type)"
);
8354
8355 GGML_ASSERT(neq0 == DK)if (!(neq0 == DK)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8355, "GGML_ASSERT(%s) failed", "neq0 == DK");
8356 GGML_ASSERT(nek0 == DK)if (!(nek0 == DK)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8356, "GGML_ASSERT(%s) failed", "nek0 == DK");
8357 GGML_ASSERT(nev0 == DV)if (!(nev0 == DV)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8357, "GGML_ASSERT(%s) failed", "nev0 == DV");
8358
8359 GGML_ASSERT(neq1 == N)if (!(neq1 == N)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8359, "GGML_ASSERT(%s) failed", "neq1 == N");
8360
8361 // dst cannot be transposed or permuted
8362 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8362, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
8363 GGML_ASSERT(nb0 <= nb1)if (!(nb0 <= nb1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8363, "GGML_ASSERT(%s) failed", "nb0 <= nb1");
8364 GGML_ASSERT(nb1 <= nb2)if (!(nb1 <= nb2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8364, "GGML_ASSERT(%s) failed", "nb1 <= nb2");
8365 GGML_ASSERT(nb2 <= nb3)if (!(nb2 <= nb3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8365, "GGML_ASSERT(%s) failed", "nb2 <= nb3");
8366
8367 // broadcast factors
8368 const int64_t rk2 = neq2/nek2;
8369 const int64_t rk3 = neq3/nek3;
8370
8371 const int64_t rv2 = neq2/nev2;
8372 const int64_t rv3 = neq3/nev3;
8373
8374 // parallelize by q rows using ggml_vec_dot_f32
8375
8376 float scale = 1.0f;
8377 float max_bias = 0.0f;
8378 float logit_softcap = 0.0f;
8379
8380 memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));
8381 memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));
8382 memcpy(&logit_softcap, (float *) dst->op_params + 2, sizeof(float));
8383
8384 if (logit_softcap != 0) {
8385 scale /= logit_softcap;
8386 }
8387
8388 const uint32_t n_head = neq2;
8389 const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));
8390
8391 const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
8392 const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
8393
8394 ggml_type const k_vec_dot_type = ggml_get_type_traits_cpu(k->type)->vec_dot_type;
8395 ggml_from_float_t const q_to_vec_dot = ggml_get_type_traits_cpu(k_vec_dot_type)->from_float;
8396 ggml_vec_dot_t const kq_vec_dot = ggml_get_type_traits_cpu(k->type)->vec_dot;
8397 ggml_to_float_t const v_to_float = ggml_get_type_traits(v->type)->to_float;
8398
8399 GGML_ASSERT(( q_to_vec_dot) && "fattn: unsupported K-type")if (!(( q_to_vec_dot) && "fattn: unsupported K-type")
) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8399, "GGML_ASSERT(%s) failed", "( q_to_vec_dot) && \"fattn: unsupported K-type\""
);
8400 GGML_ASSERT((v->type == GGML_TYPE_F32 || v_to_float ) && "fattn: unsupported V-type")if (!((v->type == GGML_TYPE_F32 || v_to_float ) &&
"fattn: unsupported V-type")) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8400, "GGML_ASSERT(%s) failed", "(v->type == GGML_TYPE_F32 || v_to_float ) && \"fattn: unsupported V-type\""
);
8401
8402 int ith = params->ith;
8403
8404 for (int ir = ir0; ir < ir1; ++ir) {
8405 // q indices
8406 const int iq3 = ir/(neq2*neq1);
8407 const int iq2 = (ir - iq3*neq2*neq1)/neq1;
8408 const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);
8409
8410 const uint32_t h = iq2; // head index
8411 const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;
8412
8413 float S = 0.0f; // sum
8414 float M = -INFINITY(__builtin_inff()); // maximum KQ value
8415
8416 float * VKQ32 = (float *) params->wdata + ith*(1*DK + 2*DV + CACHE_LINE_SIZE_F32); // FP32 VKQ accumulator
8417 float * V32 = (VKQ32 + 1*DV); // (temporary) FP32 V buffer
8418 ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*DV); // (temporary) FP16 VKQ accumulator
8419 ggml_fp16_t * Q_q = (ggml_fp16_t *) (VKQ32 + 2*DV); // (temporary) buffer for Q converted to quantized/FP16
8420
8421 if (v->type == GGML_TYPE_F16) {
8422 memset(VKQ16, 0, DV*sizeof(ggml_fp16_t));
8423 } else {
8424 memset(VKQ32, 0, DV*sizeof(float));
8425 }
8426
8427 const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]) : NULL__null;
8428
8429 // k indices
8430 const int ik3 = iq3 / rk3;
8431 const int ik2 = iq2 / rk2;
8432
8433 // v indices
8434 const int iv3 = iq3 / rv3;
8435 const int iv2 = iq2 / rv2;
8436
8437 const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3));
8438 q_to_vec_dot(pq, Q_q, DK);
8439
8440 // online softmax / attention
8441 // loop over n_kv and n_head_kv
8442 // ref: https://arxiv.org/pdf/2112.05682.pdf
8443
8444 for (int64_t ic = ic_start; ic < ic_end; ++ic) {
8445 const float mv = mp ? slope*GGML_CPU_FP16_TO_FP32(mp[ic])ggml_lookup_fp16_to_fp32(mp[ic]) : 0.0f;
8446 if (mv == -INFINITY(__builtin_inff())) {
8447 continue;
8448 }
8449
8450 float s; // KQ value
8451
8452 const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);
8453 kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1);
8454
8455 s = s*scale; // scale KQ value
8456
8457 if (logit_softcap != 0.0f) {
8458 s = logit_softcap*tanhf(s);
8459 }
8460
8461 s += mv; // apply mask
8462
8463 const float Mold = M;
8464
8465 float ms = 1.0f; // upon new higher max val, scale VKQ and KQ sum with this value
8466 float vs = 1.0f; // post-softmax KQ value, expf(s - M)
8467
8468 const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));
8469
8470 if (v->type == GGML_TYPE_F16) {
8471 if (s > M) {
8472 // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f
8473 M = s;
8474 ms = expf(Mold - M);
8475
8476 // V = V*expf(Mold - M)
8477 ggml_vec_scale_f16(DV, VKQ16, ms);
8478 } else {
8479 // no new maximum, ms == 1.0f, vs != 1.0f
8480 vs = expf(s - M);
8481 }
8482
8483 // V += v*expf(s - M)
8484 ggml_vec_mad_f16(DV, VKQ16, (const ggml_fp16_t *) v_data, vs);
8485 } else {
8486 if (s > M) {
8487 // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f
8488 M = s;
8489 ms = expf(Mold - M);
8490
8491 // V = V*expf(Mold - M)
8492 ggml_vec_scale_f32(DV, VKQ32, ms);
8493 } else {
8494 // no new maximum, ms == 1.0f, vs != 1.0f
8495 vs = expf(s - M);
8496 }
8497
8498 // V += v*expf(s - M)
8499 if (v_to_float) {
8500 v_to_float(v_data, V32, DV);
8501 ggml_vec_mad_f32(DV, VKQ32, V32, vs);
8502 } else {
8503 // V is F32
8504 ggml_vec_mad_f32(DV, VKQ32, (const float *) v_data, vs);
8505 }
8506 }
8507
8508 S = S*ms + vs; // scale and increment sum with partial sum
8509 }
8510
8511 if (v->type == GGML_TYPE_F16) {
8512 for (int64_t d = 0; d < DV; ++d) {
8513 VKQ32[d] = GGML_CPU_FP16_TO_FP32(VKQ16[d])ggml_lookup_fp16_to_fp32(VKQ16[d]);
8514 }
8515 }
8516
8517 // sinks - apply only on the first kv-chunk
8518 if (sinks && ic_start == 0) {
8519 const float s = ((float *)((char *) sinks->data))[h];
8520
8521 float ms = 1.0f;
8522 float vs = 1.0f;
8523
8524 if (s > M) {
8525 ms = expf(M - s);
8526 M = s;
8527 ggml_vec_scale_f32(DV, VKQ32, ms);
8528 } else {
8529 vs = expf(s - M);
8530 }
8531
8532 S = S*ms + vs;
8533 }
8534
8535 if (write_partials) {
8536 // Write M, S, VKQ to partials for later reduction
8537 // partials layout: [M, S, VKQ[DV]] per query head
8538 float * partial = partials + ir * partial_stride;
8539 partial[0] = M;
8540 partial[1] = S;
8541 memcpy(partial + 2, VKQ32, DV * sizeof(float));
8542 } else {
8543 // V /= S
8544 const float S_inv = S == 0.0f ? 0.0f : 1.0f/S;
8545 ggml_vec_scale_f32(DV, VKQ32, S_inv);
8546
8547 // dst indices
8548 const int i1 = iq1;
8549 const int i2 = iq2;
8550 const int i3 = iq3;
8551
8552 // permute(0, 2, 1, 3)
8553 memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, VKQ32, nb1);
8554 }
8555 }
8556}
8557
8558static void ggml_compute_forward_flash_attn_ext_tiled(
8559 const ggml_compute_params * params,
8560 ggml_tensor * dst,
8561 int ir0, int ir1) {
8562 const ggml_tensor * q = dst->src[0];
8563 const ggml_tensor * k = dst->src[1];
8564 const ggml_tensor * v = dst->src[2];
8565 const ggml_tensor * mask = dst->src[3];
8566 const ggml_tensor * sinks = dst->src[4];
8567
8568 GGML_TENSOR_LOCALS(int64_t, neq, q, ne)const int64_t neq0 = (q) ? (q)->ne[0] : 0; (void)(neq0); const
int64_t neq1 = (q) ? (q)->ne[1] : 0; (void)(neq1); const int64_t
neq2 = (q) ? (q)->ne[2] : 0; (void)(neq2); const int64_t neq3
= (q) ? (q)->ne[3] : 0; (void)(neq3);
8569 GGML_TENSOR_LOCALS(size_t, nbq, q, nb)const size_t nbq0 = (q) ? (q)->nb[0] : 0; (void)(nbq0); const
size_t nbq1 = (q) ? (q)->nb[1] : 0; (void)(nbq1); const size_t
nbq2 = (q) ? (q)->nb[2] : 0; (void)(nbq2); const size_t nbq3
= (q) ? (q)->nb[3] : 0; (void)(nbq3);
8570 GGML_TENSOR_LOCALS(int64_t, nek, k, ne)const int64_t nek0 = (k) ? (k)->ne[0] : 0; (void)(nek0); const
int64_t nek1 = (k) ? (k)->ne[1] : 0; (void)(nek1); const int64_t
nek2 = (k) ? (k)->ne[2] : 0; (void)(nek2); const int64_t nek3
= (k) ? (k)->ne[3] : 0; (void)(nek3);
8571 GGML_TENSOR_LOCALS(size_t, nbk, k, nb)const size_t nbk0 = (k) ? (k)->nb[0] : 0; (void)(nbk0); const
size_t nbk1 = (k) ? (k)->nb[1] : 0; (void)(nbk1); const size_t
nbk2 = (k) ? (k)->nb[2] : 0; (void)(nbk2); const size_t nbk3
= (k) ? (k)->nb[3] : 0; (void)(nbk3);
8572 GGML_TENSOR_LOCALS(int64_t, nev, v, ne)const int64_t nev0 = (v) ? (v)->ne[0] : 0; (void)(nev0); const
int64_t nev1 = (v) ? (v)->ne[1] : 0; (void)(nev1); const int64_t
nev2 = (v) ? (v)->ne[2] : 0; (void)(nev2); const int64_t nev3
= (v) ? (v)->ne[3] : 0; (void)(nev3);
8573 GGML_TENSOR_LOCALS(size_t, nbv, v, nb)const size_t nbv0 = (v) ? (v)->nb[0] : 0; (void)(nbv0); const
size_t nbv1 = (v) ? (v)->nb[1] : 0; (void)(nbv1); const size_t
nbv2 = (v) ? (v)->nb[2] : 0; (void)(nbv2); const size_t nbv3
= (v) ? (v)->nb[3] : 0; (void)(nbv3);
8574 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
8575 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
8576
8577 const int64_t DK = nek0;
8578 const int64_t DV = nev0;
8579 const int64_t N = neq1;
8580
8581 GGML_ASSERT(ne0 == DV)if (!(ne0 == DV)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8581, "GGML_ASSERT(%s) failed", "ne0 == DV");
8582 GGML_ASSERT(ne2 == N)if (!(ne2 == N)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8582, "GGML_ASSERT(%s) failed", "ne2 == N");
8583
8584 // input tensor rows must be contiguous
8585 GGML_ASSERT(nbq0 == ggml_type_size(q->type))if (!(nbq0 == ggml_type_size(q->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8585, "GGML_ASSERT(%s) failed", "nbq0 == ggml_type_size(q->type)"
);
8586 GGML_ASSERT(nbk0 == ggml_type_size(k->type))if (!(nbk0 == ggml_type_size(k->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8586, "GGML_ASSERT(%s) failed", "nbk0 == ggml_type_size(k->type)"
);
8587 GGML_ASSERT(nbv0 == ggml_type_size(v->type))if (!(nbv0 == ggml_type_size(v->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8587, "GGML_ASSERT(%s) failed", "nbv0 == ggml_type_size(v->type)"
);
8588
8589 GGML_ASSERT(neq0 == DK)if (!(neq0 == DK)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8589, "GGML_ASSERT(%s) failed", "neq0 == DK");
8590 GGML_ASSERT(nek0 == DK)if (!(nek0 == DK)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8590, "GGML_ASSERT(%s) failed", "nek0 == DK");
8591 GGML_ASSERT(nev0 == DV)if (!(nev0 == DV)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8591, "GGML_ASSERT(%s) failed", "nev0 == DV");
8592
8593 GGML_ASSERT(neq1 == N)if (!(neq1 == N)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8593, "GGML_ASSERT(%s) failed", "neq1 == N");
8594
8595 // dst cannot be transposed or permuted
8596 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8596, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
8597 GGML_ASSERT(nb0 <= nb1)if (!(nb0 <= nb1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8597, "GGML_ASSERT(%s) failed", "nb0 <= nb1");
8598 GGML_ASSERT(nb1 <= nb2)if (!(nb1 <= nb2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8598, "GGML_ASSERT(%s) failed", "nb1 <= nb2");
8599 GGML_ASSERT(nb2 <= nb3)if (!(nb2 <= nb3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8599, "GGML_ASSERT(%s) failed", "nb2 <= nb3");
8600
8601 GGML_ASSERT(k->type == v->type)if (!(k->type == v->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8601, "GGML_ASSERT(%s) failed", "k->type == v->type");
8602 const ggml_type kv_type = k->type;
8603
8604
8605 // broadcast factors
8606 const int64_t rk2 = neq2/nek2;
8607 const int64_t rk3 = neq3/nek3;
8608
8609 const int64_t rv2 = neq2/nev2;
8610 const int64_t rv3 = neq3/nev3;
8611
8612 float scale = 1.0f;
8613 float max_bias = 0.0f;
8614 float logit_softcap = 0.0f;
8615
8616 memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));
8617 memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));
8618 memcpy(&logit_softcap, (float *) dst->op_params + 2, sizeof(float));
8619
8620 if (logit_softcap != 0) {
8621 scale /= logit_softcap;
8622 }
8623
8624 const uint32_t n_head = neq2;
8625 const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));
8626
8627 const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
8628 const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
8629
8630 int ith = params->ith;
8631
8632 static constexpr int Q_TILE_SZ = ggml_fa_tile_config::Q;
8633 static constexpr int KV_TILE_SZ = ggml_fa_tile_config::KV;
8634
8635 int ir = ir0;
8636 while (ir < ir1) {
8637 // q indices for the start of this tile
8638 const int iq3 = ir/(neq2*neq1);
8639 const int iq2 = (ir - iq3*neq2*neq1)/neq1;
8640 const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);
8641
8642 // Number of valid rows in this tile:
8643 // - limited by tile size (Q_TILE_SZ)
8644 // - limited by chunk boundary (ir1 - ir)
8645 // - limited by head boundary (neq1 - iq1) to avoid crossing into next head
8646 const int tile_rows = MIN(Q_TILE_SZ, MIN((int)(ir1 - ir), (int)(neq1 - iq1)))((Q_TILE_SZ) < ((((int)(ir1 - ir)) < ((int)(neq1 - iq1)
) ? ((int)(ir1 - ir)) : ((int)(neq1 - iq1)))) ? (Q_TILE_SZ) :
((((int)(ir1 - ir)) < ((int)(neq1 - iq1)) ? ((int)(ir1 - ir
)) : ((int)(neq1 - iq1)))));
8647 GGML_ASSERT(tile_rows > 0)if (!(tile_rows > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8647, "GGML_ASSERT(%s) failed", "tile_rows > 0");
8648
8649 const uint32_t h = iq2; // head index
8650 const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;
8651
8652 float S[Q_TILE_SZ];
8653 float M[Q_TILE_SZ];
8654
8655 for (int i = 0 ; i < Q_TILE_SZ; ++i) {
8656 S[i] = 0.;
8657 M[i] = -INFINITY(__builtin_inff());
8658 }
8659
8660 // Per-thread scratch layout:
8661 // Q_q: Q_TILE_SZ * DK (converted Q tile — F32 for GEMM, KV type for scalar)
8662 // KQ: Q_TILE_SZ * KV_TILE_SZ (attention scores in float)
8663 // mask: Q_TILE_SZ * KV_TILE_SZ (mask in float)
8664 // VKQ32: Q_TILE_SZ * DV (FP32 output accumulator)
8665 // V32: KV_TILE_SZ * DV (F32 buffer for V tile)
8666 // K_f32: KV_TILE_SZ * DK (F32 buffer for K tile — GEMM path)
8667 float * base = (float *) params->wdata + ith*(Q_TILE_SZ*DK + 2*Q_TILE_SZ*KV_TILE_SZ + Q_TILE_SZ*DV + KV_TILE_SZ*DV + KV_TILE_SZ*DK + CACHE_LINE_SIZE_F32);
8668
8669 void * Q_q = base;
8670 float * KQ = (float *)((char *)base + Q_TILE_SZ * DK * sizeof(float));
8671 float * mask32 = KQ + Q_TILE_SZ * KV_TILE_SZ;
8672 float * VKQ32 = mask32 + Q_TILE_SZ * KV_TILE_SZ;
8673 float * V32 = VKQ32 + Q_TILE_SZ * DV;
8674 float * K_f32 = V32 + KV_TILE_SZ * DV;
8675
8676 memset(VKQ32, 0, Q_TILE_SZ * DV * sizeof(float));
8677 memset(mask32, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));
8678
8679 // k indices
8680 const int ik3 = iq3 / rk3;
8681 const int ik2 = iq2 / rk2;
8682
8683 // v indices
8684 const int iv3 = iq3 / rv3;
8685 const int iv2 = iq2 / rv2;
8686
8687 {
8688 float * Q_f32 = (float *)Q_q;
8689 for (int tq = 0; tq < tile_rows; tq++) {
8690 const float * pq = (const float *) ((char *) q->data + ((iq1 + tq)*nbq1 + iq2*nbq2 + iq3*nbq3));
8691 memcpy(Q_f32 + tq * DK, pq, DK * sizeof(float));
8692 }
8693 for (int tq = tile_rows; tq < Q_TILE_SZ; tq++) {
8694 memset(Q_f32 + tq * DK, 0, DK * sizeof(float));
8695 }
8696 }
8697
8698 memset(K_f32, 0, DK * KV_TILE_SZ * sizeof(float));
8699 memset(V32, 0, KV_TILE_SZ * DV * sizeof(float));
8700
8701 for (int64_t ic = 0; ic < nek1; ic += KV_TILE_SZ) {
8702 const int kv_tile = (int)std::min((int64_t)KV_TILE_SZ, nek1 - ic);
8703
8704 // skip the tile entirely if all the masks are -inf
8705 if (mask) {
8706 bool can_skip = true;
8707 for (int tq = 0; tq < tile_rows; tq++) {
8708 const ggml_fp16_t * mp_row = (const ggml_fp16_t *)((const char *) mask->data + (iq1 + tq)*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]);
8709 for (int tk = 0; tk < kv_tile; tk++) {
8710 mask32[tq * KV_TILE_SZ + tk] = slope * GGML_CPU_FP16_TO_FP32(mp_row[ic + tk])ggml_lookup_fp16_to_fp32(mp_row[ic + tk]);
8711 if (mask32[tq * KV_TILE_SZ + tk] != -INFINITY(__builtin_inff())) {
8712 can_skip = false;
8713 }
8714 }
8715 // Pad remaining mask entries with -inf
8716 for (int tk = kv_tile; tk < KV_TILE_SZ; tk++) {
8717 mask32[tq * KV_TILE_SZ + tk] = -INFINITY(__builtin_inff());
8718 }
8719 }
8720
8721 if (can_skip) {
8722 continue;
8723 }
8724 }
8725
8726 // Pack K tile transposed: K_f32[dk][kv] so KV_TILE is contiguous (SIMD dim)
8727 // Zero-pad the last tile so the GEMM always operates on KV_TILE_SZ columns
8728 for (int tk = 0; tk < kv_tile; tk++) {
8729 const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;
8730 if (kv_type == GGML_TYPE_F16) {
8731 const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data;
8732 for (int64_t dk = 0; dk < DK; dk++) {
8733 K_f32[dk * KV_TILE_SZ + tk] = GGML_CPU_FP16_TO_FP32(k_f16[dk])ggml_lookup_fp16_to_fp32(k_f16[dk]);
8734 }
8735 } else {
8736 const float * k_f32_src = (const float *)k_data;
8737 for (int64_t dk = 0; dk < DK; dk++) {
8738 K_f32[dk * KV_TILE_SZ + tk] = k_f32_src[dk];
8739 }
8740 }
8741 }
8742 memset(KQ, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));
8743 simd_gemm(KQ, (const float *)Q_q, K_f32, Q_TILE_SZ, DK, KV_TILE_SZ);
8744 ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, scale);
8745
8746 // Set padded KQ entries to -inf so softmax gives them zero weight
8747 if (kv_tile < KV_TILE_SZ) {
8748 for (int tq = 0; tq < Q_TILE_SZ; tq++) {
8749 for (int tk = kv_tile; tk < KV_TILE_SZ; tk++) {
8750 KQ[tq * KV_TILE_SZ + tk] = -INFINITY(__builtin_inff());
8751 }
8752 }
8753 }
8754
8755 if (logit_softcap != 0.0f) {
8756 ggml_vec_tanh_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, KQ);
8757 ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, logit_softcap);
8758 }
8759
8760 if (mask) {
8761 ggml_vec_add_f32(tile_rows * KV_TILE_SZ, KQ, KQ, mask32);
8762 }
8763
8764 bool skip[Q_TILE_SZ] = {};
8765
8766 for (int tq = 0; tq < Q_TILE_SZ; tq++) {
8767 float * kq_row = KQ + tq * KV_TILE_SZ;
8768
8769 float tile_max;
8770 ggml_vec_max_f32(KV_TILE_SZ, &tile_max, kq_row);
8771
8772 if (tile_max == -INFINITY(__builtin_inff())) {
8773 skip[tq] = true;
8774 continue;
8775 }
8776
8777 const float Mold = M[tq];
8778 const float Mnew = fmaxf(Mold, tile_max);
8779
8780 if (Mnew > Mold) {
8781 const float ms = expf(Mold - Mnew);
8782 ggml_vec_scale_f32(DV, VKQ32 + tq * DV, ms);
8783 S[tq] *= ms;
8784 }
8785 M[tq] = Mnew;
8786
8787
8788 S[tq] += ggml_vec_soft_max_f32(KV_TILE_SZ, kq_row, kq_row, Mnew);
8789 }
8790
8791 // V accumulation: VKQ32 += softmax(KQ) * V
8792 // Pack V tile to contiguous F32, zero-padded
8793 for (int tk = 0; tk < kv_tile; tk++) {
8794 const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
8795 if (kv_type == GGML_TYPE_F16) {
8796 ggml_fp16_to_fp32_row((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
8797 } else {
8798 memcpy(V32 + tk * DV, v_data, DV * sizeof(float));
8799 }
8800 }
8801 for (int tq = 0; tq < Q_TILE_SZ; tq++) {
8802 if (skip[tq]) {
8803 memset(KQ + tq * KV_TILE_SZ, 0, KV_TILE_SZ * sizeof(float));
8804 }
8805 }
8806 simd_gemm(VKQ32, KQ, V32, Q_TILE_SZ, KV_TILE_SZ, DV);
8807 }
8808
8809 // sinks (apply only to valid rows in the tile)
8810 if (sinks) {
8811 const float s = ((float *)((char *) sinks->data))[h];
8812
8813 for (int tq = 0; tq < tile_rows; tq++) {
8814 float ms = 1.0f;
8815 float vs = 1.0f;
8816
8817 if (s > M[tq]) {
8818 ms = expf(M[tq] - s);
8819 ggml_vec_scale_f32(DV, VKQ32 + tq * DV, ms);
8820 } else {
8821 vs = expf(s - M[tq]);
8822 }
8823
8824 S[tq] = S[tq] * ms + vs;
8825 }
8826 }
8827
8828 for (int tq = 0; tq < tile_rows; tq++) {
8829 // V /= S
8830 const float S_inv = S[tq] == 0.0f ? 0.0f : 1.0f / S[tq];
8831 ggml_vec_scale_f32(DV, VKQ32 + tq * DV, S_inv);
8832
8833 // dst indices
8834 const int i1 = iq1 + tq;
8835 const int i2 = iq2;
8836 const int i3 = iq3;
8837
8838 // permute(0, 2, 1, 3)
8839 memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, VKQ32 + tq * DV, nb1);
8840 }
8841
8842 ir += tile_rows;
8843 }
8844}
8845
8846// Reduction function: combines partial results across KV chunks
8847// Partials layout in wdata: [n_q_heads][n_chunks][2 + DV]
8848static void ggml_flash_attn_ext_reduce_partials(
8849 const ggml_compute_params * params,
8850 ggml_tensor * dst,
8851 const int64_t n_chunks,
8852 const int64_t chunk_size) {
8853
8854 const ggml_tensor * q = dst->src[0];
8855 const ggml_tensor * k = dst->src[1];
8856 const ggml_tensor * v = dst->src[2];
8857
8858 const int64_t DK = k->ne[0];
8859 const int64_t DV = v->ne[0];
8860 const int64_t nek1 = k->ne[1];
8861 const int64_t n_q_heads = q->ne[2];
8862
8863 const int ith = params->ith;
8864 const int nth = params->nth;
8865
8866 const int64_t wdata_per_thread = DK + 2*DV + CACHE_LINE_SIZE_F32;
8867 float * thread_wdata = (float *) params->wdata + ith * wdata_per_thread;
8868
8869 const int64_t partials_offset = nth * (DK + 2*DV + CACHE_LINE_SIZE_F32);
8870 const int64_t partial_size = 2 + DV;
8871 const float * partials_base = (const float *) params->wdata + partials_offset;
8872
8873 // Output layout
8874 const int64_t ne1 = dst->ne[1];
8875 const int64_t ne2 = dst->ne[2];
8876 const size_t nb1 = dst->nb[1];
8877
8878 // Each thread reduces a subset of query heads
8879 for (int64_t q_head = ith; q_head < n_q_heads; q_head += nth) {
8880 float M_final = -INFINITY(__builtin_inff());
8881 float S_final = 0.0f;
8882 float * VKQ_final = thread_wdata;
8883 memset(VKQ_final, 0, DV * sizeof(float));
8884
8885 // Combine partials from all chunks
8886 for (int64_t chunk_idx = 0; chunk_idx < n_chunks; ++chunk_idx) {
8887 const int64_t ic_start = chunk_idx * chunk_size;
8888 if (ic_start >= nek1) continue;
8889
8890 const float * partial = partials_base + (q_head * n_chunks + chunk_idx) * partial_size;
8891 const float M_chunk = partial[0];
8892 const float S_chunk = partial[1];
8893 const float * VKQ_chunk = partial + 2;
8894
8895 if (S_chunk == 0.0f) continue;
8896
8897 const float M_new = fmaxf(M_final, M_chunk);
8898 const float scale_old = expf(M_final - M_new);
8899 const float scale_new = expf(M_chunk - M_new);
8900
8901 for (int64_t d = 0; d < DV; ++d) {
8902 VKQ_final[d] = VKQ_final[d] * scale_old + VKQ_chunk[d] * scale_new;
8903 }
8904 S_final = S_final * scale_old + S_chunk * scale_new;
8905 M_final = M_new;
8906 }
8907
8908 // Normalize and write to output
8909 if (S_final != 0.0f) {
8910 const float S_inv = 1.0f / S_final;
8911 ggml_vec_scale_f32(DV, VKQ_final, S_inv);
8912 }
8913 // iq1=0, iq3=0 for decode
8914 memcpy((char *) dst->data + (0*ne2*ne1 + q_head + 0*ne1)*nb1, VKQ_final, nb1);
8915 }
8916}
8917
8918static void ggml_compute_forward_flash_attn_ext_f16(
8919 const ggml_compute_params * params,
8920 ggml_tensor * dst) {
8921
8922 const ggml_tensor * q = dst->src[0];
8923 const ggml_tensor * k = dst->src[1];
8924 const ggml_tensor * v = dst->src[2];
8925
8926 GGML_TENSOR_LOCALS(int64_t, neq, q, ne)const int64_t neq0 = (q) ? (q)->ne[0] : 0; (void)(neq0); const
int64_t neq1 = (q) ? (q)->ne[1] : 0; (void)(neq1); const int64_t
neq2 = (q) ? (q)->ne[2] : 0; (void)(neq2); const int64_t neq3
= (q) ? (q)->ne[3] : 0; (void)(neq3);
8927 GGML_TENSOR_LOCALS(size_t, nbq, q, nb)const size_t nbq0 = (q) ? (q)->nb[0] : 0; (void)(nbq0); const
size_t nbq1 = (q) ? (q)->nb[1] : 0; (void)(nbq1); const size_t
nbq2 = (q) ? (q)->nb[2] : 0; (void)(nbq2); const size_t nbq3
= (q) ? (q)->nb[3] : 0; (void)(nbq3);
8928 GGML_TENSOR_LOCALS(int64_t, nek, k, ne)const int64_t nek0 = (k) ? (k)->ne[0] : 0; (void)(nek0); const
int64_t nek1 = (k) ? (k)->ne[1] : 0; (void)(nek1); const int64_t
nek2 = (k) ? (k)->ne[2] : 0; (void)(nek2); const int64_t nek3
= (k) ? (k)->ne[3] : 0; (void)(nek3);
8929 GGML_TENSOR_LOCALS(size_t, nbk, k, nb)const size_t nbk0 = (k) ? (k)->nb[0] : 0; (void)(nbk0); const
size_t nbk1 = (k) ? (k)->nb[1] : 0; (void)(nbk1); const size_t
nbk2 = (k) ? (k)->nb[2] : 0; (void)(nbk2); const size_t nbk3
= (k) ? (k)->nb[3] : 0; (void)(nbk3);
8930 GGML_TENSOR_LOCALS(int64_t, nev, v, ne)const int64_t nev0 = (v) ? (v)->ne[0] : 0; (void)(nev0); const
int64_t nev1 = (v) ? (v)->ne[1] : 0; (void)(nev1); const int64_t
nev2 = (v) ? (v)->ne[2] : 0; (void)(nev2); const int64_t nev3
= (v) ? (v)->ne[3] : 0; (void)(nev3);
8931 GGML_TENSOR_LOCALS(size_t, nbv, v, nb)const size_t nbv0 = (v) ? (v)->nb[0] : 0; (void)(nbv0); const
size_t nbv1 = (v) ? (v)->nb[1] : 0; (void)(nbv1); const size_t
nbv2 = (v) ? (v)->nb[2] : 0; (void)(nbv2); const size_t nbv3
= (v) ? (v)->nb[3] : 0; (void)(nbv3);
8932 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
8933 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
8934
8935 const int64_t DK = nek0;
8936 const int64_t DV = nev0;
8937 const int64_t N = neq1;
8938
8939
8940 GGML_ASSERT(ne0 == DV)if (!(ne0 == DV)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8940, "GGML_ASSERT(%s) failed", "ne0 == DV");
8941 GGML_ASSERT(ne2 == N)if (!(ne2 == N)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8941, "GGML_ASSERT(%s) failed", "ne2 == N");
8942
8943 // input tensor rows must be contiguous
8944 GGML_ASSERT(nbq0 == ggml_type_size(q->type))if (!(nbq0 == ggml_type_size(q->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8944, "GGML_ASSERT(%s) failed", "nbq0 == ggml_type_size(q->type)"
);
8945 GGML_ASSERT(nbk0 == ggml_type_size(k->type))if (!(nbk0 == ggml_type_size(k->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8945, "GGML_ASSERT(%s) failed", "nbk0 == ggml_type_size(k->type)"
);
8946 GGML_ASSERT(nbv0 == ggml_type_size(v->type))if (!(nbv0 == ggml_type_size(v->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8946, "GGML_ASSERT(%s) failed", "nbv0 == ggml_type_size(v->type)"
);
8947
8948 GGML_ASSERT(neq0 == DK)if (!(neq0 == DK)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8948, "GGML_ASSERT(%s) failed", "neq0 == DK");
8949 GGML_ASSERT(nek0 == DK)if (!(nek0 == DK)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8949, "GGML_ASSERT(%s) failed", "nek0 == DK");
8950 GGML_ASSERT(nev0 == DV)if (!(nev0 == DV)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8950, "GGML_ASSERT(%s) failed", "nev0 == DV");
8951
8952 GGML_ASSERT(neq1 == N)if (!(neq1 == N)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8952, "GGML_ASSERT(%s) failed", "neq1 == N");
8953
8954 // dst cannot be transposed or permuted
8955 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8955, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
8956 GGML_ASSERT(nb0 <= nb1)if (!(nb0 <= nb1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8956, "GGML_ASSERT(%s) failed", "nb0 <= nb1");
8957 GGML_ASSERT(nb1 <= nb2)if (!(nb1 <= nb2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8957, "GGML_ASSERT(%s) failed", "nb1 <= nb2");
8958 GGML_ASSERT(nb2 <= nb3)if (!(nb2 <= nb3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 8958, "GGML_ASSERT(%s) failed", "nb2 <= nb3");
8959
8960 const int ith = params->ith;
8961 const int nth = params->nth;
8962
8963 // When use_ref is set, force the vec-only reference implementation (no tiling, no KV-chunking)
8964 const bool use_ref = params->use_ref;
8965
8966 const bool kv_is_f32_or_f16 = (k->type == GGML_TYPE_F32 || k->type == GGML_TYPE_F16);
8967 const bool use_split_kv_path = !use_ref && (neq1 == 1 && neq3 == 1) && kv_is_f32_or_f16 && (k->type == v->type) && q->type == GGML_TYPE_F32 && nek1 >= 512;
8968
8969 if (use_split_kv_path) {
8970 const int64_t chunk_size = (nek1 + nth - 1) / nth;
8971
8972 // Partials buffer layout: [q_head][kv_chunk][M, S, VKQ]
8973 const int64_t partial_size = 2 + DV;
8974 float * partials_base = (float *) params->wdata + nth * (DK + 2*DV + CACHE_LINE_SIZE_F32);
8975
8976 const int64_t ic_start = ith * chunk_size;
8977 const int64_t ic_end = std::min(ic_start + chunk_size, nek1);
8978
8979 const int64_t partial_stride = nth * partial_size;
8980 float * chunk_partials = partials_base + ith * partial_size;
8981
8982 if (ic_start < nek1) {
8983 for (int64_t q_head = 0; q_head < neq2; q_head++) {
8984 ggml_compute_forward_flash_attn_ext_f16_one_chunk(
8985 params, dst, q_head, q_head + 1, ic_start, ic_end,
8986 chunk_partials, partial_stride);
8987 }
8988 } else {
8989 for (int64_t q_head = 0; q_head < neq2; q_head++) {
8990 float * q_partials = chunk_partials + q_head * partial_stride;
8991 q_partials[0] = -INFINITY(__builtin_inff()); // M
8992 q_partials[1] = 0.0f; // S
8993 }
8994 }
8995
8996 ggml_barrier(params->threadpool);
8997 ggml_flash_attn_ext_reduce_partials(params, dst, nth, chunk_size);
8998 } else {
8999
9000 // total rows in q
9001 const int64_t nr = neq1*neq2*neq3;
9002
9003 // disable for NUMA
9004 const bool disable_chunking = ggml_is_numa();
9005
9006 // 4x chunks per thread
9007 int nth_scaled = nth * 4;
9008 int64_t chunk_size = (nr + nth_scaled - 1) / nth_scaled;
9009 int64_t nchunk = (nr + chunk_size - 1) / chunk_size;
9010
9011 if (nth == 1 || nchunk < nth || disable_chunking) {
9012 nchunk = nth;
9013 }
9014
9015 if (ith == 0) {
9016 ggml_threadpool_chunk_set(params->threadpool, nth);
9017 }
9018
9019 ggml_barrier(params->threadpool);
9020
9021 const int64_t dr = (nr + nchunk - 1) / nchunk;
9022
9023 static constexpr int64_t Q_TILE_SZ = ggml_fa_tile_config::Q;
9024 bool use_tiled = !use_ref &&
9025 (q->type == GGML_TYPE_F32 &&
9026 kv_is_f32_or_f16 &&
9027 k->type == v->type &&
9028 neq1 >= Q_TILE_SZ);
9029#ifdef GGML_SIMD
9030#if defined(__ARM_FEATURE_SVE)
9031 const int64_t f32_epr = svcntw();
9032#else
9033 const int64_t f32_epr = GGML_F32_EPR8;
9034#endif
9035 use_tiled &= (DV % f32_epr == 0);
9036#endif
9037 int current_chunk = ith;
9038
9039 while (current_chunk < nchunk) {
9040 const int64_t ir0 = dr * current_chunk;
9041 const int64_t ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
9042
9043 if (use_tiled) {
9044 ggml_compute_forward_flash_attn_ext_tiled(params, dst, ir0, ir1);
9045 } else {
9046 ggml_compute_forward_flash_attn_ext_f16_one_chunk(params, dst, ir0, ir1, 0, nek1, nullptr, 0);
9047 }
9048
9049 current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);
9050 }
9051 }
9052}
9053
9054void ggml_compute_forward_flash_attn_ext(
9055 const ggml_compute_params * params,
9056 ggml_tensor * dst) {
9057 switch (dst->op_params[3]) {
9058 case GGML_PREC_DEFAULT:
9059 case GGML_PREC_F32:
9060 {
9061 // uses F32 accumulators
9062 ggml_compute_forward_flash_attn_ext_f16(params, dst);
9063 } break;
9064 default:
9065 {
9066 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9066, "fatal error");
9067 }
9068 }
9069}
9070
9071// ggml_compute_forward_flash_attn_back
9072
9073static void ggml_compute_forward_flash_attn_back_f32(
9074 const ggml_compute_params * params,
9075 const bool masked,
9076 ggml_tensor * dst) {
9077
9078 const ggml_tensor * q = dst->src[0];
9079 const ggml_tensor * k = dst->src[1];
9080 const ggml_tensor * v = dst->src[2];
9081 const ggml_tensor * d = dst->src[3];
9082
9083 GGML_TENSOR_LOCALS(int64_t, neq, q, ne)const int64_t neq0 = (q) ? (q)->ne[0] : 0; (void)(neq0); const
int64_t neq1 = (q) ? (q)->ne[1] : 0; (void)(neq1); const int64_t
neq2 = (q) ? (q)->ne[2] : 0; (void)(neq2); const int64_t neq3
= (q) ? (q)->ne[3] : 0; (void)(neq3);
9084 GGML_TENSOR_LOCALS(size_t, nbq, q, nb)const size_t nbq0 = (q) ? (q)->nb[0] : 0; (void)(nbq0); const
size_t nbq1 = (q) ? (q)->nb[1] : 0; (void)(nbq1); const size_t
nbq2 = (q) ? (q)->nb[2] : 0; (void)(nbq2); const size_t nbq3
= (q) ? (q)->nb[3] : 0; (void)(nbq3);
9085 GGML_TENSOR_LOCALS(int64_t, nek, k, ne)const int64_t nek0 = (k) ? (k)->ne[0] : 0; (void)(nek0); const
int64_t nek1 = (k) ? (k)->ne[1] : 0; (void)(nek1); const int64_t
nek2 = (k) ? (k)->ne[2] : 0; (void)(nek2); const int64_t nek3
= (k) ? (k)->ne[3] : 0; (void)(nek3);
9086 GGML_TENSOR_LOCALS(size_t, nbk, k, nb)const size_t nbk0 = (k) ? (k)->nb[0] : 0; (void)(nbk0); const
size_t nbk1 = (k) ? (k)->nb[1] : 0; (void)(nbk1); const size_t
nbk2 = (k) ? (k)->nb[2] : 0; (void)(nbk2); const size_t nbk3
= (k) ? (k)->nb[3] : 0; (void)(nbk3);
9087 GGML_TENSOR_LOCALS(int64_t, nev, v, ne)const int64_t nev0 = (v) ? (v)->ne[0] : 0; (void)(nev0); const
int64_t nev1 = (v) ? (v)->ne[1] : 0; (void)(nev1); const int64_t
nev2 = (v) ? (v)->ne[2] : 0; (void)(nev2); const int64_t nev3
= (v) ? (v)->ne[3] : 0; (void)(nev3);
9088 GGML_TENSOR_LOCALS(size_t, nbv, v, nb)const size_t nbv0 = (v) ? (v)->nb[0] : 0; (void)(nbv0); const
size_t nbv1 = (v) ? (v)->nb[1] : 0; (void)(nbv1); const size_t
nbv2 = (v) ? (v)->nb[2] : 0; (void)(nbv2); const size_t nbv3
= (v) ? (v)->nb[3] : 0; (void)(nbv3);
9089 GGML_TENSOR_LOCALS(int64_t, ned, d, ne)const int64_t ned0 = (d) ? (d)->ne[0] : 0; (void)(ned0); const
int64_t ned1 = (d) ? (d)->ne[1] : 0; (void)(ned1); const int64_t
ned2 = (d) ? (d)->ne[2] : 0; (void)(ned2); const int64_t ned3
= (d) ? (d)->ne[3] : 0; (void)(ned3);
9090 GGML_TENSOR_LOCALS(size_t, nbd, d, nb)const size_t nbd0 = (d) ? (d)->nb[0] : 0; (void)(nbd0); const
size_t nbd1 = (d) ? (d)->nb[1] : 0; (void)(nbd1); const size_t
nbd2 = (d) ? (d)->nb[2] : 0; (void)(nbd2); const size_t nbd3
= (d) ? (d)->nb[3] : 0; (void)(nbd3);
9091 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
9092 GGML_TENSOR_LOCALS(size_t, nb, dst, nb)const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const
size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const
size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const
size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
9093
9094 const int ith = params->ith;
9095 const int nth = params->nth;
9096
9097 const int64_t D = neq0;
9098 const int64_t N = neq1;
9099 const int64_t P = nek1 - N;
9100 const int64_t M = P + N;
9101
9102 const int Mup = ggml_up(M, GGML_SOFT_MAX_UNROLL4);
9103 const int mxDM = MAX(D, Mup)((D) > (Mup) ? (D) : (Mup));
9104
9105 // GGML_ASSERT(ne0 == D);
9106 // GGML_ASSERT(ne1 == N);
9107 GGML_ASSERT(P >= 0)if (!(P >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9107, "GGML_ASSERT(%s) failed", "P >= 0");
9108
9109 GGML_ASSERT(nbq0 == sizeof(float))if (!(nbq0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9109, "GGML_ASSERT(%s) failed", "nbq0 == sizeof(float)");
9110 GGML_ASSERT(nbk0 == sizeof(float))if (!(nbk0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9110, "GGML_ASSERT(%s) failed", "nbk0 == sizeof(float)");
9111 GGML_ASSERT(nbv0 == sizeof(float))if (!(nbv0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9111, "GGML_ASSERT(%s) failed", "nbv0 == sizeof(float)");
9112
9113 GGML_ASSERT(neq0 == D)if (!(neq0 == D)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9113, "GGML_ASSERT(%s) failed", "neq0 == D");
9114 GGML_ASSERT(nek0 == D)if (!(nek0 == D)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9114, "GGML_ASSERT(%s) failed", "nek0 == D");
9115 GGML_ASSERT(nev1 == D)if (!(nev1 == D)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9115, "GGML_ASSERT(%s) failed", "nev1 == D");
9116 GGML_ASSERT(ned0 == D)if (!(ned0 == D)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9116, "GGML_ASSERT(%s) failed", "ned0 == D");
9117
9118 GGML_ASSERT(neq1 == N)if (!(neq1 == N)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9118, "GGML_ASSERT(%s) failed", "neq1 == N");
9119 GGML_ASSERT(nek1 == N + P)if (!(nek1 == N + P)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9119, "GGML_ASSERT(%s) failed", "nek1 == N + P");
9120 GGML_ASSERT(nev1 == D)if (!(nev1 == D)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9120, "GGML_ASSERT(%s) failed", "nev1 == D");
9121 GGML_ASSERT(ned1 == N)if (!(ned1 == N)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9121, "GGML_ASSERT(%s) failed", "ned1 == N");
9122
9123 // dst cannot be transposed or permuted
9124 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9124, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
9125 GGML_ASSERT(nb0 <= nb1)if (!(nb0 <= nb1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9125, "GGML_ASSERT(%s) failed", "nb0 <= nb1");
9126 GGML_ASSERT(nb1 <= nb2)if (!(nb1 <= nb2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9126, "GGML_ASSERT(%s) failed", "nb1 <= nb2");
9127 GGML_ASSERT(nb2 <= nb3)if (!(nb2 <= nb3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9127, "GGML_ASSERT(%s) failed", "nb2 <= nb3");
9128
9129 if (ith == 0) {
9130 memset(dst->data, 0, nb0*ne0*ne1*ne2*ne3);
9131 }
9132 ggml_barrier(params->threadpool);
9133
9134 const int64_t elem_q = ggml_nelements(q);
9135 const int64_t elem_k = ggml_nelements(k);
9136
9137 ggml_type result_type = dst->type;
9138 GGML_ASSERT(ggml_blck_size(result_type) == 1)if (!(ggml_blck_size(result_type) == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9138, "GGML_ASSERT(%s) failed", "ggml_blck_size(result_type) == 1"
);
9139 const size_t tsize = ggml_type_size(result_type);
9140
9141 const size_t offs_q = 0;
9142 const size_t offs_k = offs_q + GGML_PAD(elem_q * tsize, GGML_MEM_ALIGN)(((elem_q * tsize) + (16) - 1) & ~((16) - 1));
9143 const size_t offs_v = offs_k + GGML_PAD(elem_k * tsize, GGML_MEM_ALIGN)(((elem_k * tsize) + (16) - 1) & ~((16) - 1));
9144
9145 void * grad_q = (char *) dst->data;
9146 void * grad_k = (char *) dst->data + offs_k;
9147 void * grad_v = (char *) dst->data + offs_v;
9148
9149 const size_t nbgq1 = nb0*neq0;
9150 const size_t nbgq2 = nb0*neq0*neq1;
9151 const size_t nbgq3 = nb0*neq0*neq1*neq2;
9152
9153 const size_t nbgk1 = nb0*nek0;
9154 const size_t nbgk2 = nb0*nek0*nek1;
9155 const size_t nbgk3 = nb0*nek0*nek1*neq2;
9156
9157 const size_t nbgv1 = nb0*nev0;
9158 const size_t nbgv2 = nb0*nev0*nev1;
9159 const size_t nbgv3 = nb0*nev0*nev1*neq2;
9160
9161 // parallelize by k rows using ggml_vec_dot_f32
9162
9163 // total rows in k
9164 const int nr = nek2*nek3;
9165
9166 // rows per thread
9167 const int dr = (nr + nth - 1)/nth;
9168
9169 // row range for this thread
9170 const int ir0 = dr*ith;
9171 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
9172
9173 const float scale = 1.0f/sqrtf(D);
9174
9175 //printf("P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\n", P, N, D, ir0, ir1, scale);
9176
9177 // how often k2 (and v2) is repeated in q2
9178 int nrep = neq2/nek2;
9179
9180 for (int ir = ir0; ir < ir1; ++ir) {
9181 // q indices
9182 const int ik3 = ir/(nek2);
9183 const int ik2 = ir - ik3*nek2;
9184
9185 const int iq3 = ik3;
9186 const int id3 = ik3;
9187 const int iv3 = ik3;
9188 const int iv2 = ik2;
9189
9190 for (int irep = 0; irep < nrep; ++irep) {
9191 const int iq2 = ik2 + irep*nek2;
9192 const int id2 = iq2;
9193
9194 // (ik2 + irep*nek2) % nek2 == ik2
9195 for (int iq1 = 0; iq1 < neq1; ++iq1) {
9196 const int id1 = iq1;
9197
9198 // not sure about CACHE_LINE_SIZE_F32..
9199 // - maybe it must not be multiplied by 2 and excluded from .. in SM 1*(..) offset?
9200 float * S = (float *) params->wdata + ith*2*(mxDM + CACHE_LINE_SIZE_F32) + 0*(mxDM+CACHE_LINE_SIZE_F32);
9201 float * SM = (float *) params->wdata + ith*2*(mxDM + CACHE_LINE_SIZE_F32) + 1*(mxDM+CACHE_LINE_SIZE_F32);
9202
9203 for (int i = M; i < Mup; ++i) {
9204 S[i] = -INFINITY(__builtin_inff());
9205 }
9206
9207 const int64_t masked_begin = masked ? (P + iq1 + 1) : M;
9208 for (int64_t ic = 0; ic < masked_begin; ++ic) {
9209 // k indices
9210 const int ik1 = ic;
9211
9212 // S indices
9213 const int i1 = ik1;
9214
9215 ggml_vec_dot_f32(neq0,
9216 S + i1, 0,
9217 (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,
9218 (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);
9219 }
9220
9221 // scale
9222 ggml_vec_scale_f32(masked_begin, S, scale);
9223
9224 for (int64_t i = masked_begin; i < M; i++) {
9225 S[i] = -INFINITY(__builtin_inff());
9226 }
9227
9228 // softmax
9229 // exclude known -INF S[..] values from max and loop
9230 // dont forget to set their SM values to zero
9231 {
9232 float max = -INFINITY(__builtin_inff());
9233 ggml_vec_max_f32(masked_begin, &max, S);
9234
9235 ggml_float sum = 0.0;
9236 {
9237#ifdef GGML_SOFT_MAX_ACCELERATE
9238 max = -max;
9239 vDSP_vsadd(SM, 1, &max, SM, 1, Mup);
9240 vvexpf(SM, SM, &Mup);
9241 ggml_vec_sum_f32(Mup, &sum, SM);
9242#else
9243 sum = ggml_vec_soft_max_f32(Mup, SM, S, max);
9244#endif
9245 }
9246
9247 assert(sum > 0.0)(static_cast <bool> (sum > 0.0) ? void (0) : __assert_fail
("sum > 0.0", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
9248
9249 sum = 1.0/sum;
9250 ggml_vec_scale_f32(masked_begin, SM, sum);
9251
9252 }
9253
9254 // step-by-step explanation
9255 {
9256 // forward-process shape grads from backward process
9257 // parallel_for ik2,ik3:
9258 // for irep:
9259 // iq2 = ik2 + irep*nek2
9260 // k[:D,:M,:,:] [D,M,:,:] grad[k][:D,:M,ik2,ik3] += grad[kcur]
9261 // q[:D,:N,:,:] [D,N,:,:] grad[q][:D,iq1,iq2,iq3] += grad[qcur]
9262 // v[:M,:D,:,:] [M,D,:,:] grad[v][:M,:D,iv2,iv3] += grad[vcur]
9263 // for iq1:
9264 // kcur = k[:D,:M,ik2,ik3] [D,M,1,1] grad[kcur] = grad[S1].T @ qcur
9265 // qcur = q[:D,iq1,iq2,iq3] [D,1,1,1] grad[qcur] = grad[S1] @ kcur
9266 // vcur = v[:M,:D,iv2,iv3] [M,D,1,1] grad[vcur] = grad[S5].T @ S4
9267 // S0 = -Inf [D,1,1,1]
9268 // ~S1[i] = dot(kcur[:D,i], qcur)
9269 // S1 = qcur @ kcur.T [M,1,1,1] grad[S1] = grad[S2] * scale
9270 // S2 = S1 * scale [M,1,1,1] grad[S2] = diag_mask_zero(grad[S3], P)
9271 // S3 = diag_mask_inf(S2, P) [M,1,1,1] grad[S3] = S4 * (grad[S4] - dot(S4, grad[S4]))
9272 // S4 = softmax(S3) [M,1,1,1] grad[S4] = grad[S5] @ vcur
9273 // ~S5[i] = dot(vcur[:,i], S4)
9274 // S5 = S4 @ vcur.T [D,1,1,1] grad[S5] = d[:D,id1,id2,id3]
9275 // ~dst[i,iq1,iq2,iq3] = S5[i] ^
9276 // dst[:D,iq1,iq2,iq3] = S5 | grad[dst[:D,iq1,iq2,iq3]] = d[:D,id1,id2,id3]
9277 // dst backward-/ grad[dst] = d
9278 //
9279 // output gradients with their dependencies:
9280 //
9281 // grad[kcur] = grad[S1].T @ qcur
9282 // grad[S1] = diag_mask_zero(grad[S3], P) * scale
9283 // grad[S3] = S4 * (grad[S4] - dot(S4, grad[S4]))
9284 // grad[S4] = grad[S5] @ vcur
9285 // grad[S4] = d[:D,id1,id2,id3] @ vcur
9286 // grad[qcur] = grad[S1] @ kcur
9287 // grad[vcur] = grad[S5].T @ S4
9288 // grad[vcur] = d[:D,id1,id2,id3].T @ S4
9289 //
9290 // in post-order:
9291 //
9292 // S1 = qcur @ kcur.T
9293 // S2 = S1 * scale
9294 // S3 = diag_mask_inf(S2, P)
9295 // S4 = softmax(S3)
9296 // grad[S4] = d[:D,id1,id2,id3] @ vcur
9297 // grad[S3] = S4 * (grad[S4] - dot(S4, grad[S4]))
9298 // grad[S1] = diag_mask_zero(grad[S3], P) * scale
9299 // grad[qcur] = grad[S1] @ kcur
9300 // grad[kcur] = grad[S1].T @ qcur
9301 // grad[vcur] = d[:D,id1,id2,id3].T @ S4
9302 //
9303 // using less variables (SM=S4):
9304 //
9305 // S = diag_mask_inf(qcur @ kcur.T * scale, P)
9306 // SM = softmax(S)
9307 // S = d[:D,iq1,iq2,iq3] @ vcur
9308 // dot_SM_gradSM = dot(SM, S)
9309 // S = SM * (S - dot(SM, S))
9310 // S = diag_mask_zero(S, P) * scale
9311 //
9312 // grad[q][:D,iq1,iq2,iq3] += S @ kcur
9313 // grad[k][:D,:M,ik2,ik3] += S.T @ qcur
9314 // grad[v][:M,:D,iv2,iv3] += d[:D,id1,id2,id3].T @ SM
9315 }
9316
9317 // S = gradSM = d[:D,id1,id2,id3] @ vcur[:,:,iv2,iv3]
9318 // S = d[:D,id1,id2,id3] @ vcur[:,:,iv2,iv3]
9319 // for ic:
9320 // S[:M] += vcur[:M,ic,iv2,iv3] * d[ic,id1,id2,id3]
9321 // exclude known future zero S[..] values from operation
9322 ggml_vec_set_f32(masked_begin, S, 0);
9323 for (int64_t ic = 0; ic < D; ++ic) {
9324 ggml_vec_mad_f32(masked_begin,
9325 S,
9326 (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)),
9327 *(float *) ((char *) d->data + (ic*nbd0 + id1*nbd1 + id2*nbd2 + id3*nbd3)));
9328 }
9329
9330 // S = SM * (S - dot(SM, S))
9331 float dot_SM_gradSM = 0;
9332 ggml_vec_dot_f32 (masked_begin, &dot_SM_gradSM, 0, SM, 0, S, 0, 1);
9333 ggml_vec_acc1_f32(M, S, -dot_SM_gradSM);
9334 ggml_vec_mul_f32 (masked_begin, S, S, SM);
9335
9336 // S = diag_mask_zero(S, P) * scale
9337 // already done by above ggml_vec_set_f32
9338
9339 // exclude known zero S[..] values from operation
9340 ggml_vec_scale_f32(masked_begin, S, scale);
9341
9342 // S shape [M,1]
9343 // SM shape [M,1]
9344 // kcur shape [D,M]
9345 // qcur shape [D,1]
9346 // vcur shape [M,D]
9347
9348 // grad[q][:D,iq1,iq2,iq3] += S @ kcur
9349 // grad[q][:D,iq1,iq2,iq3] += shape[M,1] @ shape[D,M]
9350 // for ic:
9351 // grad[q][:D,iq1,iq2,iq3] += S[ic] * kcur[:D,ic,ik2,ik3]
9352 // exclude known zero S[..] values from loop
9353 for (int64_t ic = 0; ic < masked_begin; ++ic) {
9354 ggml_vec_mad_f32(D,
9355 (float *) ((char *) grad_q + (iq1*nbgq1 + iq2*nbgq2 + iq3*nbgq3)),
9356 (float *) ((char *) k->data + (ic*nbk1 + ik2*nbk2 + ik3*nbk3)),
9357 S[ic]);
9358 }
9359
9360 // grad[k][:D,:M,iq2,iq3] += S.T @ qcur
9361 // for ic:
9362 // grad[k][:D,ic,iq2,iq3] += S.T[0,ic] * qcur[:D,0]
9363 // grad[k][:D,ic,iq2,iq3] += S[ic] * qcur[:D,0]
9364 // exclude known zero S[..] values from loop
9365 for (int64_t ic = 0; ic < masked_begin; ++ic) {
9366 ggml_vec_mad_f32(D,
9367 (float *) ((char *) grad_k + (ic*nbgk1 + ik2*nbgk2 + ik3*nbgk3)),
9368 (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)),
9369 S[ic]);
9370 }
9371
9372 // grad[v][:M,:D,iv2,iv3] += d[:D,id1,id2,id3].T @ SM
9373 // for ic:
9374 // grad[v][:M,ic,iv2,iv3] += d[:D,id1,id2,id3].T[0,ic] * SM[:M]
9375 // grad[v][:M,ic,iv2,iv3] += d[ic,id1,id2,id3] * SM[:M]
9376 // exclude known zero SM[..] values from mad
9377 for (int64_t ic = 0; ic < D; ++ic) {
9378 ggml_vec_mad_f32(masked_begin,
9379 (float *) ((char *) grad_v + ( ic*nbgv1 + iv2*nbgv2 + iv3*nbgv3)),
9380 SM,
9381 *(float *) ((char *) d->data + (ic*nbd0 + id1*nbd1 + id2*nbd2 + id3*nbd3)));
9382 }
9383 }
9384 }
9385 }
9386}
9387
9388void ggml_compute_forward_flash_attn_back(
9389 const ggml_compute_params * params,
9390 const bool masked,
9391 ggml_tensor * dst) {
9392
9393 const ggml_tensor * q = dst->src[0];
9394
9395 switch (q->type) {
9396 case GGML_TYPE_F32:
9397 {
9398 ggml_compute_forward_flash_attn_back_f32(params, masked, dst);
9399 } break;
9400 default:
9401 {
9402 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9402, "fatal error");
9403 }
9404 }
9405}
9406
9407// ggml_compute_forward_ssm_conv
9408
9409static void ggml_compute_forward_ssm_conv_f32(
9410 const ggml_compute_params * params,
9411 ggml_tensor * dst) {
9412 const ggml_tensor * src0 = dst->src[0]; // conv_x
9413 const ggml_tensor * src1 = dst->src[1]; // conv1d.weight
9414
9415 const int ith = params->ith;
9416 const int nth = params->nth;
9417
9418 const int nc = src1->ne[0]; // d_conv
9419 const int ncs = src0->ne[0]; // d_conv - 1 + n_t
9420 const int nr = src0->ne[1]; // d_inner
9421 const int n_t = dst->ne[1]; // tokens per sequence
9422 const int n_s = dst->ne[2]; // number of sequences in the batch
9423
9424 GGML_ASSERT( dst->ne[0] == nr)if (!(dst->ne[0] == nr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9424, "GGML_ASSERT(%s) failed", "dst->ne[0] == nr");
9425 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9425, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
9426 GGML_ASSERT(src1->nb[0] == sizeof(float))if (!(src1->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9426, "GGML_ASSERT(%s) failed", "src1->nb[0] == sizeof(float)"
);
9427 GGML_ASSERT(src0->nb[1] == src0->ne[0]*sizeof(float))if (!(src0->nb[1] == src0->ne[0]*sizeof(float))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9427, "GGML_ASSERT(%s) failed", "src0->nb[1] == src0->ne[0]*sizeof(float)"
);
9428
9429 // rows per thread
9430 const int dr = (nr + nth - 1)/nth;
9431
9432 // row range for this thread
9433 const int ir0 = dr*ith;
9434 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
9435 const int ir = ir1 - ir0;
9436
9437 for (int i3 = 0; i3 < n_s; ++i3) {
9438 for (int i2 = 0; i2 < n_t; ++i2) {
9439 // {d_conv - 1 + n_t, d_inner, n_seqs}
9440 // sliding window
9441 const float * s = (const float *) ((const char *) src0->data + ir0*(src0->nb[1]) + i2*(src0->nb[0]) + i3*(src0->nb[2])); // {d_conv, d_inner, n_s}
9442 const float * c = (const float *) ((const char *) src1->data + ir0*(src1->nb[1])); // {d_conv, d_inner}
9443 float * x = (float *) ((char *) dst->data + ir0*(dst->nb[0]) + i2*(dst->nb[1]) + i3*(dst->nb[2])); // {d_inner, n_t, n_s}
9444
9445 // TODO: transpose the output for smaller strides for big batches?
9446 // d_inner
9447 for (int i1 = 0; i1 < ir; ++i1) {
9448 // rowwise dot product
9449 // NOTE: not using ggml_vec_dot_f32, because its sum is in double precision
9450 float sumf = 0.0f;
9451
9452 // d_conv
9453 for (int i0 = 0; i0 < nc; ++i0) {
9454 sumf += s[i0 + i1*ncs] * c[i0 + i1*nc];
9455 }
9456 x[i1] = sumf;
9457 }
9458 }
9459 }
9460}
9461
9462void ggml_compute_forward_ssm_conv(
9463 const ggml_compute_params * params,
9464 ggml_tensor * dst) {
9465 switch (dst->src[0]->type) {
9466 case GGML_TYPE_F32:
9467 {
9468 ggml_compute_forward_ssm_conv_f32(params, dst);
9469 } break;
9470 default:
9471 {
9472 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9472, "fatal error");
9473 }
9474 }
9475}
9476
9477// ggml_compute_forward_ssm_scan
9478
9479static void ggml_compute_forward_ssm_scan_f32(
9480 const ggml_compute_params * params,
9481 ggml_tensor * dst) {
9482 const ggml_tensor * src0 = dst->src[0]; // s {d_state, dim, n_head, n_seqs+}
9483 const ggml_tensor * src1 = dst->src[1]; // x {dim, n_head, n_seq_tokens, n_seqs}
9484 const ggml_tensor * src2 = dst->src[2]; // dt {n_head, n_seq_tokens, n_seqs}
9485 const ggml_tensor * src3 = dst->src[3]; // A {d_state, n_head} or {1, n_head}
9486 const ggml_tensor * src4 = dst->src[4]; // B {d_state, n_group, n_seq_tokens, n_seqs}
9487 const ggml_tensor * src5 = dst->src[5]; // C {d_state, n_group, n_seq_tokens, n_seqs}
9488 const ggml_tensor * src6 = dst->src[6]; // ids {n_seqs}
9489
9490 const int ith = params->ith;
9491 const int nth = params->nth;
9492
9493 const int64_t nc = src0->ne[0]; // d_state
9494 const int64_t nr = src0->ne[1]; // dim
9495 const int64_t nh = src1->ne[1]; // n_head
9496 const int64_t ng = src4->ne[1];
9497 const int64_t nt = src1->ne[2]; // number of tokens per sequence
9498 const int64_t ns = src1->ne[3]; // number of sequences in the batch
9499
9500 // can't use ggml_nbytes because src1 is not necessarily contiguous
9501 const int64_t s_off = ggml_nelements(src1) * ggml_element_size(src1);
9502
9503 GGML_ASSERT(ggml_nelements(src1) + nc*nr*nh*ns == ggml_nelements(dst))if (!(ggml_nelements(src1) + nc*nr*nh*ns == ggml_nelements(dst
))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9503, "GGML_ASSERT(%s) failed", "ggml_nelements(src1) + nc*nr*nh*ns == ggml_nelements(dst)"
);
9504 GGML_ASSERT(src0->nb[0] == sizeof(float))if (!(src0->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9504, "GGML_ASSERT(%s) failed", "src0->nb[0] == sizeof(float)"
);
9505 GGML_ASSERT(src1->nb[0] == sizeof(float))if (!(src1->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9505, "GGML_ASSERT(%s) failed", "src1->nb[0] == sizeof(float)"
);
9506 GGML_ASSERT(src2->nb[0] == sizeof(float))if (!(src2->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9506, "GGML_ASSERT(%s) failed", "src2->nb[0] == sizeof(float)"
);
9507 GGML_ASSERT(src3->nb[0] == sizeof(float))if (!(src3->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9507, "GGML_ASSERT(%s) failed", "src3->nb[0] == sizeof(float)"
);
9508 GGML_ASSERT(src4->nb[0] == sizeof(float))if (!(src4->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9508, "GGML_ASSERT(%s) failed", "src4->nb[0] == sizeof(float)"
);
9509 GGML_ASSERT(src5->nb[0] == sizeof(float))if (!(src5->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9509, "GGML_ASSERT(%s) failed", "src5->nb[0] == sizeof(float)"
);
9510 GGML_ASSERT(src6->nb[0] == sizeof(int32_t))if (!(src6->nb[0] == sizeof(int32_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9510, "GGML_ASSERT(%s) failed", "src6->nb[0] == sizeof(int32_t)"
);
9511 GGML_ASSERT(nh % ng == 0)if (!(nh % ng == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9511, "GGML_ASSERT(%s) failed", "nh % ng == 0");
9512
9513 // heads per thread
9514 const int dh = (nh + nth - 1)/nth;
9515
9516 // head range for this thread
9517 const int ih0 = dh*ith;
9518 const int ih1 = MIN(ih0 + dh, nh)((ih0 + dh) < (nh) ? (ih0 + dh) : (nh));
9519
9520 const int32_t * ids = (const int32_t *) src6->data;
9521
9522 for (int i3 = 0; i3 < ns; ++i3) {
9523 const float * s0 = (const float *) ((const char *) src0->data + ids[i3]*(src0->nb[3])); // {d_state, dim, nh, ns}
9524 float * s = ( float *) (( char *) dst->data + i3*(src0->nb[3]) + s_off); // {d_state, dim, nh, ns}
9525
9526 for (int i2 = 0; i2 < nt; ++i2) {
9527 const float * x = (const float *) ((const char *) src1->data + i2*(src1->nb[2]) + i3*(src1->nb[3])); // {dim, nh, nt, ns}
9528 const float * dt = (const float *) ((const char *) src2->data + i2*(src2->nb[1]) + i3*(src2->nb[2])); // {nh, nt, ns}
9529 const float * A = (const float *) ((const char *) src3->data); // {d_state, nh} or {1, nh}
9530 const float * B = (const float *) ((const char *) src4->data + i2*(src4->nb[2]) + i3*(src4->nb[3])); // {d_state, ng, nt, ns}
9531 const float * C = (const float *) ((const char *) src5->data + i2*(src5->nb[2]) + i3*(src5->nb[3])); // {d_state, ng, nt, ns}
9532 float * y = ( float *) (( char *) dst->data + i2*(nh*nr*sizeof(float)) + i3*(nt*nh*nr*sizeof(float))); // {dim, nh, nt, ns}
9533
9534 if (src3->ne[0] == 1) {
9535 // Mamba-2 has a scalar decay factor per head; dA can be outside the state-wise loop
9536
9537 // n_head
9538 for (int h = ih0; h < ih1; ++h) {
9539 // ref: https://github.com/state-spaces/mamba/blob/62db608da60f6fc790b8ed9f4b3225e95ca15fde/mamba_ssm/ops/triton/softplus.py#L16
9540 const float dt_soft_plus = ggml_compute_softplus_f32(dt[h]);
9541 const float dA = expf(dt_soft_plus * A[h]);
9542 const int g = h / (nh / ng); // repeat_interleave
9543
9544 // dim
9545 for (int i1 = 0; i1 < nr; ++i1) {
9546 const int ii = i1 + h*nr;
9547 const float x_dt = x[ii] * dt_soft_plus;
9548 float sumf = 0.0f;
9549#if defined(GGML_SIMD)
9550 #if defined(__ARM_FEATURE_SVE)
9551 const int ggml_f32_epr = svcntw();
9552 const int ggml_f32_step = 1 * ggml_f32_epr;
9553
9554 const int np = (nc & ~(ggml_f32_step - 1));
9555
9556 GGML_F32_VEC__m256 sum = GGML_F32_VEC_ZERO_mm256_setzero_ps();
9557
9558 GGML_F32_VEC__m256 adA = GGML_F32_VEC_SET1(dA)_mm256_set1_ps(dA);
9559 GGML_F32_VEC__m256 axdt = GGML_F32_VEC_SET1(x_dt)_mm256_set1_ps(x_dt);
9560
9561 for (int i = 0; i < np; i += ggml_f32_step) {
9562 // TODO: maybe unroll more?
9563 for (int j = 0; j < 1; j++) {
9564 GGML_F32_VEC__m256 t0 = GGML_F32_VEC_LOAD_mm256_loadu_ps(s0 + i + j*ggml_f32_epr + ii*nc);
9565 GGML_F32_VEC__m256 t1 = GGML_F32_VEC_LOAD_mm256_loadu_ps(B + i + j*ggml_f32_epr + g*nc);
9566 GGML_F32_VEC__m256 t2 = GGML_F32_VEC_LOAD_mm256_loadu_ps(C + i + j*ggml_f32_epr + g*nc);
9567
9568 t0 = GGML_F32_VEC_MUL_mm256_mul_ps(t0, adA);
9569 t1 = GGML_F32_VEC_MUL_mm256_mul_ps(t1, axdt);
9570
9571 t0 = GGML_F32_VEC_ADD_mm256_add_ps(t0, t1);
9572
9573 sum = GGML_F32_VEC_FMA(sum, t0, t2)_mm256_fmadd_ps(t0, t2, sum);
9574
9575 GGML_F32_VEC_STORE_mm256_storeu_ps(s + i + j*ggml_f32_epr + ii*nc, t0);
9576 }
9577 }
9578
9579 sumf = GGML_F32xt_REDUCE_ONE(sum);
9580 #elif defined(__riscv_v_intrinsic)
9581 // todo: RVV implementation
9582 const int np = 0;
9583 #else
9584 const int np = (nc & ~(GGML_F32_STEP32 - 1));
9585
9586 GGML_F32_VEC__m256 sum[GGML_F32_ARR(32/8)] = { GGML_F32_VEC_ZERO_mm256_setzero_ps() };
9587
9588 GGML_F32_VEC__m256 adA = GGML_F32_VEC_SET1(dA)_mm256_set1_ps(dA);
9589 GGML_F32_VEC__m256 axdt = GGML_F32_VEC_SET1(x_dt)_mm256_set1_ps(x_dt);
9590
9591 GGML_F32_VEC__m256 ax[GGML_F32_ARR(32/8)];
9592 GGML_F32_VEC__m256 ay[GGML_F32_ARR(32/8)];
9593 GGML_F32_VEC__m256 az[GGML_F32_ARR(32/8)];
9594
9595 for (int i = 0; i < np; i += GGML_F32_STEP32) {
9596 for (int j = 0; j < GGML_F32_ARR(32/8); j++) {
9597 ax[j] = GGML_F32_VEC_LOAD_mm256_loadu_ps(s0 + i + j*GGML_F32_EPR8 + ii*nc);
9598 ay[j] = GGML_F32_VEC_LOAD_mm256_loadu_ps(B + i + j*GGML_F32_EPR8 + g*nc);
9599 az[j] = GGML_F32_VEC_LOAD_mm256_loadu_ps(C + i + j*GGML_F32_EPR8 + g*nc);
9600
9601 ax[j] = GGML_F32_VEC_MUL_mm256_mul_ps(ax[j], adA);
9602 ay[j] = GGML_F32_VEC_MUL_mm256_mul_ps(ay[j], axdt);
9603
9604 ax[j] = GGML_F32_VEC_ADD_mm256_add_ps(ax[j], ay[j]);
9605
9606 sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], az[j])_mm256_fmadd_ps(ax[j], az[j], sum[j]);
9607
9608 GGML_F32_VEC_STORE_mm256_storeu_ps(s + i + j*GGML_F32_EPR8 + ii*nc, ax[j]);
9609 }
9610 }
9611
9612 // reduce sum0..sum3 to sum0
9613 GGML_F32_VEC_REDUCE(sumf, sum)do { int offset = (32/8) >> 1; for (int i = 0; i < offset
; ++i) { sum[i] = _mm256_add_ps(sum[i], sum[offset+i]); } offset
>>= 1; for (int i = 0; i < offset; ++i) { sum[i] = _mm256_add_ps
(sum[i], sum[offset+i]); } offset >>= 1; for (int i = 0
; i < offset; ++i) { sum[i] = _mm256_add_ps(sum[i], sum[offset
+i]); } const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(sum
[0]), ((__m128)__builtin_ia32_vextractf128_ps256((__v8sf)(__m256
)(sum[0]), (int)(1)))); const __m128 t1 = _mm_hadd_ps(t0, t0)
; sumf = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); } while
(0);
9614 #endif
9615#else
9616 const int np = 0;
9617#endif
9618 // d_state
9619 for (int i0 = np; i0 < nc; ++i0) {
9620 const int i = i0 + ii*nc;
9621 const int ig = i0 + g*nc;
9622 // state = prev_state * dA + dB * x
9623 const float state = (s0[i] * dA) + (B[ig] * x_dt);
9624 // y = rowwise_dotprod(state, C)
9625 sumf += state * C[ig];
9626 s[i] = state;
9627 }
9628 y[ii] = sumf;
9629 }
9630 }
9631 } else {
9632 // Mamba-1 has an element-wise decay factor for the states
9633
9634 // n_head
9635 for (int h = ih0; h < ih1; ++h) {
9636 // ref: https://github.com/state-spaces/mamba/blob/62db608da60f6fc790b8ed9f4b3225e95ca15fde/mamba_ssm/ops/triton/softplus.py#L16
9637 const float dt_soft_plus = ggml_compute_softplus_f32(dt[h]);
9638 const int g = h / (nh / ng); // repeat_interleave
9639
9640 // dim
9641 for (int i1 = 0; i1 < nr; ++i1) {
9642 const int ii = i1 + h*nr;
9643 const float x_dt = x[ii] * dt_soft_plus;
9644#if defined(__ARM_FEATURE_SVE)
9645 svfloat32_t vx_dt = GGML_F32_VEC_SET1(x_dt)_mm256_set1_ps(x_dt);
9646 svfloat32_t vdt_soft_plus = GGML_F32_VEC_SET1(dt_soft_plus)_mm256_set1_ps(dt_soft_plus);
9647 svfloat32_t r1_vector = GGML_F32_VEC_ZERO_mm256_setzero_ps();
9648
9649 // d_state
9650 // TODO: what happens when (d_state % svcntw()) != 0?
9651 for (int64_t k = 0; k < nc; k += svcntw()) {
9652 svfloat32_t vA = GGML_F32_VEC_LOAD_mm256_loadu_ps(&A[h*nc + k]);
9653 svfloat32_t vB = GGML_F32_VEC_LOAD_mm256_loadu_ps(&B[k + g*nc]);
9654 svfloat32_t vC = GGML_F32_VEC_LOAD_mm256_loadu_ps(&C[k + g*nc]);
9655 svfloat32_t vs0 = GGML_F32_VEC_LOAD_mm256_loadu_ps(&s0[ii*nc + k]);
9656
9657 svfloat32_t t1 = GGML_F32_VEC_MUL_mm256_mul_ps(vdt_soft_plus, vA);
9658 t1 = exp_ps_sve(svptrue_b32(), t1);
9659 svfloat32_t t2 = GGML_F32_VEC_MUL_mm256_mul_ps(vx_dt, vB);
9660
9661 vs0 = GGML_F32_VEC_FMA(t2, vs0, t1)_mm256_fmadd_ps(vs0, t1, t2);
9662 r1_vector = GGML_F32_VEC_ADD_mm256_add_ps(GGML_F32_VEC_MUL_mm256_mul_ps(vs0, vC), r1_vector);
9663
9664 GGML_F32_VEC_STORE_mm256_storeu_ps(&s[ii*nc + k], vs0);
9665 }
9666 y[ii] = GGML_F32xt_REDUCE_ONE(r1_vector);
9667#else
9668 float sumf = 0.0f;
9669 // NOTE: can't really use GGML_SIMD here because d_state is usually 16
9670 // and also because expf is used within the loop.
9671 // d_state
9672 for (int i0 = 0; i0 < nc; ++i0) {
9673 const int i = i0 + ii*nc;
9674 const int ig = i0 + g*nc;
9675 // state = prev_state * dA + dB * x
9676 const float state = (s0[i] * expf(dt_soft_plus * A[i0 + h*nc])) + (B[ig] * x_dt);
9677 // y = rowwise_dotprod(state, C)
9678 sumf += state * C[ig];
9679 s[i] = state;
9680 }
9681 y[ii] = sumf;
9682#endif
9683 }
9684 }
9685 }
9686 // use the output as the source when it's not the first token-wise iteration
9687 s0 = s;
9688 }
9689 }
9690}
9691
9692void ggml_compute_forward_ssm_scan(
9693 const ggml_compute_params * params,
9694 ggml_tensor * dst) {
9695 switch (dst->src[0]->type) {
9696 case GGML_TYPE_F32:
9697 {
9698 ggml_compute_forward_ssm_scan_f32(params, dst);
9699 } break;
9700 default:
9701 {
9702 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9702, "fatal error");
9703 }
9704 }
9705}
9706
9707// ggml_compute_forward_win_part
9708
9709static void ggml_compute_forward_win_part_f32(
9710 const ggml_compute_params * params,
9711 ggml_tensor * dst) {
9712 GGML_UNUSED(params)(void)(params);
9713
9714 const ggml_tensor * src0 = dst->src[0];
9715
9716 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
9717 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
9718
9719 const int32_t nep0 = ((const int32_t *)(dst->op_params))[0];
9720 const int32_t nep1 = ((const int32_t *)(dst->op_params))[1];
9721 const int32_t w = ((const int32_t *)(dst->op_params))[2];
9722
9723 assert(ne00 == ne0)(static_cast <bool> (ne00 == ne0) ? void (0) : __assert_fail
("ne00 == ne0", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
9724 assert(ne3 == nep0*nep1)(static_cast <bool> (ne3 == nep0*nep1) ? void (0) : __assert_fail
("ne3 == nep0*nep1", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
9725
9726 // TODO: optimize / multi-thread
9727 for (int py = 0; py < nep1; ++py) {
9728 for (int px = 0; px < nep0; ++px) {
9729 const int64_t i3 = py*nep0 + px;
9730 for (int64_t i2 = 0; i2 < ne2; ++i2) {
9731 for (int64_t i1 = 0; i1 < ne1; ++i1) {
9732 for (int64_t i0 = 0; i0 < ne0; ++i0) {
9733 const int64_t i02 = py*w + i2;
9734 const int64_t i01 = px*w + i1;
9735 const int64_t i00 = i0;
9736
9737 const int64_t i = i3*ne2*ne1*ne0 + i2*ne1*ne0 + i1*ne0 + i0;
9738 const int64_t j = i02*ne01*ne00 + i01*ne00 + i00;
9739
9740 if (py*w + i2 >= ne02 || px*w + i1 >= ne01) {
9741 ((float *) dst->data)[i] = 0.0f;
9742 } else {
9743 ((float *) dst->data)[i] = ((float *) src0->data)[j];
9744 }
9745 }
9746 }
9747 }
9748 }
9749 }
9750}
9751
9752void ggml_compute_forward_win_part(
9753 const ggml_compute_params * params,
9754 ggml_tensor * dst) {
9755
9756 const ggml_tensor * src0 = dst->src[0];
9757
9758 switch (src0->type) {
9759 case GGML_TYPE_F32:
9760 {
9761 ggml_compute_forward_win_part_f32(params, dst);
9762 } break;
9763 default:
9764 {
9765 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9765, "fatal error");
9766 }
9767 }
9768}
9769
9770// ggml_compute_forward_win_unpart
9771
9772static void ggml_compute_forward_win_unpart_f32(
9773 const ggml_compute_params * params,
9774 ggml_tensor * dst) {
9775 GGML_UNUSED(params)(void)(params);
9776
9777 const ggml_tensor * src0 = dst->src[0];
9778
9779 GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)const int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03);
9780 GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0);
const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1)
; const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2
); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3
);
9781
9782 const int32_t w = ((const int32_t *)(dst->op_params))[0];
9783
9784 // padding
9785 const int px = (w - ne1%w)%w;
9786 //const int py = (w - ne2%w)%w;
9787
9788 const int npx = (px + ne1)/w;
9789 //const int npy = (py + ne2)/w;
9790
9791 assert(ne0 == ne00)(static_cast <bool> (ne0 == ne00) ? void (0) : __assert_fail
("ne0 == ne00", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
9792
9793 // TODO: optimize / multi-thread
9794 for (int64_t i2 = 0; i2 < ne2; ++i2) {
9795 for (int64_t i1 = 0; i1 < ne1; ++i1) {
9796 for (int64_t i0 = 0; i0 < ne0; ++i0) {
9797 const int ip2 = i2/w;
9798 const int ip1 = i1/w;
9799
9800 const int64_t i02 = i2%w;
9801 const int64_t i01 = i1%w;
9802 const int64_t i00 = i0;
9803
9804 const int64_t i = (ip2*npx + ip1)*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00 + i00;
9805 const int64_t j = i2*ne1*ne0 + i1*ne0 + i0;
9806
9807 ((float *) dst->data)[j] = ((float *) src0->data)[i];
9808 }
9809 }
9810 }
9811}
9812
9813void ggml_compute_forward_win_unpart(
9814 const ggml_compute_params * params,
9815 ggml_tensor * dst) {
9816
9817 const ggml_tensor * src0 = dst->src[0];
9818
9819 switch (src0->type) {
9820 case GGML_TYPE_F32:
9821 {
9822 ggml_compute_forward_win_unpart_f32(params, dst);
9823 } break;
9824 default:
9825 {
9826 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9826, "fatal error");
9827 }
9828 }
9829}
9830
9831//ggml_compute_forward_unary
9832
9833void ggml_compute_forward_unary(
9834 const ggml_compute_params * params,
9835 ggml_tensor * dst) {
9836
9837 const ggml_unary_op op = ggml_get_unary_op(dst);
9838
9839 switch (op) {
9840 case GGML_UNARY_OP_ABS:
9841 {
9842 ggml_compute_forward_abs(params, dst);
9843 } break;
9844 case GGML_UNARY_OP_SGN:
9845 {
9846 ggml_compute_forward_sgn(params, dst);
9847 } break;
9848 case GGML_UNARY_OP_NEG:
9849 {
9850 ggml_compute_forward_neg(params, dst);
9851 } break;
9852 case GGML_UNARY_OP_STEP:
9853 {
9854 ggml_compute_forward_step(params, dst);
9855 } break;
9856 case GGML_UNARY_OP_TANH:
9857 {
9858 ggml_compute_forward_tanh(params, dst);
9859 } break;
9860 case GGML_UNARY_OP_ELU:
9861 {
9862 ggml_compute_forward_elu(params, dst);
9863 } break;
9864 case GGML_UNARY_OP_RELU:
9865 {
9866 ggml_compute_forward_relu(params, dst);
9867 } break;
9868 case GGML_UNARY_OP_SIGMOID:
9869 {
9870 ggml_compute_forward_sigmoid(params, dst);
9871 } break;
9872 case GGML_UNARY_OP_GELU:
9873 {
9874 ggml_compute_forward_gelu(params, dst);
9875 } break;
9876 case GGML_UNARY_OP_GELU_ERF:
9877 {
9878 ggml_compute_forward_gelu_erf(params, dst);
9879 } break;
9880 case GGML_UNARY_OP_GELU_QUICK:
9881 {
9882 ggml_compute_forward_gelu_quick(params, dst);
9883 } break;
9884 case GGML_UNARY_OP_SILU:
9885 {
9886 ggml_compute_forward_silu(params, dst);
9887 } break;
9888 case GGML_UNARY_OP_HARDSWISH:
9889 {
9890 ggml_compute_forward_hardswish(params, dst);
9891 } break;
9892 case GGML_UNARY_OP_HARDSIGMOID:
9893 {
9894 ggml_compute_forward_hardsigmoid(params, dst);
9895 } break;
9896 case GGML_UNARY_OP_EXP:
9897 {
9898 ggml_compute_forward_exp(params, dst);
9899 } break;
9900 case GGML_UNARY_OP_FLOOR:
9901 {
9902 ggml_compute_forward_floor(params, dst);
9903 } break;
9904 case GGML_UNARY_OP_CEIL:
9905 {
9906 ggml_compute_forward_ceil(params, dst);
9907 } break;
9908 case GGML_UNARY_OP_ROUND:
9909 {
9910 ggml_compute_forward_round(params, dst);
9911 } break;
9912 case GGML_UNARY_OP_TRUNC:
9913 {
9914 ggml_compute_forward_trunc(params, dst);
9915 } break;
9916 case GGML_UNARY_OP_XIELU:
9917 {
9918 ggml_compute_forward_xielu(params, dst);
9919 } break;
9920 case GGML_UNARY_OP_EXPM1:
9921 {
9922 ggml_compute_forward_expm1(params, dst);
9923 } break;
9924 case GGML_UNARY_OP_SOFTPLUS:
9925 {
9926 ggml_compute_forward_softplus(params, dst);
9927 } break;
9928 default:
9929 {
9930 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9930, "fatal error");
9931 }
9932 }
9933}
9934
9935//ggml_compute_forward_glu
9936
9937void ggml_compute_forward_glu(
9938 const ggml_compute_params * params,
9939 ggml_tensor * dst) {
9940
9941 const ggml_glu_op op = ggml_get_glu_op(dst);
9942
9943 switch (op) {
9944 case GGML_GLU_OP_REGLU:
9945 {
9946 ggml_compute_forward_reglu(params, dst);
9947 } break;
9948 case GGML_GLU_OP_GEGLU:
9949 {
9950 ggml_compute_forward_geglu(params, dst);
9951 } break;
9952 case GGML_GLU_OP_SWIGLU:
9953 {
9954 ggml_compute_forward_swiglu(params, dst);
9955 } break;
9956 case GGML_GLU_OP_SWIGLU_OAI:
9957 {
9958 ggml_compute_forward_swiglu_oai(params, dst);
9959 } break;
9960 case GGML_GLU_OP_GEGLU_ERF:
9961 {
9962 ggml_compute_forward_geglu_erf(params, dst);
9963 } break;
9964 case GGML_GLU_OP_GEGLU_QUICK:
9965 {
9966 ggml_compute_forward_geglu_quick(params, dst);
9967 } break;
9968 default:
9969 {
9970 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 9970, "fatal error");
9971 }
9972 }
9973}
9974
9975// ggml_compute_forward_get_rel_pos
9976
9977static void ggml_compute_forward_get_rel_pos_f16(
9978 const ggml_compute_params * params,
9979 ggml_tensor * dst) {
9980 GGML_UNUSED(params)(void)(params);
9981
9982 const ggml_tensor * src0 = dst->src[0];
9983
9984 // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L292-L322
9985
9986 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
9987
9988 const int64_t w = ne1;
9989
9990 ggml_fp16_t * src0_data = (ggml_fp16_t *) src0->data;
9991 ggml_fp16_t * dst_data = (ggml_fp16_t *) dst->data;
9992
9993 for (int64_t i2 = 0; i2 < ne2; ++i2) {
9994 for (int64_t i1 = 0; i1 < ne1; ++i1) {
9995 const int64_t pos = (w - i1 - 1) + i2;
9996 for (int64_t i0 = 0; i0 < ne0; ++i0) {
9997 dst_data[i2*ne1*ne0 + i1*ne0 + i0] = src0_data[pos*ne00 + i0];
9998 }
9999 }
10000 }
10001}
10002
10003void ggml_compute_forward_get_rel_pos(
10004 const ggml_compute_params * params,
10005 ggml_tensor * dst) {
10006
10007 const ggml_tensor * src0 = dst->src[0];
10008
10009 switch (src0->type) {
10010 case GGML_TYPE_F16:
10011 case GGML_TYPE_BF16:
10012 {
10013 ggml_compute_forward_get_rel_pos_f16(params, dst);
10014 } break;
10015 default:
10016 {
10017 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10017, "fatal error");
10018 }
10019 }
10020}
10021
10022// ggml_compute_forward_add_rel_pos
10023
10024static void ggml_compute_forward_add_rel_pos_f32(
10025 const ggml_compute_params * params,
10026 ggml_tensor * dst) {
10027
10028 const ggml_tensor * src0 = dst->src[0];
10029 const ggml_tensor * src1 = dst->src[1];
10030 const ggml_tensor * src2 = dst->src[2];
10031
10032 const bool inplace = (bool) ((int32_t *) dst->op_params)[0];
10033 if (!inplace) {
10034 if (params->ith == 0) {
10035 memcpy((char *) dst->data, (char *) src0->data, ggml_nbytes(dst));
10036 }
10037 ggml_barrier(params->threadpool);
10038 }
10039 // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L357-L359
10040
10041 float * src1_data = (float *) src1->data;
10042 float * src2_data = (float *) src2->data;
10043 float * dst_data = (float *) dst->data;
10044
10045 const int64_t ne10 = src1->ne[0];
10046 const int64_t ne11 = src1->ne[1];
10047 const int64_t ne12 = src1->ne[2];
10048 const int64_t ne13 = src1->ne[3];
10049
10050 const int ith = params->ith;
10051 const int nth = params->nth;
10052
10053 // total patches in dst
10054 const int np = ne13;
10055
10056 // patches per thread
10057 const int dp = (np + nth - 1)/nth;
10058
10059 // patch range for this thread
10060 const int ip0 = dp*ith;
10061 const int ip1 = MIN(ip0 + dp, np)((ip0 + dp) < (np) ? (ip0 + dp) : (np));
10062
10063 for (int64_t i13 = ip0; i13 < ip1; ++i13) {
10064 for (int64_t i12 = 0; i12 < ne12; ++i12) {
10065 for (int64_t i11 = 0; i11 < ne11; ++i11) {
10066 const int64_t jp1 = i13*ne12*ne11*ne10 + i12*ne11*ne10 + i11*ne10;
10067 for (int64_t i10 = 0; i10 < ne10; ++i10) {
10068 const int64_t jp0 = jp1 + i10;
10069 const float src1_e = src1_data[jp0];
10070 const float src2_e = src2_data[jp0];
10071
10072 const int64_t jdh = jp0 * ne10;
10073 const int64_t jdw = jdh - (ne10 - 1) * i10;
10074
10075 for (int64_t j = 0; j < ne10; ++j) {
10076 dst_data[jdh + j ] += src2_e;
10077 dst_data[jdw + j*ne10] += src1_e;
10078 }
10079 }
10080 }
10081 }
10082 }
10083}
10084
10085void ggml_compute_forward_add_rel_pos(
10086 const ggml_compute_params * params,
10087 ggml_tensor * dst) {
10088
10089 const ggml_tensor * src0 = dst->src[0];
10090
10091 switch (src0->type) {
10092 case GGML_TYPE_F32:
10093 {
10094 ggml_compute_forward_add_rel_pos_f32(params, dst);
10095 } break;
10096 default:
10097 {
10098 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10098, "fatal error");
10099 }
10100 }
10101}
10102
10103// ggml_compute_forward_rwkv_wkv6
10104
10105static void ggml_compute_forward_rwkv_wkv6_f32(
10106 const ggml_compute_params * params,
10107 ggml_tensor * dst) {
10108 const int64_t T = dst->src[1]->ne[2];
10109 const int64_t C = dst->ne[0];
10110 const int64_t HEADS = dst->src[1]->ne[1];
10111 const int64_t n_seqs = dst->src[5]->ne[1];
10112 const int64_t head_size = C / HEADS;
10113
10114 float * dst_data = (float *) dst->data;
10115 float * state = ((float *) dst->data) + C * T;
10116
10117 const int ith = params->ith;
10118 const int nth = params->nth;
10119
10120 const int h_start = (HEADS * (ith )) / nth;
10121 const int h_end = ((HEADS * (ith + 1)) / nth < HEADS) ?
10122 (HEADS * (ith + 1)) / nth : HEADS;
10123
10124 float * k = (float *) dst->src[0]->data;
10125 float * v = (float *) dst->src[1]->data;
10126 float * r = (float *) dst->src[2]->data;
10127 float * time_faaaa = (float *) dst->src[3]->data;
10128 float * time_decay = (float *) dst->src[4]->data;
10129
10130 size_t t_stride = HEADS * head_size; // Same to C
10131
10132 size_t h_stride = C / HEADS;
10133 GGML_ASSERT(C % HEADS == 0)if (!(C % HEADS == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10133, "GGML_ASSERT(%s) failed", "C % HEADS == 0"); // C must be divisible by HEADS
10134 size_t h_stride_2d = head_size * head_size;
10135
10136 if (ith == 0) {
10137 memset(dst_data, 0, T * C * sizeof(float));
10138 }
10139 ggml_barrier(params->threadpool);
10140
10141
10142 #if defined(__AVX__1) && !defined(__AVX512F__)
10143 #define GGML_F32X__m256 GGML_F32x8__m256
10144 #define GGML_F32X_SET1GGML_F32x8_SET1 GGML_F32x8_SET1
10145 #define GGML_F32X_LOAD_mm256_loadu_ps GGML_F32x8_LOAD_mm256_loadu_ps
10146 #define GGML_F32X_STORE_mm256_storeu_ps GGML_F32x8_STORE_mm256_storeu_ps
10147 #define GGML_F32X_MUL_mm256_mul_ps GGML_F32x8_MUL_mm256_mul_ps
10148 #define GGML_F32X_FMAGGML_F32x8_FMA GGML_F32x8_FMA
10149 #define WKV_VECTOR_SIZE8 8
10150 #elif defined(__AVX512F__)
10151 #define GGML_F32X__m256 GGML_F32x16
10152 #define GGML_F32X_SET1GGML_F32x8_SET1 GGML_F32x16_SET1
10153 #define GGML_F32X_LOAD_mm256_loadu_ps GGML_F32x16_LOAD
10154 #define GGML_F32X_STORE_mm256_storeu_ps GGML_F32x16_STORE
10155 #define GGML_F32X_MUL_mm256_mul_ps GGML_F32x16_MUL
10156 #define GGML_F32X_FMAGGML_F32x8_FMA GGML_F32x16_FMA
10157 #define WKV_VECTOR_SIZE8 16
10158 #elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)
10159 #define GGML_F32X__m256 GGML_F32xt
10160 #define GGML_F32X_SET1GGML_F32x8_SET1 GGML_F32xt_SET1
10161 #define GGML_F32X_LOAD_mm256_loadu_ps GGML_F32xt_LOAD
10162 #define GGML_F32X_STORE_mm256_storeu_ps GGML_F32xt_STORE
10163 #define GGML_F32X_MUL_mm256_mul_ps GGML_F32xt_MUL
10164 #define GGML_F32X_FMAGGML_F32x8_FMA GGML_F32xt_FMA
10165 #define WKV_VECTOR_SIZE8 8
10166 #elif defined(__ARM_NEON) && defined(__aarch64__)
10167 #define GGML_F32X__m256 GGML_F32x4
10168 #define GGML_F32X_SET1GGML_F32x8_SET1 GGML_F32x4_SET1
10169 #define GGML_F32X_LOAD_mm256_loadu_ps GGML_F32x4_LOAD
10170 #define GGML_F32X_STORE_mm256_storeu_ps GGML_F32x4_STORE
10171 #define GGML_F32X_MUL_mm256_mul_ps GGML_F32x4_MUL
10172 #define GGML_F32X_FMAGGML_F32x8_FMA GGML_F32x4_FMA
10173 #define WKV_VECTOR_SIZE8 4
10174 #endif
10175
10176 #ifdef WKV_VECTOR_SIZE8
10177 int wkv_vector_size;
10178 #if defined(__ARM_FEATURE_SVE)
10179 wkv_vector_size = svcntw();
10180 #else
10181 wkv_vector_size = WKV_VECTOR_SIZE8;
10182 #endif
10183 const int64_t vec_count = head_size / wkv_vector_size;
10184
10185 for (int64_t t = 0; t < T; t++) {
10186 size_t t_offset = t * t_stride;
10187 size_t state_offset = head_size * C * (t / (T / n_seqs));
10188 float * state_cur = state + state_offset;
10189 float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[5]->data + state_offset;
10190
10191 for (int64_t h = h_start; h < h_end; h++) {
10192 size_t h_offset = h * h_stride;
10193 size_t t_h_offset = t_offset + h_offset;
10194 size_t h_2d_offset = h * h_stride_2d;
10195
10196 for (int64_t i = 0; i < head_size; i++) {
10197 size_t t_h_i_offset = t_h_offset + i;
10198 size_t h_i_offset = h_offset + i;
10199 size_t h_2d_i_offset = h_2d_offset + i * h_stride;
10200
10201 float k_val = k[t_h_i_offset];
10202 float r_val = r[t_h_i_offset];
10203 float time_faaaa_val = time_faaaa[h_i_offset];
10204 float time_decay_val = time_decay[t_h_i_offset];
10205
10206 // Broadcast scalar values to vectors
10207 GGML_F32X__m256 k_vec = GGML_F32X_SET1(k_val)_mm256_set1_ps(k_val);
10208 GGML_F32X__m256 r_vec = GGML_F32X_SET1(r_val)_mm256_set1_ps(r_val);
10209 GGML_F32X__m256 time_faaaa_vec = GGML_F32X_SET1(time_faaaa_val)_mm256_set1_ps(time_faaaa_val);
10210 GGML_F32X__m256 time_decay_vec = GGML_F32X_SET1(time_decay_val)_mm256_set1_ps(time_decay_val);
10211
10212 for (int64_t j = 0; j < vec_count; j++) {
10213 size_t base_j = j * wkv_vector_size;
10214 size_t t_h_j_offset = t_h_offset + base_j;
10215 size_t h_2d_i_j_offset = h_2d_i_offset + base_j;
10216
10217 // Load x elements at once
10218 GGML_F32X__m256 v_vec = GGML_F32X_LOAD_mm256_loadu_ps(&v[t_h_j_offset]);
10219 GGML_F32X__m256 prev_state_vec = GGML_F32X_LOAD_mm256_loadu_ps(&state_prev[h_2d_i_j_offset]);
10220 GGML_F32X__m256 dst_vec = GGML_F32X_LOAD_mm256_loadu_ps(&dst_data[t_h_j_offset]);
10221
10222 // Compute kv = v * k
10223 GGML_F32X__m256 kv_vec = GGML_F32X_MUL_mm256_mul_ps(v_vec, k_vec);
10224
10225 // Compute temp = kv * time_faaaa + prev_state
10226 GGML_F32X__m256 temp_vec = GGML_F32X_FMA(prev_state_vec, kv_vec, time_faaaa_vec)_mm256_fmadd_ps(kv_vec, time_faaaa_vec, prev_state_vec);
10227
10228 // Update dst: dst += temp * r
10229 dst_vec = GGML_F32X_FMA(dst_vec, temp_vec, r_vec)_mm256_fmadd_ps(temp_vec, r_vec, dst_vec);
10230 GGML_F32X_STORE_mm256_storeu_ps(&dst_data[t_h_j_offset], dst_vec);
10231
10232 // Update state: state = prev_state * time_decay + kv
10233 GGML_F32X__m256 new_state_vec = GGML_F32X_FMA(kv_vec, prev_state_vec, time_decay_vec)_mm256_fmadd_ps(prev_state_vec, time_decay_vec, kv_vec);
10234 GGML_F32X_STORE_mm256_storeu_ps(&state_cur[h_2d_i_j_offset], new_state_vec);
10235 }
10236
10237 // Handle remaining elements, this will not be used.
10238 for (int64_t j = vec_count * wkv_vector_size; j < head_size; j++) {
10239 size_t t_h_j_offset = t_h_offset + j;
10240 size_t h_2d_i_j_offset = h_2d_i_offset + j;
10241 float v_val = v[t_h_j_offset];
10242 float kv_val = v_val * k_val;
10243 float prev_state_val = state_prev[h_2d_i_j_offset];
10244 float temp_val = kv_val * time_faaaa_val + prev_state_val;
10245 dst_data[t_h_j_offset] += temp_val * r_val;
10246 state_cur[h_2d_i_j_offset] = prev_state_val * time_decay_val + kv_val;
10247 }
10248 }
10249 }
10250 }
10251
10252 #else
10253 // basically fused operations:
10254 // dst = r @ (time_faaaa * (k @ v) + state),
10255 // state = time_decay * state + (k @ v),
10256 // recursive through each token
10257 for (int64_t t = 0; t < T; t++) {
10258 size_t t_offset = t * t_stride;
10259 size_t state_offset = head_size * C * (t / (T / n_seqs));
10260 float * state_cur = state + state_offset;
10261 float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[5]->data + state_offset;
10262
10263 for (int64_t h = h_start; h < h_end; h++) {
10264 size_t h_offset = h * h_stride;
10265 size_t t_h_offset = t_offset + h_offset;
10266 size_t h_2d_offset = h * h_stride_2d;
10267
10268 for (int64_t i = 0; i < head_size; i++) {
10269 size_t t_h_i_offset = t_h_offset + i;
10270 size_t h_i_offset = h_offset + i;
10271 size_t h_2d_i_offset = h_2d_offset + i * h_stride;
10272
10273 float k_val = k[t_h_i_offset];
10274 float r_val = r[t_h_i_offset];
10275 float time_faaaa_val = time_faaaa[h_i_offset];
10276 // RWKV v6: different time_decay for each token.
10277 float time_decay_val = time_decay[t_h_i_offset];
10278
10279 for (int64_t j = 0; j < head_size; j++) {
10280 size_t t_h_j_offset = t_h_offset + j;
10281 size_t h_2d_i_j_offset = h_2d_i_offset + j;
10282
10283 float v_val = v[t_h_j_offset];
10284 float kv_val = v_val * k_val;
10285 float prev_state_val = state_prev[h_2d_i_j_offset];
10286 float temp_val = kv_val * time_faaaa_val + prev_state_val;
10287 dst_data[t_h_j_offset] += temp_val * r_val;
10288 state_cur[h_2d_i_j_offset] = prev_state_val * time_decay_val + kv_val;
10289 }
10290 }
10291 }
10292 }
10293 #endif
10294}
10295
10296
10297void ggml_compute_forward_rwkv_wkv6(
10298 const ggml_compute_params * params,
10299 ggml_tensor * dst) {
10300
10301 const ggml_tensor * src0 = dst->src[0];
10302
10303 switch (src0->type) {
10304 case GGML_TYPE_F32:
10305 {
10306 ggml_compute_forward_rwkv_wkv6_f32(params, dst);
10307 } break;
10308 default:
10309 {
10310 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10310, "fatal error");
10311 }
10312 }
10313}
10314
10315// ggml_compute_forward_gla
10316
10317static void ggml_compute_forward_gla_f32(
10318 const ggml_compute_params * params,
10319 ggml_tensor * dst) {
10320 const int64_t T = dst->src[1]->ne[2];
10321 const int64_t C = dst->ne[0];
10322 const int64_t HEADS = dst->src[1]->ne[1];
10323 const int64_t n_seqs = dst->src[4]->ne[1];
10324 const int64_t head_size = C / HEADS;
10325 const float scale = ggml_get_op_params_f32(dst, 0);
10326
10327 float * dst_data = (float *) dst->data;
10328 float * state = ((float *) dst->data) + C * T;
10329
10330 const int ith = params->ith;
10331 const int nth = params->nth;
10332
10333 const int h_start = (HEADS * (ith )) / nth;
10334 const int h_end = ((HEADS * (ith + 1)) / nth < HEADS) ?
10335 (HEADS * (ith + 1)) / nth : HEADS;
10336
10337 float * k = (float *) dst->src[0]->data;
10338 float * v = (float *) dst->src[1]->data;
10339 float * q = (float *) dst->src[2]->data;
10340 float * g = (float *) dst->src[3]->data;
10341
10342 size_t t_stride = HEADS * head_size; // Same to C
10343
10344 size_t h_stride = C / HEADS;
10345 GGML_ASSERT(C % HEADS == 0)if (!(C % HEADS == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10345, "GGML_ASSERT(%s) failed", "C % HEADS == 0"); // C must be divisible by HEADS
10346 size_t h_stride_2d = head_size * head_size;
10347
10348 if (ith == 0) {
10349 memset(dst_data, 0, T * C * sizeof(float));
10350 }
10351 ggml_barrier(params->threadpool);
10352
10353
10354 #if defined(__AVX__1) && !defined(__AVX512F__)
10355 #define GGML_F32X__m256 GGML_F32x8__m256
10356 #define GGML_F32X_SET1GGML_F32x8_SET1 GGML_F32x8_SET1
10357 #define GGML_F32X_LOAD_mm256_loadu_ps GGML_F32x8_LOAD_mm256_loadu_ps
10358 #define GGML_F32X_STORE_mm256_storeu_ps GGML_F32x8_STORE_mm256_storeu_ps
10359 #define GGML_F32X_MUL_mm256_mul_ps GGML_F32x8_MUL_mm256_mul_ps
10360 #define GGML_F32X_FMAGGML_F32x8_FMA GGML_F32x8_FMA
10361 #define GLA_VECTOR_SIZE8 8
10362 #elif defined(__AVX512F__)
10363 #define GGML_F32X__m256 GGML_F32x16
10364 #define GGML_F32X_SET1GGML_F32x8_SET1 GGML_F32x16_SET1
10365 #define GGML_F32X_LOAD_mm256_loadu_ps GGML_F32x16_LOAD
10366 #define GGML_F32X_STORE_mm256_storeu_ps GGML_F32x16_STORE
10367 #define GGML_F32X_MUL_mm256_mul_ps GGML_F32x16_MUL
10368 #define GGML_F32X_FMAGGML_F32x8_FMA GGML_F32x16_FMA
10369 #define GLA_VECTOR_SIZE8 16
10370 #elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)
10371 #define GGML_F32X__m256 GGML_F32xt
10372 #define GGML_F32X_SET1GGML_F32x8_SET1 GGML_F32xt_SET1
10373 #define GGML_F32X_LOAD_mm256_loadu_ps GGML_F32xt_LOAD
10374 #define GGML_F32X_STORE_mm256_storeu_ps GGML_F32xt_STORE
10375 #define GGML_F32X_MUL_mm256_mul_ps GGML_F32xt_MUL
10376 #define GGML_F32X_FMAGGML_F32x8_FMA GGML_F32xt_FMA
10377 #define GLA_VECTOR_SIZE8 8
10378 #elif defined(__ARM_NEON) && defined(__aarch64__)
10379 #define GGML_F32X__m256 GGML_F32x4
10380 #define GGML_F32X_SET1GGML_F32x8_SET1 GGML_F32x4_SET1
10381 #define GGML_F32X_LOAD_mm256_loadu_ps GGML_F32x4_LOAD
10382 #define GGML_F32X_STORE_mm256_storeu_ps GGML_F32x4_STORE
10383 #define GGML_F32X_MUL_mm256_mul_ps GGML_F32x4_MUL
10384 #define GGML_F32X_FMAGGML_F32x8_FMA GGML_F32x4_FMA
10385 #define GLA_VECTOR_SIZE8 4
10386 #endif
10387
10388 #ifdef GLA_VECTOR_SIZE8
10389 int gla_vector_size;
10390 #if defined(__ARM_FEATURE_SVE)
10391 gla_vector_size = svcntw();
10392 #else
10393 gla_vector_size = GLA_VECTOR_SIZE8;
10394 #endif
10395 const int64_t vec_count = head_size / gla_vector_size;
10396
10397 for (int64_t t = 0; t < T; t++) {
10398 size_t t_offset = t * t_stride;
10399 size_t state_offset = head_size * C * (t / (T / n_seqs));
10400 float * state_cur = state + state_offset;
10401 float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[4]->data + state_offset;
10402
10403 for (int64_t h = h_start; h < h_end; h++) {
10404 size_t h_offset = h * h_stride;
10405 size_t t_h_offset = t_offset + h_offset;
10406 size_t h_2d_offset = h * h_stride_2d;
10407
10408 for (int64_t i = 0; i < head_size; i++) {
10409 size_t t_h_i_offset = t_h_offset + i;
10410 size_t h_2d_i_offset = h_2d_offset + i * h_stride;
10411
10412 float k_val = k[t_h_i_offset];
10413 float q_val = q[t_h_i_offset] * scale;
10414 float g_val = g[t_h_i_offset];
10415
10416 // Broadcast scalar values to vectors
10417 GGML_F32X__m256 k_vec = GGML_F32X_SET1(k_val)_mm256_set1_ps(k_val);
10418 GGML_F32X__m256 q_vec = GGML_F32X_SET1(q_val)_mm256_set1_ps(q_val);
10419 GGML_F32X__m256 g_vec = GGML_F32X_SET1(g_val)_mm256_set1_ps(g_val);
10420
10421 for (int64_t j = 0; j < vec_count; j++) {
10422 size_t base_j = j * gla_vector_size;
10423 size_t t_h_j_offset = t_h_offset + base_j;
10424 size_t h_2d_i_j_offset = h_2d_i_offset + base_j;
10425
10426 // Load x elements at once
10427 GGML_F32X__m256 v_vec = GGML_F32X_LOAD_mm256_loadu_ps(&v[t_h_j_offset]);
10428 GGML_F32X__m256 prev_state_vec = GGML_F32X_LOAD_mm256_loadu_ps(&state_prev[h_2d_i_j_offset]);
10429 GGML_F32X__m256 dst_vec = GGML_F32X_LOAD_mm256_loadu_ps(&dst_data[t_h_j_offset]);
10430
10431 // Compute kv = v * k
10432 GGML_F32X__m256 kv_vec = GGML_F32X_MUL_mm256_mul_ps(v_vec, k_vec);
10433
10434 // Compute temp = prev_state * g + kv
10435 GGML_F32X__m256 temp_vec = GGML_F32X_FMA(kv_vec, prev_state_vec, g_vec)_mm256_fmadd_ps(prev_state_vec, g_vec, kv_vec);
10436
10437 // Update dst: dst += temp * q
10438 dst_vec = GGML_F32X_FMA(dst_vec, temp_vec, q_vec)_mm256_fmadd_ps(temp_vec, q_vec, dst_vec);
10439 GGML_F32X_STORE_mm256_storeu_ps(&dst_data[t_h_j_offset], dst_vec);
10440
10441 // Update state
10442 GGML_F32X_STORE_mm256_storeu_ps(&state_cur[h_2d_i_j_offset], temp_vec);
10443 }
10444
10445 // Handle remaining elements, this will not be used.
10446 for (int64_t j = vec_count * gla_vector_size; j < head_size; j++) {
10447 size_t t_h_j_offset = t_h_offset + j;
10448 size_t h_2d_i_j_offset = h_2d_i_offset + j;
10449 float v_val = v[t_h_j_offset];
10450 float kv_val = v_val * k_val;
10451 float prev_state_val = state_prev[h_2d_i_j_offset];
10452 float temp_val = kv_val + prev_state_val * g_val;
10453 dst_data[t_h_j_offset] += temp_val * q_val;
10454 state_cur[h_2d_i_j_offset] = temp_val;
10455 }
10456 }
10457 }
10458 }
10459
10460 #else
10461 for (int64_t t = 0; t < T; t++) {
10462 size_t t_offset = t * t_stride;
10463 size_t state_offset = head_size * C * (t / (T / n_seqs));
10464 float * state_cur = state + state_offset;
10465 float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[4]->data + state_offset;
10466
10467 for (int64_t h = h_start; h < h_end; h++) {
10468 size_t h_offset = h * h_stride;
10469 size_t t_h_offset = t_offset + h_offset;
10470 size_t h_2d_offset = h * h_stride_2d;
10471
10472 for (int64_t i = 0; i < head_size; i++) {
10473 size_t t_h_i_offset = t_h_offset + i;
10474 size_t h_2d_i_offset = h_2d_offset + i * h_stride;
10475
10476 float k_val = k[t_h_i_offset];
10477 float q_val = q[t_h_i_offset] * scale;
10478 float g_val = g[t_h_i_offset];
10479
10480 for (int64_t j = 0; j < head_size; j++) {
10481 size_t t_h_j_offset = t_h_offset + j;
10482 size_t h_2d_i_j_offset = h_2d_i_offset + j;
10483
10484 float v_val = v[t_h_j_offset];
10485 float kv_val = v_val * k_val;
10486 float prev_state_val = state_prev[h_2d_i_j_offset];
10487 float temp_val = prev_state_val * g_val + kv_val;
10488 dst_data[t_h_j_offset] += temp_val * q_val;
10489 state_cur[h_2d_i_j_offset] = temp_val;
10490 }
10491 }
10492 }
10493 }
10494 #endif
10495}
10496
10497
10498void ggml_compute_forward_gla(
10499 const ggml_compute_params * params,
10500 ggml_tensor * dst) {
10501
10502 const ggml_tensor * src0 = dst->src[0];
10503
10504 switch (src0->type) {
10505 case GGML_TYPE_F32:
10506 {
10507 ggml_compute_forward_gla_f32(params, dst);
10508 } break;
10509 default:
10510 {
10511 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10511, "fatal error");
10512 }
10513 }
10514}
10515
10516static void ggml_compute_forward_solve_tri_f32(const struct ggml_compute_params * params, struct ggml_tensor * dst) {
10517 const struct ggml_tensor * src0 = dst->src[0]; // A (lower triangular)
10518 const struct ggml_tensor * src1 = dst->src[1]; // B (RHS)
10519
10520 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);;
10521
10522 GGML_ASSERT(src0->type == GGML_TYPE_F32)if (!(src0->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10522, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F32"
);
10523 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10523, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
10524 GGML_ASSERT(dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10524, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
10525
10526 GGML_ASSERT(ne00 == ne01)if (!(ne00 == ne01)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10526, "GGML_ASSERT(%s) failed", "ne00 == ne01"); // A must be square
10527 GGML_ASSERT(ne0 == ne10)if (!(ne0 == ne10)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10527, "GGML_ASSERT(%s) failed", "ne0 == ne10"); // solution cols == B cols
10528 GGML_ASSERT(ne1 == ne11)if (!(ne1 == ne11)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10528, "GGML_ASSERT(%s) failed", "ne1 == ne11"); // solution rows == B rows
10529
10530 GGML_ASSERT(ne02 == ne12 && ne12 == ne2)if (!(ne02 == ne12 && ne12 == ne2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10530, "GGML_ASSERT(%s) failed", "ne02 == ne12 && ne12 == ne2"
);
10531 GGML_ASSERT(ne03 == ne13 && ne13 == ne3)if (!(ne03 == ne13 && ne13 == ne3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10531, "GGML_ASSERT(%s) failed", "ne03 == ne13 && ne13 == ne3"
);
10532
10533 const int ith = params->ith;
10534 const int nth = params->nth;
10535
10536 const int64_t k = ne10; // number of RHS columns
10537 const int64_t n = ne11; // A is n×n
10538 const int64_t nr = ne02 * ne03 * k; // we're parallelizing on columns here, so seq x token x column will be the unit
10539
10540 // chunks per thread
10541 const int64_t dr = (nr + nth - 1)/nth;
10542
10543 // chunk range for this thread
10544 const int64_t ir0 = dr*ith;
10545 const int64_t ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
10546
10547 const float * A = (const float *) src0->data; // [n, n, B1, B2]
10548 const float * B = (const float *) src1->data; // [n, k, B1, B2]
10549 float * X = ( float *) dst->data; // [n, k, B1, B2]
10550
10551 for (int64_t ir = ir0; ir < ir1; ++ir) {
10552 const int64_t i03 = ir/(ne02*k);
10553 const int64_t i02 = (ir - i03*ne02*k)/k;
10554 const int64_t i01 = (ir - i03*ne02*k - i02*k);
10555
10556 const float * A_batch = A + i02 * nb02 / sizeof(float) + i03 * nb03 / sizeof(float);
10557 const float * B_batch = B + i02 * nb12 / sizeof(float) + i03 * nb13 / sizeof(float);
10558
10559 float * X_batch = X + i02 * nb2 / sizeof(float) + i03 * nb3 / sizeof(float);
10560
10561 for (int64_t i00 = 0; i00 < n; ++i00) {
10562 float sum = 0.0f;
10563 for (int64_t t = 0; t < i00; ++t) {
10564 sum += A_batch[i00 * n + t] * X_batch[t * k + i01];
10565 }
10566
10567 const float diag = A_batch[i00 * n + i00];
10568 assert(diag != 0.0f && "Zero diagonal in triangular matrix")(static_cast <bool> (diag != 0.0f && "Zero diagonal in triangular matrix"
) ? void (0) : __assert_fail ("diag != 0.0f && \"Zero diagonal in triangular matrix\""
, __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__
));
10569
10570 X_batch[i00 * k + i01] = (B_batch[i00 * k + i01] - sum) / diag;
10571 }
10572 }
10573}
10574
10575void ggml_compute_forward_solve_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst) {
10576 const ggml_tensor * src0 = dst->src[0];
10577 const ggml_tensor * src1 = dst->src[1];
10578
10579 if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {
10580 ggml_compute_forward_solve_tri_f32(params, dst);
10581 } else {
10582 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10582, "fatal error");
10583 }
10584}
10585
10586// ggml_compute_forward_gated_delta_net
10587static void ggml_compute_forward_gated_delta_net_one_chunk(
10588 const ggml_compute_params * params,
10589 ggml_tensor * dst,
10590 int64_t ir0,
10591 int64_t ir1) {
10592
10593 ggml_tensor * src_q = dst->src[0];
10594 ggml_tensor * src_k = dst->src[1];
10595 ggml_tensor * src_v = dst->src[2];
10596 ggml_tensor * src_g = dst->src[3];
10597 ggml_tensor * src_beta = dst->src[4];
10598 ggml_tensor * src_state = dst->src[5];
10599
10600 const int64_t S_v = src_v->ne[0];
10601 const int64_t H = src_v->ne[1];
10602 const int64_t n_tokens = src_v->ne[2];
10603 const int64_t n_seqs = src_v->ne[3];
10604
10605 GGML_ASSERT(ggml_is_contiguous_rows(src_q))if (!(ggml_is_contiguous_rows(src_q))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10605, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_rows(src_q)"
);
10606 GGML_ASSERT(ggml_is_contiguous_rows(src_k))if (!(ggml_is_contiguous_rows(src_k))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10606, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_rows(src_k)"
);
10607 GGML_ASSERT(ggml_is_contiguous_rows(src_v))if (!(ggml_is_contiguous_rows(src_v))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10607, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_rows(src_v)"
);
10608 GGML_ASSERT(ggml_is_contiguous(src_g))if (!(ggml_is_contiguous(src_g))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10608, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(src_g)"
);
10609 GGML_ASSERT(ggml_is_contiguous(src_beta))if (!(ggml_is_contiguous(src_beta))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10609, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(src_beta)"
);
10610 GGML_ASSERT(ggml_is_contiguous(src_state))if (!(ggml_is_contiguous(src_state))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10610, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(src_state)"
);
10611
10612 GGML_ASSERT(src_g->ne[0] == 1 || src_g->ne[0] == S_v)if (!(src_g->ne[0] == 1 || src_g->ne[0] == S_v)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10612, "GGML_ASSERT(%s) failed", "src_g->ne[0] == 1 || src_g->ne[0] == S_v"
);
10613 GGML_ASSERT(src_beta->ne[0] == 1)if (!(src_beta->ne[0] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10613, "GGML_ASSERT(%s) failed", "src_beta->ne[0] == 1");
10614
10615 GGML_TENSOR_LOCALS(int64_t, neq, src_q, ne)const int64_t neq0 = (src_q) ? (src_q)->ne[0] : 0; (void)(
neq0); const int64_t neq1 = (src_q) ? (src_q)->ne[1] : 0; (
void)(neq1); const int64_t neq2 = (src_q) ? (src_q)->ne[2]
: 0; (void)(neq2); const int64_t neq3 = (src_q) ? (src_q)->
ne[3] : 0; (void)(neq3);;
10616 GGML_TENSOR_LOCALS(size_t, nbq, src_q, nb)const size_t nbq0 = (src_q) ? (src_q)->nb[0] : 0; (void)(nbq0
); const size_t nbq1 = (src_q) ? (src_q)->nb[1] : 0; (void
)(nbq1); const size_t nbq2 = (src_q) ? (src_q)->nb[2] : 0;
(void)(nbq2); const size_t nbq3 = (src_q) ? (src_q)->nb[3
] : 0; (void)(nbq3);;
10617 GGML_TENSOR_LOCALS(int64_t, nek, src_k, ne)const int64_t nek0 = (src_k) ? (src_k)->ne[0] : 0; (void)(
nek0); const int64_t nek1 = (src_k) ? (src_k)->ne[1] : 0; (
void)(nek1); const int64_t nek2 = (src_k) ? (src_k)->ne[2]
: 0; (void)(nek2); const int64_t nek3 = (src_k) ? (src_k)->
ne[3] : 0; (void)(nek3);;
10618 GGML_TENSOR_LOCALS(size_t, nbk, src_k, nb)const size_t nbk0 = (src_k) ? (src_k)->nb[0] : 0; (void)(nbk0
); const size_t nbk1 = (src_k) ? (src_k)->nb[1] : 0; (void
)(nbk1); const size_t nbk2 = (src_k) ? (src_k)->nb[2] : 0;
(void)(nbk2); const size_t nbk3 = (src_k) ? (src_k)->nb[3
] : 0; (void)(nbk3);;
10619 GGML_TENSOR_LOCALS(int64_t, nev, src_v, ne)const int64_t nev0 = (src_v) ? (src_v)->ne[0] : 0; (void)(
nev0); const int64_t nev1 = (src_v) ? (src_v)->ne[1] : 0; (
void)(nev1); const int64_t nev2 = (src_v) ? (src_v)->ne[2]
: 0; (void)(nev2); const int64_t nev3 = (src_v) ? (src_v)->
ne[3] : 0; (void)(nev3);;
10620 GGML_TENSOR_LOCALS(size_t, nbv, src_v, nb)const size_t nbv0 = (src_v) ? (src_v)->nb[0] : 0; (void)(nbv0
); const size_t nbv1 = (src_v) ? (src_v)->nb[1] : 0; (void
)(nbv1); const size_t nbv2 = (src_v) ? (src_v)->nb[2] : 0;
(void)(nbv2); const size_t nbv3 = (src_v) ? (src_v)->nb[3
] : 0; (void)(nbv3);;
10621 GGML_TENSOR_LOCALS(int64_t, neg, src_g, ne)const int64_t neg0 = (src_g) ? (src_g)->ne[0] : 0; (void)(
neg0); const int64_t neg1 = (src_g) ? (src_g)->ne[1] : 0; (
void)(neg1); const int64_t neg2 = (src_g) ? (src_g)->ne[2]
: 0; (void)(neg2); const int64_t neg3 = (src_g) ? (src_g)->
ne[3] : 0; (void)(neg3);;
10622 GGML_TENSOR_LOCALS(size_t, nbg, src_g, nb)const size_t nbg0 = (src_g) ? (src_g)->nb[0] : 0; (void)(nbg0
); const size_t nbg1 = (src_g) ? (src_g)->nb[1] : 0; (void
)(nbg1); const size_t nbg2 = (src_g) ? (src_g)->nb[2] : 0;
(void)(nbg2); const size_t nbg3 = (src_g) ? (src_g)->nb[3
] : 0; (void)(nbg3);;
10623 GGML_TENSOR_LOCALS(size_t, nbb, src_beta, nb)const size_t nbb0 = (src_beta) ? (src_beta)->nb[0] : 0; (void
)(nbb0); const size_t nbb1 = (src_beta) ? (src_beta)->nb[1
] : 0; (void)(nbb1); const size_t nbb2 = (src_beta) ? (src_beta
)->nb[2] : 0; (void)(nbb2); const size_t nbb3 = (src_beta)
? (src_beta)->nb[3] : 0; (void)(nbb3);;
10624
10625 const bool kda = (neg0 == S_v);
10626
10627 // K (snapshot slot count) is an op param; state holds s0 only [S_v, S_v, H, n_seqs].
10628 const int64_t K = ggml_get_op_params_i32(dst, 0);
10629 GGML_ASSERT(K >= 1)if (!(K >= 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10629, "GGML_ASSERT(%s) failed", "K >= 1");
10630 // per-seq stride in floats (seq s starts at state + s * seq_stride)
10631 const int64_t state_seq_stride = src_state->nb[3] / sizeof(float);
10632
10633 const int64_t per_thread = S_v + (K > 1 ? S_v * S_v : 0);
10634 const int ith = params->ith;
10635
10636 float * delta = (float *)params->wdata + ith * per_thread + CACHE_LINE_SIZE_F32;
10637 float * state_work = K > 1 ? (delta + S_v) : nullptr;
10638
10639 // output layout: [attn_scores | new_states]
10640 // attn_scores: S_v * H * n_tokens * n_seqs floats
10641 // new_states: S_v * S_v * H * n_seqs * K floats (K snapshot slots; last min(n_tokens, K))
10642 const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs;
10643 const int64_t state_size_per_snap = S_v * S_v * H * n_seqs;
10644 float * attn_out_base = (float *)dst->data;
10645 float * state_out_base = (float *)dst->data + attn_score_elems;
10646
10647 // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back.
10648 // When n_tokens < K only slots 0..n_tokens-1 are written; older slots are caller-owned.
10649
10650 const float * state_in_base = (const float *)src_state->data;
10651
10652 //const int64_t rq1 = nev1 / neq1;
10653 //const int64_t rk1 = nev1 / nek1;
10654 const int64_t rq3 = nev3 / neq3;
10655 const int64_t rk3 = nev3 / nek3;
10656
10657 const float scale = 1.0f / sqrtf((float) S_v);
10658
10659 for (int64_t ir = ir0; ir < ir1; ++ir) {
10660 const int64_t iv1 = ir % H; // head_index
10661 const int64_t iv3 = ir / H; // sequence
10662
10663 const int64_t iq1 = iv1 % neq1;
10664 const int64_t ik1 = iv1 % nek1;
10665
10666 const int64_t iq3 = iv3 / rq3;
10667 const int64_t ik3 = iv3 / rk3;
10668
10669 // For K=1, write directly to the single output slot to avoid an extra memcpy at the end.
10670 // For K>1, work in scratch and copy out per-token when the slot is in range.
10671 float * s_out = (K > 1)
10672 ? state_work
10673 : state_out_base + (iv3 * H + iv1) * S_v * S_v;
10674
10675 // copy input state into the working buffer and operate in-place
10676 // state layout [S_v, S_v, H, n_seqs]: seq iv3 starts at iv3 * state_seq_stride.
10677 const float * s_in = state_in_base + iv3 * state_seq_stride + iv1 * S_v * S_v;
10678 memcpy(s_out, s_in, S_v * S_v * sizeof(float));
10679
10680 // attn output pointer for first token of this (head, seq)
10681 float * attn_data = attn_out_base + (iv3 * n_tokens * H + iv1) * S_v;
10682
10683 for (int64_t t = 0; t < n_tokens; t++) {
10684 const float * q_d = (const float *)((const char *)src_q->data + iq3 * nbq3 + t * nbq2 + iq1 * nbq1);
10685 const float * k_d = (const float *)((const char *)src_k->data + ik3 * nbk3 + t * nbk2 + ik1 * nbk1);
10686 const float * v_d = (const float *)((const char *)src_v->data + iv3 * nbv3 + t * nbv2 + iv1 * nbv1);
10687
10688 const float beta_val = *(const float *)((const char *)src_beta->data + iv3 * nbb3 + t * nbb2 + iv1 * nbb1);
10689 const float * g_d = (const float *)((const char *)src_g->data + iv3 * nbg3 + t * nbg2 + iv1 * nbg1);
10690
10691 // state is stored transposed: s_out[j*S_v + i] = S[i][j]
10692 // so row j of s_out = column j of S (contiguous access)
10693
10694 if (kda) {
10695 // precompute exp(g) into delta scratch (reused below)
10696 for (int64_t i = 0; i < S_v; ++i) {
10697 delta[i] = expf(g_d[i]);
10698 }
10699 // S[i][:] *= exp(g[i]) => for each row j of M: M[j][i] *= exp(g[i])
10700 for (int64_t j = 0; j < S_v; ++j) {
10701 ggml_vec_mul_f32(S_v, &s_out[j * S_v], &s_out[j * S_v], delta);
10702 }
10703 } else {
10704 ggml_vec_scale_f32(S_v * S_v, s_out, expf(g_d[0]));
10705 }
10706
10707 // delta[j] = sum_i S[i][j] * k[i] = dot(row j of M, k)
10708 for (int64_t j = 0; j < S_v; ++j) {
10709 float sum = 0.0f;
10710 ggml_vec_dot_f32(S_v, &sum, 0, &s_out[j * S_v], 0, k_d, 0, 1);
10711 delta[j] = (v_d[j] - sum) * beta_val;
10712 }
10713
10714 // outer product: S[i][j] += k[i] * delta[j] => M[j][i] += delta[j] * k[i]
10715 for (int64_t j = 0; j < S_v; ++j) {
10716 ggml_vec_mad_f32(S_v, &s_out[j * S_v], k_d, delta[j]);
10717 }
10718
10719 // attn_out[j] = sum_i S[i][j] * q[i] = dot(row j of M, q)
10720 for (int64_t j = 0; j < S_v; ++j) {
10721 float sum = 0.0f;
10722 ggml_vec_dot_f32(S_v, &sum, 0, &s_out[j * S_v], 0, q_d, 0, 1);
10723 attn_data[j] = sum * scale;
10724 }
10725
10726 attn_data += S_v * H; // advance to next token
10727
10728 if (K > 1) {
10729 const int64_t target_slot = n_tokens - 1 - t;
10730 if (target_slot >= 0 && target_slot < K) {
10731 float * curr_state_o = state_out_base + target_slot * state_size_per_snap +
10732 (iv3 * H + iv1) * S_v * S_v;
10733 memcpy(curr_state_o, s_out, S_v * S_v * sizeof(float));
10734 }
10735 }
10736 }
10737 }
10738}
10739
10740
10741static void ggml_compute_forward_gated_delta_net_f32(
10742 const ggml_compute_params * params,
10743 ggml_tensor * dst) {
10744
10745 ggml_tensor * V = dst->src[2];
10746 int64_t nr = V->ne[1] * V->ne[3];
10747
10748 // disable for NUMA
10749 const bool disable_chunking = ggml_is_numa();
10750
10751 int nth = params->nth;
10752 int ith = params->ith;
10753
10754 // 4x chunks per thread
10755 int nth_scaled = nth * 4;
10756 int64_t chunk_size = (nr + nth_scaled - 1) / nth_scaled;
10757 int64_t nchunk = (nr + chunk_size - 1) / chunk_size;
10758
10759 if (nth == 1 || nchunk < nth || disable_chunking) {
10760 nchunk = nth;
10761 }
10762
10763 if (ith == 0) {
10764 ggml_threadpool_chunk_set(params->threadpool, nth);
10765 }
10766
10767 ggml_barrier(params->threadpool);
10768
10769 const int64_t dr = (nr + nchunk - 1) / nchunk;
10770
10771 int current_chunk = ith;
10772
10773 while (current_chunk < nchunk) {
10774 const int64_t ir0 = dr * current_chunk;
10775 const int64_t ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
10776
10777 ggml_compute_forward_gated_delta_net_one_chunk(params, dst, ir0, ir1);
10778 current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);
10779 }
10780}
10781
10782void ggml_compute_forward_gated_delta_net(
10783 const ggml_compute_params * params,
10784 ggml_tensor * dst) {
10785 const ggml_tensor * src0 = dst->src[0];
10786
10787 switch (src0->type) {
10788 case GGML_TYPE_F32:
10789 {
10790 ggml_compute_forward_gated_delta_net_f32(params, dst);
10791 } break;
10792 default:
10793 {
10794 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10794, "fatal error");
10795 }
10796 }
10797}
10798
10799// ggml_compute_forward_rwkv_wkv7
10800
10801static void ggml_compute_forward_rwkv_wkv7_f32(
10802 const ggml_compute_params * params,
10803 ggml_tensor * dst) {
10804 const int64_t T = dst->src[1]->ne[2];
10805 const int64_t C = dst->ne[0];
10806 const int64_t HEADS = dst->src[1]->ne[1];
10807 const int64_t n_seqs = dst->src[6]->ne[1];
10808 const int64_t head_size = C / HEADS;
10809
10810 float * dst_data = (float *) dst->data;
10811 float * state = ((float *) dst->data) + C * T;
10812
10813 const int ith = params->ith;
10814 const int nth = params->nth;
10815
10816 const int h_start = (HEADS * (ith )) / nth;
10817 const int h_end = ((HEADS * (ith + 1)) / nth < HEADS) ?
10818 (HEADS * (ith + 1)) / nth : HEADS;
10819
10820 float * r = (float *) dst->src[0]->data;
10821 float * w = (float *) dst->src[1]->data;
10822 float * k = (float *) dst->src[2]->data;
10823 float * v = (float *) dst->src[3]->data;
10824 float * a = (float *) dst->src[4]->data;
10825 float * b = (float *) dst->src[5]->data;
10826
10827 int64_t t_stride = HEADS * head_size; // Same to C
10828
10829 int64_t h_stride = C / HEADS;
10830 GGML_ASSERT(C % HEADS == 0)if (!(C % HEADS == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 10830, "GGML_ASSERT(%s) failed", "C % HEADS == 0"); // C must be divisible by HEADS
10831 int64_t h_stride_2d = head_size * head_size;
10832
10833 #if defined(GGML_SIMD)
10834 #if defined(__ARM_FEATURE_SVE) || defined(__riscv_v_intrinsic)
10835 // scalar Route to scalar implementation //TODO: Write SVE code and RVV code
10836 for (int64_t t = 0; t < T; t++) {
10837 int64_t t_offset = t * t_stride;
10838 int64_t state_offset = head_size * C * (t / (T / n_seqs));
10839 float * state_cur = state + state_offset;
10840 float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset;
10841
10842 for (int64_t h = h_start; h < h_end; h++) {
10843 int64_t h_offset = h * h_stride;
10844 int64_t t_h_offset = t_offset + h_offset;
10845 int64_t h_2d_offset = h * h_stride_2d;
10846
10847 for (int64_t i = 0; i < head_size; i++) {
10848 int64_t t_h_i_offset = t_h_offset + i;
10849 int64_t h_2d_i_offset = h_2d_offset + i * h_stride;
10850
10851 float v_val = v[t_h_i_offset];
10852
10853 float sa = 0, result = 0;
10854 for (int64_t j = 0; j < head_size; j++) {
10855 sa += a[t_h_offset + j] * state_prev[h_2d_i_offset + j];
10856 }
10857
10858 for (int64_t j = 0; j < head_size; j++) {
10859 int64_t t_h_j_offset = t_h_offset + j;
10860 int64_t h_2d_i_j_offset = h_2d_i_offset + j;
10861
10862 float r_val = r[t_h_j_offset];
10863 float w_val = w[t_h_j_offset];
10864 float k_val = k[t_h_j_offset];
10865 float b_val = b[t_h_j_offset];
10866 float kv_val = v_val * k_val;
10867 float prev_state_val = state_prev[h_2d_i_j_offset];
10868 state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val;
10869 result += state_cur[h_2d_i_j_offset] * r_val;
10870 }
10871 dst_data[t_h_i_offset] = result;
10872 }
10873 }
10874 }
10875 #else
10876 for (int64_t t = 0; t < T; t++) {
10877 int64_t t_offset = t * t_stride;
10878 int64_t state_offset = head_size * C * (t / (T / n_seqs));
10879 float * state_cur = state + state_offset;
10880 float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset;
10881
10882 for (int64_t h = h_start; h < h_end; h++) {
10883 int64_t h_offset = h * h_stride;
10884 int64_t t_h_offset = t_offset + h_offset;
10885 int64_t h_2d_offset = h * h_stride_2d;
10886
10887 for (int64_t ii = 0; ii < head_size; ii++) {
10888 int64_t t_h_i_offset = t_h_offset + ii;
10889 int64_t h_2d_i_offset = h_2d_offset + ii * h_stride;
10890
10891 GGML_F32_VEC__m256 v_vec = GGML_F32_VEC_SET1(v[t_h_i_offset])_mm256_set1_ps(v[t_h_i_offset]);
10892
10893 float sa = 0;
10894 {
10895 GGML_F32_VEC__m256 sum[GGML_F32_ARR(32/8)] = { GGML_F32_VEC_ZERO_mm256_setzero_ps() };
10896 GGML_F32_VEC__m256 ax[GGML_F32_ARR(32/8)];
10897 GGML_F32_VEC__m256 ay[GGML_F32_ARR(32/8)];
10898 for (int64_t j = 0; j < head_size; j += GGML_F32_STEP32) {
10899 for (int64_t kk = 0; kk < GGML_F32_ARR(32/8); kk++) {
10900 ax[kk] = GGML_F32_VEC_LOAD_mm256_loadu_ps(&a[t_h_offset + j + kk * GGML_F32_EPR8]);
10901 ay[kk] = GGML_F32_VEC_LOAD_mm256_loadu_ps(&state_prev[h_2d_i_offset + j + kk * GGML_F32_EPR8]);
10902 sum[kk] = GGML_F32_VEC_FMA(sum[kk], ax[kk], ay[kk])_mm256_fmadd_ps(ax[kk], ay[kk], sum[kk]);
10903 }
10904 }
10905 GGML_F32_VEC_REDUCE(sa, sum)do { int offset = (32/8) >> 1; for (int i = 0; i < offset
; ++i) { sum[i] = _mm256_add_ps(sum[i], sum[offset+i]); } offset
>>= 1; for (int i = 0; i < offset; ++i) { sum[i] = _mm256_add_ps
(sum[i], sum[offset+i]); } offset >>= 1; for (int i = 0
; i < offset; ++i) { sum[i] = _mm256_add_ps(sum[i], sum[offset
+i]); } const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(sum
[0]), ((__m128)__builtin_ia32_vextractf128_ps256((__v8sf)(__m256
)(sum[0]), (int)(1)))); const __m128 t1 = _mm_hadd_ps(t0, t0)
; sa = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); } while
(0);
10906 }
10907
10908 GGML_F32_VEC__m256 sa_vec = GGML_F32_VEC_SET1(sa)_mm256_set1_ps(sa);
10909
10910 int64_t j = 0;
10911 GGML_F32_VEC__m256 result_vec[GGML_F32_ARR(32/8)] = { GGML_F32_VEC_ZERO_mm256_setzero_ps() };
10912 for (; j < head_size; j += GGML_F32_STEP32) {
10913 for (int64_t kk = 0; kk < GGML_F32_ARR(32/8); kk++) {
10914 int64_t t_h_j_offset = t_h_offset + j + kk * GGML_F32_EPR8;
10915 int64_t h_2d_i_j_offset = h_2d_i_offset + j + kk * GGML_F32_EPR8;
10916
10917 GGML_F32_VEC__m256 r_vec = GGML_F32_VEC_LOAD_mm256_loadu_ps(&r[t_h_j_offset]);
10918 GGML_F32_VEC__m256 w_vec = GGML_F32_VEC_LOAD_mm256_loadu_ps(&w[t_h_j_offset]);
10919 GGML_F32_VEC__m256 k_vec = GGML_F32_VEC_LOAD_mm256_loadu_ps(&k[t_h_j_offset]);
10920 GGML_F32_VEC__m256 b_vec = GGML_F32_VEC_LOAD_mm256_loadu_ps(&b[t_h_j_offset]);
10921
10922 k_vec = GGML_F32_VEC_MUL_mm256_mul_ps(v_vec, k_vec);
10923
10924 GGML_F32_VEC__m256 state_vec = GGML_F32_VEC_LOAD_mm256_loadu_ps(&state_prev[h_2d_i_j_offset]);
10925 // kv + s * decay + sa * b
10926 state_vec = GGML_F32_VEC_FMA(k_vec, state_vec, w_vec)_mm256_fmadd_ps(state_vec, w_vec, k_vec);
10927 state_vec = GGML_F32_VEC_FMA(state_vec, sa_vec, b_vec)_mm256_fmadd_ps(sa_vec, b_vec, state_vec);
10928 GGML_F32_VEC_STORE_mm256_storeu_ps(&state_cur[h_2d_i_j_offset], state_vec);
10929
10930 result_vec[kk] = GGML_F32_VEC_FMA(result_vec[kk], state_vec, r_vec)_mm256_fmadd_ps(state_vec, r_vec, result_vec[kk]);
10931 }
10932 }
10933 GGML_F32_VEC_REDUCE(dst_data[t_h_i_offset], result_vec)do { int offset = (32/8) >> 1; for (int i = 0; i < offset
; ++i) { result_vec[i] = _mm256_add_ps(result_vec[i], result_vec
[offset+i]); } offset >>= 1; for (int i = 0; i < offset
; ++i) { result_vec[i] = _mm256_add_ps(result_vec[i], result_vec
[offset+i]); } offset >>= 1; for (int i = 0; i < offset
; ++i) { result_vec[i] = _mm256_add_ps(result_vec[i], result_vec
[offset+i]); } const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128
(result_vec[0]), ((__m128)__builtin_ia32_vextractf128_ps256((
__v8sf)(__m256)(result_vec[0]), (int)(1)))); const __m128 t1 =
_mm_hadd_ps(t0, t0); dst_data[t_h_i_offset] = (ggml_float) _mm_cvtss_f32
(_mm_hadd_ps(t1, t1)); } while (0);
10934
10935 // There shouldn't be left-overs though.
10936 for (; j < head_size; j++) {
10937 int64_t t_h_j_offset = t_h_offset + j;
10938 int64_t h_2d_i_j_offset = h_2d_i_offset + j;
10939
10940 float r_val = r[t_h_j_offset];
10941 float w_val = w[t_h_j_offset];
10942 float k_val = k[t_h_j_offset];
10943 float b_val = b[t_h_j_offset];
10944 float kv_val = v[t_h_i_offset] * k_val;
10945
10946 float prev_state_val = state_prev[h_2d_i_j_offset];
10947 state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val;
10948 dst_data[t_h_i_offset] += state_cur[h_2d_i_j_offset] * r_val;
10949 }
10950 }
10951 }
10952 }
10953 #endif
10954 #else
10955 for (int64_t t = 0; t < T; t++) {
10956 int64_t t_offset = t * t_stride;
10957 int64_t state_offset = head_size * C * (t / (T / n_seqs));
10958 float * state_cur = state + state_offset;
10959 float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset;
10960
10961 for (int64_t h = h_start; h < h_end; h++) {
10962 int64_t h_offset = h * h_stride;
10963 int64_t t_h_offset = t_offset + h_offset;
10964 int64_t h_2d_offset = h * h_stride_2d;
10965
10966 for (int64_t i = 0; i < head_size; i++) {
10967 int64_t t_h_i_offset = t_h_offset + i;
10968 int64_t h_2d_i_offset = h_2d_offset + i * h_stride;
10969
10970 float v_val = v[t_h_i_offset];
10971
10972 float sa = 0, result = 0;
10973 for (int64_t j = 0; j < head_size; j++) {
10974 sa += a[t_h_offset + j] * state_prev[h_2d_i_offset + j];
10975 }
10976
10977 for (int64_t j = 0; j < head_size; j++) {
10978 int64_t t_h_j_offset = t_h_offset + j;
10979 int64_t h_2d_i_j_offset = h_2d_i_offset + j;
10980
10981 float r_val = r[t_h_j_offset];
10982 float w_val = w[t_h_j_offset];
10983 float k_val = k[t_h_j_offset];
10984 float b_val = b[t_h_j_offset];
10985 float kv_val = v_val * k_val;
10986 float prev_state_val = state_prev[h_2d_i_j_offset];
10987 state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val;
10988 result += state_cur[h_2d_i_j_offset] * r_val;
10989 }
10990 dst_data[t_h_i_offset] = result;
10991 }
10992 }
10993 }
10994 #endif
10995}
10996
10997
10998void ggml_compute_forward_rwkv_wkv7(
10999 const ggml_compute_params * params,
11000 ggml_tensor * dst) {
11001
11002 const ggml_tensor * src0 = dst->src[0];
11003
11004 switch (src0->type) {
11005 case GGML_TYPE_F32:
11006 {
11007 ggml_compute_forward_rwkv_wkv7_f32(params, dst);
11008 } break;
11009 default:
11010 {
11011 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11011, "fatal error");
11012 }
11013 }
11014}
11015
11016// ggml_compute_forward_map_custom1
11017
11018void ggml_compute_forward_map_custom1(
11019 const ggml_compute_params * params,
11020 ggml_tensor * dst) {
11021
11022 const ggml_tensor * a = dst->src[0];
11023
11024 struct ggml_map_custom1_op_params p;
11025 memcpy(&p, dst->op_params, sizeof(p));
11026
11027 p.fun(dst, a, params->ith, params->nth, p.userdata);
11028}
11029
11030// ggml_compute_forward_map_custom2
11031
11032void ggml_compute_forward_map_custom2(
11033 const ggml_compute_params * params,
11034 ggml_tensor * dst) {
11035
11036 const ggml_tensor * a = dst->src[0];
11037 const ggml_tensor * b = dst->src[1];
11038
11039 struct ggml_map_custom2_op_params p;
11040 memcpy(&p, dst->op_params, sizeof(p));
11041
11042 p.fun(dst, a, b, params->ith, params->nth, p.userdata);
11043}
11044
11045// ggml_compute_forward_map_custom3
11046
11047void ggml_compute_forward_map_custom3(
11048 const ggml_compute_params * params,
11049 ggml_tensor * dst) {
11050
11051 const ggml_tensor * a = dst->src[0];
11052 const ggml_tensor * b = dst->src[1];
11053 const ggml_tensor * c = dst->src[2];
11054
11055 struct ggml_map_custom3_op_params p;
11056 memcpy(&p, dst->op_params, sizeof(p));
11057
11058 p.fun(dst, a, b, c, params->ith, params->nth, p.userdata);
11059}
11060
11061// ggml_compute_forward_custom
11062
11063void ggml_compute_forward_custom(
11064 const struct ggml_compute_params * params,
11065 struct ggml_tensor * dst) {
11066
11067 struct ggml_custom_op_params p;
11068 memcpy(&p, dst->op_params, sizeof(p));
11069
11070 p.fun(dst, params->ith, params->nth, p.userdata);
11071}
11072
11073// ggml_compute_forward_cross_entropy_loss
11074
11075static void ggml_compute_forward_cross_entropy_loss_f32(
11076 const ggml_compute_params * params,
11077 ggml_tensor * dst) {
11078
11079 const ggml_tensor * src0 = dst->src[0];
11080 const ggml_tensor * src1 = dst->src[1];
11081
11082 GGML_ASSERT(src0->type == GGML_TYPE_F32)if (!(src0->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11082, "GGML_ASSERT(%s) failed", "src0->type == GGML_TYPE_F32"
);
11083 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11083, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
11084 GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type))if (!(src0->nb[0] == ggml_type_size(src0->type))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11084, "GGML_ASSERT(%s) failed", "src0->nb[0] == ggml_type_size(src0->type)"
);
11085 GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type))if (!(src1->nb[0] == ggml_type_size(src1->type))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11085, "GGML_ASSERT(%s) failed", "src1->nb[0] == ggml_type_size(src1->type)"
);
11086 GGML_ASSERT(ggml_are_same_shape(src0, src1))if (!(ggml_are_same_shape(src0, src1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11086, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, src1)"
);
11087 GGML_ASSERT(ggml_is_scalar(dst))if (!(ggml_is_scalar(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11087, "GGML_ASSERT(%s) failed", "ggml_is_scalar(dst)");
11088 GGML_ASSERT(dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11088, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
11089
11090 // TODO: handle transposed/permuted matrices
11091 const int64_t nc = src0->ne[0];
11092 const int64_t nr = ggml_nrows(src0);
11093
11094 const int ith = params->ith;
11095 const int nth = params->nth;
11096
11097 float * sums = (float *) params->wdata;
11098 float * st = ((float *) params->wdata) + nth + ith*nc;
11099 float sum_thread = 0.0f;
11100
11101 GGML_ASSERT(params->wsize >= sizeof(float) * (nth + nth * nc))if (!(params->wsize >= sizeof(float) * (nth + nth * nc)
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11101, "GGML_ASSERT(%s) failed", "params->wsize >= sizeof(float) * (nth + nth * nc)"
);
11102
11103 // rows per thread
11104 const int64_t dr = (nr + nth - 1)/nth;
11105
11106 // row range for this thread
11107 const int64_t ir0 = dr*ith;
11108 const int64_t ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
11109
11110 for (int64_t i1 = ir0; i1 < ir1; ++i1) {
11111 const float * s0 = (const float *)((const char *) src0->data + i1*src0->nb[1]);
11112 const float * s1 = (const float *)((const char *) src1->data + i1*src1->nb[1]);
11113
11114#ifndef NDEBUG
11115 for (int64_t i = 0; i < nc; ++i) {
11116 //printf("p[%d] = %f\n", i, p[i]);
11117 assert(!isnan(s0[i]))(static_cast <bool> (!isnan(s0[i])) ? void (0) : __assert_fail
("!isnan(s0[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
11118 assert(!isnan(s1[i]))(static_cast <bool> (!isnan(s1[i])) ? void (0) : __assert_fail
("!isnan(s1[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
11119 }
11120#endif // NDEBUG
11121
11122 float max = -INFINITY(__builtin_inff());
11123 ggml_vec_max_f32(nc, &max, s0);
11124 const ggml_float sum_softmax = ggml_vec_log_soft_max_f32(nc, st, s0, max);
11125 assert(sum_softmax >= 0.0)(static_cast <bool> (sum_softmax >= 0.0) ? void (0) :
__assert_fail ("sum_softmax >= 0.0", __builtin_FILE (), __builtin_LINE
(), __extension__ __PRETTY_FUNCTION__));
11126
11127 ggml_vec_add1_f32(nc, st, st, -sum_softmax);
11128 ggml_vec_mul_f32(nc, st, st, s1);
11129
11130 float sum_st = 0.0f;
11131 ggml_vec_sum_f32(nc, &sum_st, st);
11132 sum_thread += sum_st;
11133
11134#ifndef NDEBUG
11135 for (int64_t i = 0; i < nc; ++i) {
11136 assert(!isnan(st[i]))(static_cast <bool> (!isnan(st[i])) ? void (0) : __assert_fail
("!isnan(st[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
11137 assert(!isinf(st[i]))(static_cast <bool> (!isinf(st[i])) ? void (0) : __assert_fail
("!isinf(st[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
11138 }
11139#endif // NDEBUG
11140 }
11141 sums[ith] = sum_thread;
11142 ggml_barrier(params->threadpool);
11143
11144 if (ith == 0) {
11145 float * dp = (float *) dst->data;
11146 ggml_vec_sum_f32(nth, dp, sums);
11147 dp[0] *= -1.0f / (float) nr;
11148 }
11149}
11150
11151void ggml_compute_forward_cross_entropy_loss(
11152 const ggml_compute_params * params,
11153 ggml_tensor * dst) {
11154
11155 const ggml_tensor * src0 = dst->src[0];
11156
11157 switch (src0->type) {
11158 case GGML_TYPE_F32:
11159 {
11160 ggml_compute_forward_cross_entropy_loss_f32(params, dst);
11161 } break;
11162 default:
11163 {
11164 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11164, "fatal error");
11165 }
11166 }
11167}
11168
11169// ggml_compute_forward_cross_entropy_loss_back
11170
11171static void ggml_compute_forward_cross_entropy_loss_back_f32(
11172 const ggml_compute_params * params,
11173 ggml_tensor * dst) {
11174
11175 const ggml_tensor * grad = dst->src[0]; // gradient of forward pass output
11176 const ggml_tensor * src0f = dst->src[1]; // src0 of forward pass
11177 const ggml_tensor * src1f = dst->src[2]; // src1 of forward pass
11178
11179 GGML_ASSERT(ggml_is_contiguous(dst))if (!(ggml_is_contiguous(dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11179, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dst)");
11180 GGML_ASSERT(ggml_is_contiguous(src0f))if (!(ggml_is_contiguous(src0f))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11180, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(src0f)"
);
11181 GGML_ASSERT(ggml_is_contiguous(src1f))if (!(ggml_is_contiguous(src1f))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11181, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(src1f)"
);
11182 GGML_ASSERT(ggml_is_contiguous(grad))if (!(ggml_is_contiguous(grad))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11182, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(grad)"
);
11183 GGML_ASSERT(ggml_are_same_shape(src0f, src1f) && ggml_are_same_shape(src0f, dst))if (!(ggml_are_same_shape(src0f, src1f) && ggml_are_same_shape
(src0f, dst))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11183, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0f, src1f) && ggml_are_same_shape(src0f, dst)"
);
11184
11185 const int64_t ith = params->ith;
11186 const int64_t nth = params->nth;
11187
11188 // TODO: handle transposed/permuted matrices
11189 const int64_t nc = src0f->ne[0];
11190 const int64_t nr = ggml_nrows(src0f);
11191
11192 // rows per thread
11193 const int64_t dr = (nr + nth - 1)/nth;
11194
11195 // row range for this thread
11196 const int64_t ir0 = dr*ith;
11197 const int64_t ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
11198
11199 const float d_by_nr = ((const float *) grad->data)[0] / (float) nr;
11200
11201 for (int64_t i1 = ir0; i1 < ir1; i1++) {
11202 float * ds0 = (float *)((char *) dst->data + i1*dst->nb[1]);
11203 const float * s0 = (const float *)((const char *) src0f->data + i1*src0f->nb[1]);
11204 const float * s1 = (const float *)((const char *) src1f->data + i1*src1f->nb[1]);
11205
11206#ifndef NDEBUG
11207 for (int64_t i = 0; i < nc; ++i) {
11208 //printf("p[%d] = %f\n", i, p[i]);
11209 assert(!isnan(s0[i]))(static_cast <bool> (!isnan(s0[i])) ? void (0) : __assert_fail
("!isnan(s0[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
11210 assert(!isnan(s1[i]))(static_cast <bool> (!isnan(s1[i])) ? void (0) : __assert_fail
("!isnan(s1[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
11211 }
11212#endif // NDEBUG
11213
11214 // soft_max
11215 float max = -INFINITY(__builtin_inff());
11216 ggml_vec_max_f32(nc, &max, s0);
11217 const ggml_float sum = ggml_vec_soft_max_f32(nc, ds0, s0, max);
11218 assert(sum > 0.0)(static_cast <bool> (sum > 0.0) ? void (0) : __assert_fail
("sum > 0.0", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
11219 ggml_vec_scale_f32(nc, ds0, 1.0/sum);
11220
11221 // grad(src0f) = (softmax(src0f) - src1f) * grad(cross_entropy_loss(src0f, src1f)) / nr
11222 ggml_vec_sub_f32(nc, ds0, ds0, s1);
11223 ggml_vec_scale_f32(nc, ds0, d_by_nr);
11224
11225#ifndef NDEBUG
11226 for (int64_t i = 0; i < nc; ++i) {
11227 assert(!isnan(ds0[i]))(static_cast <bool> (!isnan(ds0[i])) ? void (0) : __assert_fail
("!isnan(ds0[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
11228 assert(!isinf(ds0[i]))(static_cast <bool> (!isinf(ds0[i])) ? void (0) : __assert_fail
("!isinf(ds0[i])", __builtin_FILE (), __builtin_LINE (), __extension__
__PRETTY_FUNCTION__));
11229 }
11230#endif // NDEBUG
11231 }
11232}
11233
11234void ggml_compute_forward_cross_entropy_loss_back(
11235 const ggml_compute_params * params,
11236 ggml_tensor * dst) {
11237
11238 const ggml_tensor * src0 = dst->src[0];
11239
11240 switch (src0->type) {
11241 case GGML_TYPE_F32:
11242 {
11243 ggml_compute_forward_cross_entropy_loss_back_f32(params, dst);
11244 } break;
11245 default:
11246 {
11247 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11247, "fatal error");
11248 }
11249 }
11250}
11251
11252static void ggml_compute_forward_opt_step_adamw_f32(
11253 const ggml_compute_params * params,
11254 ggml_tensor * dst) {
11255
11256 const ggml_tensor * src0 = dst->src[0];
11257 const ggml_tensor * src0_grad = dst->src[1];
11258 const ggml_tensor * src0_grad_m = dst->src[2];
11259 const ggml_tensor * src0_grad_v = dst->src[3];
11260 const ggml_tensor * adamw_params = dst->src[4];
11261
11262 GGML_ASSERT(ggml_are_same_shape(src0, src0_grad))if (!(ggml_are_same_shape(src0, src0_grad))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11262, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, src0_grad)"
);
11263 GGML_ASSERT(ggml_are_same_shape(src0, src0_grad_m))if (!(ggml_are_same_shape(src0, src0_grad_m))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11263, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, src0_grad_m)"
);
11264 GGML_ASSERT(ggml_are_same_shape(src0, src0_grad_v))if (!(ggml_are_same_shape(src0, src0_grad_v))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11264, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, src0_grad_v)"
);
11265 GGML_ASSERT(ggml_nelements(adamw_params) == 7)if (!(ggml_nelements(adamw_params) == 7)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11265, "GGML_ASSERT(%s) failed", "ggml_nelements(adamw_params) == 7"
);
11266
11267 const int ith = params->ith;
11268 const int nth = params->nth;
11269
11270 const int nr = ggml_nrows(src0);
11271
11272 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
11273 GGML_ASSERT(nb00 == sizeof(float))if (!(nb00 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11273, "GGML_ASSERT(%s) failed", "nb00 == sizeof(float)");
11274
11275 // rows per thread
11276 const int dr = (nr + nth - 1)/nth;
11277
11278 // row range for this thread
11279 const int ir0 = dr*ith;
11280 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
11281
11282 const float * adamw_params_ptr = ggml_get_data_f32(adamw_params);
11283
11284 const float alpha = adamw_params_ptr[0];
11285 const float beta1 = adamw_params_ptr[1];
11286 const float beta2 = adamw_params_ptr[2];
11287 const float eps = adamw_params_ptr[3];
11288 const float wd = adamw_params_ptr[4];
11289 const float beta1h = adamw_params_ptr[5];
11290 const float beta2h = adamw_params_ptr[6];
11291 const float keep = 1.f - alpha * wd;
11292 for (int ir = ir0; ir < ir1; ++ir) {
11293 const int64_t i03 = ir/(ne02*ne01);
11294 const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
11295 const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
11296
11297 const size_t offset = i03*nb03 + i02*nb02 + i01*nb01;
11298
11299 float * w = (float *) ((char *) src0->data + offset); // weight
11300 const float * g = (const float *) ((const char *) src0_grad->data + offset); // grad
11301 float * m = (float *) ((char *) src0_grad_m->data + offset);
11302 float * v = (float *) ((char *) src0_grad_v->data + offset);
11303
11304 for (int i00 = 0; i00 < ne00; ++i00) {
11305 m[i00] = m[i00]*beta1 + g[i00]*(1.0f - beta1);
11306 v[i00] = v[i00]*beta2 + g[i00]*g[i00]*(1.0f - beta2);
11307
11308 const float mh = m[i00]*beta1h;
11309 const float vh = sqrtf(v[i00]*beta2h) + eps;
11310
11311 // The weight decay is applied independently of the Adam momenta m and v.
11312 // This is NOT equivalent to l2 regularization that adds w[i00]*w[i00] to the loss.
11313 // See: https://arxiv.org/pdf/1711.05101v3.pdf
11314 w[i00] = w[i00] * keep - alpha * mh / vh;
11315 }
11316 }
11317}
11318
11319void ggml_compute_forward_opt_step_adamw(
11320 const ggml_compute_params * params,
11321 ggml_tensor * dst) {
11322
11323 const ggml_tensor * src0 = dst->src[0];
11324
11325 switch (src0->type) {
11326 case GGML_TYPE_F32:
11327 {
11328 ggml_compute_forward_opt_step_adamw_f32(params, dst);
11329 } break;
11330 default:
11331 {
11332 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11332, "fatal error");
11333 }
11334 }
11335}
11336
11337static void ggml_compute_forward_opt_step_sgd_f32(const ggml_compute_params * params, ggml_tensor * dst) {
11338 const ggml_tensor * src0 = dst->src[0];
11339 const ggml_tensor * src0_grad = dst->src[1];
11340 const ggml_tensor * sgd_params = dst->src[2];
11341
11342 GGML_ASSERT(ggml_are_same_shape(src0, src0_grad))if (!(ggml_are_same_shape(src0, src0_grad))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11342, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(src0, src0_grad)"
);
11343 GGML_ASSERT(ggml_nelements(sgd_params) == 2)if (!(ggml_nelements(sgd_params) == 2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11343, "GGML_ASSERT(%s) failed", "ggml_nelements(sgd_params) == 2"
);
11344
11345 const int ith = params->ith;
11346 const int nth = params->nth;
11347
11348 const int nr = ggml_nrows(src0);
11349
11350 GGML_TENSOR_UNARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne0 = (dst) ? (dst)->
ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->
ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->
ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->
ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->
nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->
nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->
nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->
nb[3] : 0; (void)(nb3);
11351 GGML_ASSERT(nb00 == sizeof(float))if (!(nb00 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11351, "GGML_ASSERT(%s) failed", "nb00 == sizeof(float)");
11352
11353 // rows per thread
11354 const int dr = (nr + nth - 1) / nth;
11355
11356 // row range for this thread
11357 const int ir0 = dr * ith;
11358 const int ir1 = MIN(ir0 + dr, nr)((ir0 + dr) < (nr) ? (ir0 + dr) : (nr));
11359
11360 // using adamw param subset we care about - alpha, wd - could have a separate struct
11361 const float * sgd_params_ptr = ggml_get_data_f32(sgd_params);
11362 const float alpha = sgd_params_ptr[0];
11363 const float keep = 1.f - alpha * sgd_params_ptr[1];
11364
11365 for (int ir = ir0; ir < ir1; ++ir) {
11366 const int64_t i03 = ir / (ne02 * ne01);
11367 const int64_t i02 = (ir - i03 * ne02 * ne01) / ne01;
11368 const int64_t i01 = (ir - i03 * ne02 * ne01 - i02 * ne01);
11369
11370 const size_t offset = i03 * nb03 + i02 * nb02 + i01 * nb01;
11371
11372 float * w = (float *) ((char *) src0->data + offset); // weight
11373 const float * g = (const float *) ((const char *) src0_grad->data + offset); // grad
11374
11375 for (int i00 = 0; i00 < ne00; ++i00) {
11376 w[i00] = w[i00] * keep - alpha * g[i00];
11377 }
11378 }
11379}
11380
11381void ggml_compute_forward_opt_step_sgd(const ggml_compute_params * params, ggml_tensor * dst) {
11382 const ggml_tensor * src0 = dst->src[0];
11383
11384 switch (src0->type) {
11385 case GGML_TYPE_F32:
11386 {
11387 ggml_compute_forward_opt_step_sgd_f32(params, dst);
11388 }
11389 break;
11390 default:
11391 {
11392 GGML_ABORT("fatal error - sgd is F32 only")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11392, "fatal error - sgd is F32 only");
11393 }
11394 }
11395}
11396
11397static void ggml_compute_forward_fwht_f32(const ggml_compute_params * params, ggml_tensor * dst) {
11398 const ggml_tensor * src0 = dst->src[0];
11399 const ggml_tensor * src1 = dst->src[1];
11400
11401 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11401, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
11402 GGML_ASSERT(dst->type == GGML_TYPE_F32)if (!(dst->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11402, "GGML_ASSERT(%s) failed", "dst->type == GGML_TYPE_F32"
);
11403
11404 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
11405
11406 const int ith = params->ith;
11407 const int nth = params->nth;
11408
11409 const int64_t n = ne10;
11410 GGML_ASSERT((n & (n - 1)) == 0)if (!((n & (n - 1)) == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11410, "GGML_ASSERT(%s) failed", "(n & (n - 1)) == 0"); // must be power of 2
11411
11412 const int64_t nr = ne11 * ne12 * ne13;
11413 const int64_t rows_per_thread = (nr + nth - 1) / nth;
11414 const int64_t start_row = ith * rows_per_thread;
11415 const int64_t end_row = MIN(start_row + rows_per_thread, nr)((start_row + rows_per_thread) < (nr) ? (start_row + rows_per_thread
) : (nr));
11416
11417 const float scale = 1.0f / sqrtf((float)n);
11418
11419#if defined(GGML_SIMD)
11420 const GGML_F32_VEC__m256 v_minus_one = GGML_F32_VEC_SET1(-1.0f)_mm256_set1_ps(-1.0f);
11421#endif
11422
11423 for (int64_t r = start_row; r < end_row; r++) {
11424 const int64_t i13 = r / (ne11 * ne12);
11425 const int64_t i12 = (r - i13 * ne11 * ne12) / ne11;
11426 const int64_t i11 = r - i13 * ne11 * ne12 - i12 * ne11;
11427
11428 const float * src_row = (const float *) ((const char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13);
11429 float * dst_row = (float *) ((char *) dst->data + i11 * nb1 + i12 * nb2 + i13 * nb3);
11430
11431 for (int64_t j = 0; j < n; j++) {
11432 dst_row[j] = src_row[j] * scale;
11433 }
11434
11435 // Scalar passes
11436#if defined(GGML_SIMD)
11437#if defined(__ARM_FEATURE_SVE)
11438 const int step = svcntw();
11439#else
11440 const int step = GGML_F32_EPR8;
11441#endif
11442#else
11443 const int step = n;
11444#endif
11445 for (int64_t len = 1; len < step && len < n; len <<= 1) {
11446 for (int64_t i = 0; i < n; i += 2 * len) {
11447 for (int64_t j = 0; j < len; j++) {
11448 float u = dst_row[i + j];
11449 float v = dst_row[i + len + j];
11450 dst_row[i + j] = u + v;
11451 dst_row[i + len + j] = u - v;
11452 }
11453 }
11454 }
11455
11456 // SIMD passes using GGML_F32_VEC_* macros for multi-architecture support
11457#if defined(GGML_SIMD)
11458 for (int64_t len = step; len < n; len <<= 1) {
11459 for (int64_t i = 0; i < n; i += 2 * len) {
11460 for (int64_t j = 0; j < len; j += step) {
11461 GGML_F32_VEC__m256 u = GGML_F32_VEC_LOAD_mm256_loadu_ps(dst_row + i + j);
11462 GGML_F32_VEC__m256 v = GGML_F32_VEC_LOAD_mm256_loadu_ps(dst_row + i + len + j);
11463
11464 GGML_F32_VEC_STORE_mm256_storeu_ps(dst_row + i + j, GGML_F32_VEC_ADD_mm256_add_ps(u, v));
11465 GGML_F32_VEC_STORE_mm256_storeu_ps(dst_row + i + len + j, GGML_F32_VEC_FMA(u, v, v_minus_one)_mm256_fmadd_ps(v, v_minus_one, u));
11466 }
11467 }
11468 }
11469#endif
11470 }
11471}
11472
11473void ggml_compute_forward_fwht(const ggml_compute_params * params, ggml_tensor * dst) {
11474 const ggml_tensor * src1 = dst->src[1];
11475
11476 switch (src1->type) {
11477 case GGML_TYPE_F32:
11478 {
11479 ggml_compute_forward_fwht_f32(params, dst);
11480 }
11481 break;
11482 default:
11483 {
11484 GGML_ABORT("fatal error - fwht is F32 only")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ops.cpp"
, 11484, "fatal error - fwht is F32 only");
11485 }
11486 }
11487}