Bug Summary

File:root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c
Warning:line 1514, column 17
The first element of the 2nd argument is undefined
Note:line 1514, column 17
Other elements might also be undefined

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O2 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ggml-cpu-c.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +avx -target-feature +avx2 -target-feature +bmi2 -target-feature +f16c -target-feature +fma -target-feature +sse4.2 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -resource-dir /usr/lib/llvm-23/lib/clang/23 -include /root/firefox-clang/config/gcc_hidden.h -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/system_wrappers -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG=1 -D _GNU_SOURCE=1 -D GGML_USE_CPU=1 -D GGML_VERSION="GGML_VERSION" -D GGML_COMMIT="GGML_COMMIT" -D GGML_SHARED=1 -D LLAMA_SHARED=1 -D GGML_BUILD=1 -D LLAMA_BUILD=1 -D GGML_BACKEND_SHARED=1 -D GGML_BACKEND_BUILD=1 -D MOZ_HAS_MOZGLUE -I /root/firefox-clang/third_party/llama.cpp -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -I /root/firefox-clang/third_party/llama.cpp/ggml -I /root/firefox-clang/third_party/llama.cpp/ggml/include -I /root/firefox-clang/third_party/llama.cpp/ggml/src -I /root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu -I /root/firefox-clang/third_party/llama.cpp/include -I /root/firefox-clang/third_party/llama.cpp/src -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/llvm-23/lib/clang/23/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=tautological-type-limit-compare -Wno-range-loop-analysis -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-unknown-warning-option -Wno-character-conversion -Wno-sign-compare -Wno-unused-function -Wno-unreachable-code -Wno-tautological-unsigned-enum-zero-compare -Wno-absolute-value -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -fdiagnostics-absolute-paths -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -mllvm -dwarf-linkage-names=Abstract -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-09-01-224014-2642839-1 -x c /root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c
1#define _CRT_SECURE_NO_DEPRECATE // Disables "unsafe" warnings on Windows
2#define _USE_MATH_DEFINES // For M_PI on MSVC
3
4#include "ggml-backend-impl.h"
5#include "ggml-backend.h"
6#include "traits.h"
7#include "ggml-cpu-impl.h"
8#include "ggml-impl.h"
9#include "quants.h"
10#include "ggml-threading.h"
11#include "unary-ops.h"
12#include "binary-ops.h"
13#include "vec.h"
14#include "ops.h"
15#include "ggml.h"
16#include "common.h"
17
18#if defined(_MSC_VER) || defined(__MINGW32__)
19#include <malloc.h> // using malloc.h with MSC/MINGW
20#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
21#include <alloca.h>
22#endif
23
24#include <assert.h>
25#include <errno(*__errno_location ()).h>
26#include <time.h>
27#include <math.h>
28#include <stdlib.h>
29#include <string.h>
30#include <stdint.h>
31#include <inttypes.h>
32#include <stdio.h>
33#include <float.h>
34#include <limits.h>
35#include <stdarg.h>
36#include <signal.h>
37#if defined(__gnu_linux__1)
38#include <syscall.h>
39#endif
40
41#ifdef GGML_USE_OPENMP
42#include <omp.h>
43#endif
44
45#if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_MATMUL_INT8)
46#undef GGML_USE_LLAMAFILE
47#endif
48
49#ifdef GGML_USE_LLAMAFILE
50#include "llamafile/sgemm.h"
51#endif
52
53#ifdef GGML_USE_CPU_RISCV64_SPACEMIT
54# include "spacemit/ime.h"
55#endif
56
57// Note: once we move threading into a separate C++ file
58// will use std::hardware_destructive_interference_size instead of hardcoding it here
59// and we'll use C++ attribute syntax.
60#define GGML_CACHE_LINE64 64
61
62#if defined(__clang__1) || defined(__GNUC__4)
63#define GGML_CACHE_ALIGN__attribute__((aligned(64))) __attribute__((aligned(GGML_CACHE_LINE64)))
64#endif
65
66#if defined(__has_feature)0
67#if __has_feature(thread_sanitizer)0
68#define GGML_TSAN_ENABLED 1
69#endif
70#else // __has_feature
71#if defined(__SANITIZE_THREAD__)
72#define GGML_TSAN_ENABLED 1
73#endif
74#endif // __has_feature
75
76#define UNUSEDGGML_UNUSED GGML_UNUSED
77#define SWAP(x, y, T)do { T SWAP = x; (x) = y; (y) = SWAP; } while (0) do { T SWAP = x; (x) = y; (y) = SWAP; } while (0)
78
79// precomputed f32 table for f16 (256 KB) (simd-mappings.h)
80float ggml_table_f32_f16[1 << 16];
81
82// precomputed f32 table for e8m0 half (1 KB) (simd-mappings.h)
83float ggml_table_f32_e8m0_half[1 << 8];
84
85#if defined(__ARM_ARCH)
86struct ggml_arm_arch_features_type {
87 int sve_cnt;
88} ggml_arm_arch_features = { 0 };
89#endif
90
91#if defined(__riscv)
92struct ggml_riscv_arch_features_type {
93 int rvv_vlen;
94} ggml_riscv_arch_features = { 0 };
95#endif
96
97#if defined(_WIN32)
98
99#define WIN32_LEAN_AND_MEAN
100#ifndef NOMINMAX
101 #define NOMINMAX
102#endif
103#include <windows.h>
104
105#if defined(_MSC_VER) && !defined(__clang__1)
106#define GGML_CACHE_ALIGN__attribute__((aligned(64))) __declspec(align(GGML_CACHE_LINE64))
107
108typedef volatile LONG atomic_int;
109typedef atomic_int atomic_bool;
110typedef atomic_int atomic_flag;
111
112#define ATOMIC_FLAG_INIT{ 0 } 0
113
114typedef enum {
115 memory_order_relaxed,
116 memory_order_consume,
117 memory_order_acquire,
118 memory_order_release,
119 memory_order_acq_rel,
120 memory_order_seq_cst
121} memory_order;
122
123static void atomic_store(atomic_int * ptr, LONG val)__c11_atomic_store(atomic_int * ptr, LONG val, 5) {
124 InterlockedExchange(ptr, val);
125}
126static void atomic_store_explicit__c11_atomic_store(atomic_int * ptr, LONG val, memory_order mo) {
127 // TODO: add support for explicit memory order
128 InterlockedExchange(ptr, val);
129}
130static LONG atomic_load(atomic_int * ptr)__c11_atomic_load(atomic_int * ptr, 5) {
131 return InterlockedCompareExchange(ptr, 0, 0);
132}
133static LONG atomic_load_explicit__c11_atomic_load(atomic_int * ptr, memory_order mo) {
134 // TODO: add support for explicit memory order
135 return InterlockedCompareExchange(ptr, 0, 0);
136}
137static LONG atomic_fetch_add(atomic_int * ptr, LONG inc)__c11_atomic_fetch_add(atomic_int * ptr, LONG inc, 5) {
138 return InterlockedExchangeAdd(ptr, inc);
139}
140static LONG atomic_fetch_add_explicit__c11_atomic_fetch_add(atomic_int * ptr, LONG inc, memory_order mo) {
141 // TODO: add support for explicit memory order
142 return InterlockedExchangeAdd(ptr, inc);
143}
144static atomic_bool atomic_flag_test_and_set(atomic_flag * ptr)__c11_atomic_exchange(&(atomic_flag * ptr)->_Value, 1,
5) {
145 return InterlockedExchange(ptr, 1);
146}
147static void atomic_flag_clear(atomic_flag * ptr)__c11_atomic_store(&(atomic_flag * ptr)->_Value, 0, 5) {
148 InterlockedExchange(ptr, 0);
149}
150static void atomic_thread_fence(memory_order mo)__c11_atomic_thread_fence(memory_order mo) {
151 MemoryBarrier();
152}
153#else // clang
154#include <stdatomic.h>
155#endif
156
157typedef HANDLE pthread_t;
158
159typedef DWORD thread_ret_t;
160static int pthread_create(pthread_t * out, void * unused, thread_ret_t(*func)(void *), void * arg) {
161 (void) unused;
162 HANDLE handle = CreateThread(NULL((void*)0), 0, (LPTHREAD_START_ROUTINE) func, arg, 0, NULL((void*)0));
163 if (handle == NULL((void*)0))
164 {
165 return EAGAIN11;
166 }
167
168 *out = handle;
169 return 0;
170}
171
172static int pthread_join(pthread_t thread, void * unused) {
173 (void) unused;
174 int ret = (int) WaitForSingleObject(thread, INFINITE);
175 CloseHandle(thread);
176 return ret;
177}
178
179static int sched_yield (void) {
180 Sleep (0);
181 return 0;
182}
183#else
184
185#include <pthread.h>
186#include <stdatomic.h>
187#include <sched.h>
188#if defined(__FreeBSD__)
189#include <pthread_np.h>
190#endif
191
192typedef void * thread_ret_t;
193
194#include <sys/types.h>
195#include <sys/stat.h>
196#include <unistd.h>
197
198#endif
199
200typedef pthread_t ggml_thread_t;
201
202#define GGML_THREADPOOL_N_THREADS_MASK(0xffffU) (0xffffU)
203#define GGML_THREADPOOL_N_THREADS_BITS(16) (16)
204
205#if defined(__APPLE__)
206#include <unistd.h>
207#include <mach/mach.h>
208#include <TargetConditionals.h>
209#endif
210
211static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
212 [GGML_TYPE_F32] = {
213 .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp32,
214 .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f32,
215 .vec_dot_type = GGML_TYPE_F32,
216 .nrows = 1,
217 },
218 [GGML_TYPE_F16] = {
219 .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp16,
220 .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16,
221 .vec_dot_type = GGML_TYPE_F16,
222 .nrows = 1,
223 },
224 [GGML_TYPE_Q1_0] = {
225 .from_float = quantize_row_q1_0,
226 .vec_dot = ggml_vec_dot_q1_0_q8_0,
227 .vec_dot_type = GGML_TYPE_Q8_0,
228 .nrows = 1,
229 },
230 [GGML_TYPE_Q4_0] = {
231 .from_float = quantize_row_q4_0,
232 .vec_dot = ggml_vec_dot_q4_0_q8_0,
233 .vec_dot_type = GGML_TYPE_Q8_0,
234#if defined (__ARM_FEATURE_MATMUL_INT8)
235 .nrows = 2,
236#else
237 .nrows = 1,
238#endif
239 },
240 [GGML_TYPE_Q4_1] = {
241 .from_float = quantize_row_q4_1,
242 .vec_dot = ggml_vec_dot_q4_1_q8_1,
243 .vec_dot_type = GGML_TYPE_Q8_1,
244#if defined (__ARM_FEATURE_MATMUL_INT8)
245 .nrows = 2,
246#else
247 .nrows = 1,
248#endif
249 },
250 [GGML_TYPE_Q5_0] = {
251 .from_float = quantize_row_q5_0,
252 .vec_dot = ggml_vec_dot_q5_0_q8_0,
253 .vec_dot_type = GGML_TYPE_Q8_0,
254 .nrows = 1,
255 },
256 [GGML_TYPE_Q5_1] = {
257 .from_float = quantize_row_q5_1,
258 .vec_dot = ggml_vec_dot_q5_1_q8_1,
259 .vec_dot_type = GGML_TYPE_Q8_1,
260 .nrows = 1,
261 },
262 [GGML_TYPE_Q8_0] = {
263 .from_float = quantize_row_q8_0,
264 .vec_dot = ggml_vec_dot_q8_0_q8_0,
265 .vec_dot_type = GGML_TYPE_Q8_0,
266#if defined (__ARM_FEATURE_MATMUL_INT8)
267 .nrows = 2,
268#else
269 .nrows = 1,
270#endif
271 },
272 [GGML_TYPE_Q8_1] = {
273 .from_float = quantize_row_q8_1,
274 .vec_dot_type = GGML_TYPE_Q8_1,
275 .nrows = 1,
276 },
277 [GGML_TYPE_MXFP4] = {
278 .from_float = quantize_row_mxfp4,
279 .vec_dot = ggml_vec_dot_mxfp4_q8_0,
280 .vec_dot_type = GGML_TYPE_Q8_0,
281 .nrows = 1,
282 },
283 [GGML_TYPE_NVFP4] = {
284 .from_float = quantize_row_nvfp4,
285 .vec_dot = ggml_vec_dot_nvfp4_q8_0,
286 .vec_dot_type = GGML_TYPE_Q8_0,
287 .nrows = 1,
288 },
289 [GGML_TYPE_Q2_K] = {
290 .from_float = quantize_row_q2_K,
291 .vec_dot = ggml_vec_dot_q2_K_q8_K,
292 .vec_dot_type = GGML_TYPE_Q8_K,
293 .nrows = 1,
294 },
295 [GGML_TYPE_Q3_K] = {
296 .from_float = quantize_row_q3_K,
297 .vec_dot = ggml_vec_dot_q3_K_q8_K,
298 .vec_dot_type = GGML_TYPE_Q8_K,
299 .nrows = 1,
300 },
301 [GGML_TYPE_Q4_K] = {
302 .from_float = quantize_row_q4_K,
303 .vec_dot = ggml_vec_dot_q4_K_q8_K,
304 .vec_dot_type = GGML_TYPE_Q8_K,
305#if defined (__ARM_FEATURE_MATMUL_INT8)
306 .nrows = 2,
307#else
308 .nrows = 1,
309#endif
310 },
311 [GGML_TYPE_Q5_K] = {
312 .from_float = quantize_row_q5_K,
313 .vec_dot = ggml_vec_dot_q5_K_q8_K,
314 .vec_dot_type = GGML_TYPE_Q8_K,
315 .nrows = 1,
316 },
317 [GGML_TYPE_Q6_K] = {
318 .from_float = quantize_row_q6_K,
319 .vec_dot = ggml_vec_dot_q6_K_q8_K,
320 .vec_dot_type = GGML_TYPE_Q8_K,
321#if defined (__ARM_FEATURE_MATMUL_INT8)
322 .nrows = 2,
323#else
324 .nrows = 1,
325#endif
326 },
327 [GGML_TYPE_IQ2_XXS] = {
328 .from_float = NULL((void*)0),
329 .vec_dot = ggml_vec_dot_iq2_xxs_q8_K,
330 .vec_dot_type = GGML_TYPE_Q8_K,
331 .nrows = 1,
332 },
333 [GGML_TYPE_IQ2_XS] = {
334 .from_float = NULL((void*)0),
335 .vec_dot = ggml_vec_dot_iq2_xs_q8_K,
336 .vec_dot_type = GGML_TYPE_Q8_K,
337 .nrows = 1,
338 },
339 [GGML_TYPE_IQ3_XXS] = {
340 // NOTE: from_float for iq3 and iq2_s was removed because these quants require initialization in ggml_quantize_init
341 //.from_float = quantize_row_iq3_xxs,
342 .vec_dot = ggml_vec_dot_iq3_xxs_q8_K,
343 .vec_dot_type = GGML_TYPE_Q8_K,
344 .nrows = 1,
345 },
346 [GGML_TYPE_IQ3_S] = {
347 //.from_float = quantize_row_iq3_s,
348 .vec_dot = ggml_vec_dot_iq3_s_q8_K,
349 .vec_dot_type = GGML_TYPE_Q8_K,
350 .nrows = 1,
351 },
352 [GGML_TYPE_IQ2_S] = {
353 //.from_float = quantize_row_iq2_s,
354 .vec_dot = ggml_vec_dot_iq2_s_q8_K,
355 .vec_dot_type = GGML_TYPE_Q8_K,
356 .nrows = 1,
357 },
358 [GGML_TYPE_IQ1_S] = {
359 .from_float = NULL((void*)0),
360 .vec_dot = ggml_vec_dot_iq1_s_q8_K,
361 .vec_dot_type = GGML_TYPE_Q8_K,
362 .nrows = 1,
363 },
364 [GGML_TYPE_IQ1_M] = {
365 .from_float = NULL((void*)0),
366 .vec_dot = ggml_vec_dot_iq1_m_q8_K,
367 .vec_dot_type = GGML_TYPE_Q8_K,
368 .nrows = 1,
369 },
370 [GGML_TYPE_IQ4_NL] = {
371 .from_float = quantize_row_iq4_nl,
372 .vec_dot = ggml_vec_dot_iq4_nl_q8_0,
373 .vec_dot_type = GGML_TYPE_Q8_0,
374 .nrows = 1,
375 },
376 [GGML_TYPE_IQ4_XS] = {
377 .from_float = quantize_row_iq4_xs,
378 .vec_dot = ggml_vec_dot_iq4_xs_q8_K,
379 .vec_dot_type = GGML_TYPE_Q8_K,
380 .nrows = 1,
381 },
382 [GGML_TYPE_Q8_K] = {
383 .from_float = quantize_row_q8_K,
384 },
385 [GGML_TYPE_BF16] = {
386 .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_bf16,
387 .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_bf16,
388 .vec_dot_type = GGML_TYPE_BF16,
389 .nrows = 1,
390 },
391 [GGML_TYPE_TQ1_0] = {
392 .from_float = quantize_row_tq1_0,
393 .vec_dot = ggml_vec_dot_tq1_0_q8_K,
394 .vec_dot_type = GGML_TYPE_Q8_K,
395 .nrows = 1,
396 },
397 [GGML_TYPE_TQ2_0] = {
398 .from_float = quantize_row_tq2_0,
399 .vec_dot = ggml_vec_dot_tq2_0_q8_K,
400 .vec_dot_type = GGML_TYPE_Q8_K,
401 .nrows = 1,
402 },
403 [GGML_TYPE_I32] = {
404 .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_i32,
405 },
406};
407
408const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) {
409 return &type_traits_cpu[type];
410}
411
412//
413// Threading defs
414//
415
416typedef pthread_t ggml_thread_t;
417
418#if defined(_WIN32)
419
420typedef CONDITION_VARIABLE ggml_cond_t;
421typedef SRWLOCK ggml_mutex_t;
422
423#define ggml_mutex_init(m)pthread_mutex_init(m, ((void*)0)) InitializeSRWLock(m)
424#define ggml_mutex_destroy(m)pthread_mutex_destroy(m)
425#define ggml_mutex_lock(m)pthread_mutex_lock(m) AcquireSRWLockExclusive(m)
426#define ggml_mutex_unlock(m)pthread_mutex_unlock(m) ReleaseSRWLockExclusive(m)
427#define ggml_mutex_lock_shared(m)pthread_mutex_lock(m) AcquireSRWLockShared(m)
428#define ggml_mutex_unlock_shared(m)pthread_mutex_unlock(m) ReleaseSRWLockShared(m)
429
430#define ggml_cond_init(c)pthread_cond_init(c, ((void*)0)) InitializeConditionVariable(c)
431#define ggml_cond_destroy(c)pthread_cond_destroy(c)
432#define ggml_cond_wait(c, m)pthread_cond_wait(c, m) SleepConditionVariableSRW(c, m, INFINITE, CONDITION_VARIABLE_LOCKMODE_SHARED)
433#define ggml_cond_broadcast(c)pthread_cond_broadcast(c) WakeAllConditionVariable(c)
434
435#define ggml_thread_createpthread_create pthread_create
436#define ggml_thread_joinpthread_join pthread_join
437
438#else
439
440typedef pthread_cond_t ggml_cond_t;
441typedef pthread_mutex_t ggml_mutex_t;
442
443#define ggml_mutex_init(m)pthread_mutex_init(m, ((void*)0)) pthread_mutex_init(m, NULL((void*)0))
444#define ggml_mutex_destroy(m)pthread_mutex_destroy(m) pthread_mutex_destroy(m)
445#define ggml_mutex_lock(m)pthread_mutex_lock(m) pthread_mutex_lock(m)
446#define ggml_mutex_unlock(m)pthread_mutex_unlock(m) pthread_mutex_unlock(m)
447#define ggml_mutex_lock_shared(m)pthread_mutex_lock(m) pthread_mutex_lock(m)
448#define ggml_mutex_unlock_shared(m)pthread_mutex_unlock(m) pthread_mutex_unlock(m)
449
450#define ggml_lock_init(x)(void)(x) UNUSED(x)(void)(x)
451#define ggml_lock_destroy(x)(void)(x) UNUSED(x)(void)(x)
452#if defined(__x86_64__1) || (defined(_MSC_VER) && defined(_M_AMD64))
453#define ggml_lock_lock(x)_mm_pause() _mm_pause()
454#else
455#define ggml_lock_lock(x)_mm_pause() UNUSED(x)(void)(x)
456#endif
457#define ggml_lock_unlock(x)(void)(x) UNUSED(x)(void)(x)
458
459#define GGML_LOCK_INITIALIZER0 0
460#define ggml_cond_init(c)pthread_cond_init(c, ((void*)0)) pthread_cond_init(c, NULL((void*)0))
461#define ggml_cond_destroy(c)pthread_cond_destroy(c) pthread_cond_destroy(c)
462#define ggml_cond_wait(c, m)pthread_cond_wait(c, m) pthread_cond_wait(c, m)
463#define ggml_cond_broadcast(c)pthread_cond_broadcast(c) pthread_cond_broadcast(c)
464
465#define ggml_thread_createpthread_create pthread_create
466#define ggml_thread_joinpthread_join pthread_join
467
468#endif
469
470// Threadpool def
471struct ggml_threadpool {
472 ggml_mutex_t mutex; // mutex for cond.var
473 ggml_cond_t cond; // cond.var for waiting for new work
474
475 struct ggml_cgraph * cgraph;
476 struct ggml_cplan * cplan;
477
478 // synchronization primitives
479 atomic_int n_graph; // updated when there is work to be done (i.e each graph) holds graph and active thread counts.
480 atomic_int GGML_CACHE_ALIGN__attribute__((aligned(64))) n_barrier;
481 atomic_int GGML_CACHE_ALIGN__attribute__((aligned(64))) n_barrier_passed;
482 atomic_int GGML_CACHE_ALIGN__attribute__((aligned(64))) current_chunk; // currently processing chunk during Mat_Mul, shared between all the threads.
483
484 // these are atomic as an annotation for thread-sanitizer
485 atomic_bool stop; // Used for stopping the threadpool altogether
486 atomic_bool pause; // Used for pausing the threadpool or individual threads
487 atomic_int abort; // Used for aborting processing of a graph
488
489 struct ggml_compute_state * workers; // per thread state
490 int n_threads; // Number of threads in the pool
491 int32_t prio; // Scheduling priority
492 uint32_t poll; // Polling level (0 - no polling)
493
494 void (*thread_create_callback)(void);
495 void (*thread_destroy_callback)(void);
496
497 enum ggml_status ec;
498};
499
500// Per-thread state
501struct ggml_compute_state {
502#ifndef GGML_USE_OPENMP
503 ggml_thread_t thrd;
504 int last_graph;
505 bool_Bool pending;
506#endif
507 bool_Bool cpumask[GGML_MAX_N_THREADS512];
508 struct ggml_threadpool * threadpool;
509 int ith;
510};
511
512// Helpers for polling loops
513#if defined(__aarch64__) && ( defined(__clang__1) || defined(__GNUC__4) )
514static inline void ggml_thread_cpu_relax(void) {
515 __asm__ volatile("yield" ::: "memory");
516}
517#elif defined(__x86_64__1)
518static inline void ggml_thread_cpu_relax(void) {
519 _mm_pause();
520}
521#elif defined(__riscv)
522static inline void ggml_thread_cpu_relax(void) {
523 #ifdef __riscv_zihintpause
524 __asm__ __volatile__ ("pause");
525 #else
526 /* Encoding of the pause instruction */
527 __asm__ __volatile__ (".4byte 0x100000F");
528 #endif
529}
530#else
531static inline void ggml_thread_cpu_relax(void) {;}
532#endif
533
534//
535// NUMA support
536//
537
538#define GGML_NUMA_MAX_NODES8 8
539#define GGML_NUMA_MAX_CPUS512 512
540
541struct ggml_numa_node {
542 uint32_t cpus[GGML_NUMA_MAX_CPUS512]; // hardware threads on this node
543 uint32_t n_cpus;
544};
545
546struct ggml_numa_nodes {
547 enum ggml_numa_strategy numa_strategy;
548 struct ggml_numa_node nodes[GGML_NUMA_MAX_NODES8];
549 uint32_t n_nodes;
550 uint32_t total_cpus; // hardware threads on system
551 uint32_t current_node; // node on which main process is execting
552#if defined(__gnu_linux__1)
553 cpu_set_t cpuset; // cpuset from numactl
554#else
555 uint32_t cpuset; // no NUMA support outside of Linux at this time. Use a portable datatype
556#endif
557};
558
559//
560// ggml state
561//
562
563struct ggml_state {
564 struct ggml_numa_nodes numa;
565};
566
567static struct ggml_state g_state = {0};
568
569void ggml_barrier(struct ggml_threadpool * tp) {
570 int n_threads = atomic_load_explicit__c11_atomic_load(&tp->n_graph, memory_order_relaxed) & GGML_THREADPOOL_N_THREADS_MASK(0xffffU);
571 if (n_threads == 1) {
572 return;
573 }
574
575#ifdef GGML_USE_OPENMP
576 #pragma omp barrier
577#else
578 int n_passed = atomic_load_explicit__c11_atomic_load(&tp->n_barrier_passed, memory_order_relaxed);
579
580 // enter barrier (full seq-cst fence)
581 int n_barrier = atomic_fetch_add_explicit__c11_atomic_fetch_add(&tp->n_barrier, 1, memory_order_seq_cst);
582
583 if (n_barrier == (n_threads - 1)) {
584 // last thread
585 atomic_store_explicit__c11_atomic_store(&tp->n_barrier, 0, memory_order_relaxed);
586
587 // exit barrier (full seq-cst fence)
588 atomic_fetch_add_explicit__c11_atomic_fetch_add(&tp->n_barrier_passed, 1, memory_order_seq_cst);
589 return;
590 }
591
592 // wait for other threads
593 while (atomic_load_explicit__c11_atomic_load(&tp->n_barrier_passed, memory_order_relaxed) == n_passed) {
594 ggml_thread_cpu_relax();
595 }
596
597 // exit barrier (full seq-cst fence)
598 // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead
599 #ifdef GGML_TSAN_ENABLED
600 atomic_fetch_add_explicit__c11_atomic_fetch_add(&tp->n_barrier_passed, 0, memory_order_seq_cst);
601 #else
602 atomic_thread_fence(memory_order_seq_cst)__c11_atomic_thread_fence(memory_order_seq_cst);
603 #endif
604#endif
605}
606
607void ggml_threadpool_chunk_set(struct ggml_threadpool * tp, int value) {
608 atomic_store_explicit__c11_atomic_store(&tp->current_chunk, value, memory_order_relaxed);
609}
610
611int ggml_threadpool_chunk_add(struct ggml_threadpool * tp, int value) {
612 return atomic_fetch_add_explicit__c11_atomic_fetch_add(&tp->current_chunk, value, memory_order_relaxed);
613}
614
615#if defined(__gnu_linux__1)
616static cpu_set_t ggml_get_numa_affinity(void) {
617 cpu_set_t cpuset;
618 pthread_t thread;
619 thread = pthread_self();
620 CPU_ZERO(&cpuset)do __builtin_memset (&cpuset, '\0', sizeof (cpu_set_t)); while
(0);
621 pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
622 return cpuset;
623}
624#else
625static uint32_t ggml_get_numa_affinity(void) {
626 return 0; // no NUMA support
627}
628#endif
629
630void ggml_numa_init(enum ggml_numa_strategy numa_flag) {
631 if (g_state.numa.n_nodes > 0) {
632 fprintf(stderrstderr, "ggml_numa_init: NUMA already initialized\n");
633
634 return;
635 }
636
637#if defined(__gnu_linux__1)
638 struct stat st;
639 char path[256];
640 int rv;
641
642 // set numa scheme
643 g_state.numa.numa_strategy = numa_flag;
644
645 GGML_PRINT_DEBUG("numa strategy %u\n",g_state.numa.numa_strategy);
646
647 g_state.numa.cpuset = ggml_get_numa_affinity();
648
649 // enumerate nodes
650 while (g_state.numa.n_nodes < GGML_NUMA_MAX_NODES8) {
651 rv = snprintf(path, sizeof(path), "/sys/devices/system/node/node%u", g_state.numa.n_nodes);
652 GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path))if (!(rv > 0 && (unsigned)rv < sizeof(path))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 652, "GGML_ASSERT(%s) failed", "rv > 0 && (unsigned)rv < sizeof(path)"
);
653 if (stat(path, &st) != 0) { break; }
654 ++g_state.numa.n_nodes;
655 }
656
657 // enumerate CPUs
658 while (g_state.numa.total_cpus < GGML_NUMA_MAX_CPUS512) {
659 rv = snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%u", g_state.numa.total_cpus);
660 GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path))if (!(rv > 0 && (unsigned)rv < sizeof(path))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 660, "GGML_ASSERT(%s) failed", "rv > 0 && (unsigned)rv < sizeof(path)"
);
661 if (stat(path, &st) != 0) { break; }
662 ++g_state.numa.total_cpus;
663 }
664
665 GGML_PRINT_DEBUG("found %u numa nodes, %u CPUs\n", g_state.numa.n_nodes, g_state.numa.total_cpus);
666
667 // figure out which node we're on
668 uint current_cpu;
669 int getcpu_ret = 0;
670#if __GLIBC__2 > 2 || (__GLIBC__2 == 2 && __GLIBC_MINOR__43 > 33) || defined(__COSMOPOLITAN__)
671 getcpu_ret = getcpu(&current_cpu, &g_state.numa.current_node);
672#else
673 // old glibc doesn't have a wrapper for this call. Fall back on direct syscall
674# if !defined(SYS_getcpu309) && defined(SYS_get_cpu)
675# define SYS_getcpu309 SYS_get_cpu // some older glibc versions use this name
676# endif
677 getcpu_ret = syscall(SYS_getcpu309, &current_cpu, &g_state.numa.current_node);
678#endif
679
680 if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1 || getcpu_ret != 0) {
681 g_state.numa.n_nodes = 0;
682 return;
683 }
684
685 GGML_PRINT_DEBUG("found our process on numa node %u, CPU %u\n", g_state.numa.current_node, current_cpu);
686
687 for (uint32_t n = 0; n < g_state.numa.n_nodes; ++n) {
688 struct ggml_numa_node * node = &g_state.numa.nodes[n];
689 GGML_PRINT_DEBUG("CPUs on node %u:", n);
690 node->n_cpus = 0;
691 for (uint32_t c = 0; c < g_state.numa.total_cpus; ++c) {
692 rv = snprintf(path, sizeof(path), "/sys/devices/system/node/node%u/cpu%u", n, c);
693 GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path))if (!(rv > 0 && (unsigned)rv < sizeof(path))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 693, "GGML_ASSERT(%s) failed", "rv > 0 && (unsigned)rv < sizeof(path)"
);
694 if (stat(path, &st) == 0) {
695 node->cpus[node->n_cpus++] = c;
696 GGML_PRINT_DEBUG(" %u", c);
697 }
698 }
699 GGML_PRINT_DEBUG("\n");
700 }
701
702 if (ggml_is_numa()) {
703 FILE *fptr = fopen("/proc/sys/kernel/numa_balancing", "r");
704 if (fptr != NULL((void*)0)) {
705 char buf[42];
706 if (fgets(buf, sizeof(buf), fptr) && strncmp(buf, "0\n", sizeof(buf)) != 0) {
707 GGML_LOG_WARN("/proc/sys/kernel/numa_balancing is enabled, this has been observed to impair performance\n")ggml_log_internal(GGML_LOG_LEVEL_WARN , "/proc/sys/kernel/numa_balancing is enabled, this has been observed to impair performance\n"
);
708 }
709 fclose(fptr);
710 }
711 }
712#else
713 UNUSED(numa_flag)(void)(numa_flag);
714 // TODO
715#endif
716}
717
718bool_Bool ggml_is_numa(void) {
719 return g_state.numa.n_nodes > 1;
720}
721
722#if defined(__ARM_ARCH)
723#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
724#include <arm_sve.h>
725static void ggml_init_arm_arch_features(void) {
726 ggml_arm_arch_features.sve_cnt = svcntb();
727}
728#else
729static void ggml_init_arm_arch_features(void) {}
730#endif
731#endif // __ARM_ARCH
732
733#if defined(__riscv) && defined(__riscv_v_intrinsic)
734#include <riscv_vector.h>
735static void ggml_init_riscv_arch_features(void) {
736 ggml_riscv_arch_features.rvv_vlen = __riscv_vlenb();
737}
738#else
739static void ggml_init_riscv_arch_features(void) {}
740#endif
741
742struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value) {
743 GGML_ASSERT(!ggml_get_no_alloc(ctx))if (!(!ggml_get_no_alloc(ctx))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 743, "GGML_ASSERT(%s) failed", "!ggml_get_no_alloc(ctx)");
744
745 struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);
746
747 ggml_set_i32(result, value);
748
749 return result;
750}
751
752struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value) {
753 GGML_ASSERT(!ggml_get_no_alloc(ctx))if (!(!ggml_get_no_alloc(ctx))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 753, "GGML_ASSERT(%s) failed", "!ggml_get_no_alloc(ctx)");
754
755 struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
756
757 ggml_set_f32(result, value);
758
759 return result;
760}
761
762struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value) {
763 const int n = ggml_nrows(tensor);
764 const int nc = tensor->ne[0];
765 const size_t n1 = tensor->nb[1];
766
767 char * const data = tensor->data;
768
769 switch (tensor->type) {
770 case GGML_TYPE_I8:
771 {
772 assert(tensor->nb[0] == sizeof(int8_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(int8_t))), __extension__ ({ if (tensor->nb[0] == sizeof(int8_t
)) ; else __assert_fail ("tensor->nb[0] == sizeof(int8_t)"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 772, __extension__ __PRETTY_FUNCTION__); }));
773 for (int i = 0; i < n; i++) {
774 ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);
775 }
776 } break;
777 case GGML_TYPE_I16:
778 {
779 assert(tensor->nb[0] == sizeof(int16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(int16_t))), __extension__ ({ if (tensor->nb[0] == sizeof(
int16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(int16_t)"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 779, __extension__ __PRETTY_FUNCTION__); }));
780 for (int i = 0; i < n; i++) {
781 ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);
782 }
783 } break;
784 case GGML_TYPE_I32:
785 {
786 assert(tensor->nb[0] == sizeof(int32_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(int32_t))), __extension__ ({ if (tensor->nb[0] == sizeof(
int32_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(int32_t)"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 786, __extension__ __PRETTY_FUNCTION__); }));
787 for (int i = 0; i < n; i++) {
788 ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);
789 }
790 } break;
791 case GGML_TYPE_F16:
792 {
793 assert(tensor->nb[0] == sizeof(ggml_fp16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(ggml_fp16_t))), __extension__ ({ if (tensor->nb[0] == sizeof
(ggml_fp16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(ggml_fp16_t)"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 793, __extension__ __PRETTY_FUNCTION__); }));
794 for (int i = 0; i < n; i++) {
795 ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value));
796 }
797 } break;
798 case GGML_TYPE_BF16:
799 {
800 assert(tensor->nb[0] == sizeof(ggml_fp16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(ggml_fp16_t))), __extension__ ({ if (tensor->nb[0] == sizeof
(ggml_fp16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(ggml_fp16_t)"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 800, __extension__ __PRETTY_FUNCTION__); }));
801 for (int i = 0; i < n; i++) {
802 ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value));
803 }
804 } break;
805 case GGML_TYPE_F32:
806 {
807 assert(tensor->nb[0] == sizeof(float))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(float))), __extension__ ({ if (tensor->nb[0] == sizeof(float
)) ; else __assert_fail ("tensor->nb[0] == sizeof(float)",
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 807, __extension__ __PRETTY_FUNCTION__); }));
808 for (int i = 0; i < n; i++) {
809 ggml_vec_set_f32(nc, (float *)(data + i*n1), value);
810 }
811 } break;
812 default:
813 {
814 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 814, "fatal error");
815 }
816 }
817
818 return tensor;
819}
820
821struct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) {
822 const int n = ggml_nrows(tensor);
823 const int nc = tensor->ne[0];
824 const size_t n1 = tensor->nb[1];
825
826 char * const data = tensor->data;
827
828 switch (tensor->type) {
829 case GGML_TYPE_I8:
830 {
831 assert(tensor->nb[0] == sizeof(int8_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(int8_t))), __extension__ ({ if (tensor->nb[0] == sizeof(int8_t
)) ; else __assert_fail ("tensor->nb[0] == sizeof(int8_t)"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 831, __extension__ __PRETTY_FUNCTION__); }));
832 for (int i = 0; i < n; i++) {
833 ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);
834 }
835 } break;
836 case GGML_TYPE_I16:
837 {
838 assert(tensor->nb[0] == sizeof(int16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(int16_t))), __extension__ ({ if (tensor->nb[0] == sizeof(
int16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(int16_t)"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 838, __extension__ __PRETTY_FUNCTION__); }));
839 for (int i = 0; i < n; i++) {
840 ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);
841 }
842 } break;
843 case GGML_TYPE_I32:
844 {
845 assert(tensor->nb[0] == sizeof(int32_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(int32_t))), __extension__ ({ if (tensor->nb[0] == sizeof(
int32_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(int32_t)"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 845, __extension__ __PRETTY_FUNCTION__); }));
846 for (int i = 0; i < n; i++) {
847 ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);
848 }
849 } break;
850 case GGML_TYPE_F16:
851 {
852 assert(tensor->nb[0] == sizeof(ggml_fp16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(ggml_fp16_t))), __extension__ ({ if (tensor->nb[0] == sizeof
(ggml_fp16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(ggml_fp16_t)"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 852, __extension__ __PRETTY_FUNCTION__); }));
853 for (int i = 0; i < n; i++) {
854 ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value));
855 }
856 } break;
857 case GGML_TYPE_BF16:
858 {
859 assert(tensor->nb[0] == sizeof(ggml_bf16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(ggml_bf16_t))), __extension__ ({ if (tensor->nb[0] == sizeof
(ggml_bf16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(ggml_bf16_t)"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 859, __extension__ __PRETTY_FUNCTION__); }));
860 for (int i = 0; i < n; i++) {
861 ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value));
862 }
863 } break;
864 case GGML_TYPE_F32:
865 {
866 assert(tensor->nb[0] == sizeof(float))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof
(float))), __extension__ ({ if (tensor->nb[0] == sizeof(float
)) ; else __assert_fail ("tensor->nb[0] == sizeof(float)",
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 866, __extension__ __PRETTY_FUNCTION__); }));
867 for (int i = 0; i < n; i++) {
868 ggml_vec_set_f32(nc, (float *)(data + i*n1), value);
869 }
870 } break;
871 default:
872 {
873 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 873, "fatal error");
874 }
875 }
876
877 return tensor;
878}
879
880int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i) {
881 if (!ggml_is_contiguous(tensor)) {
882 int64_t id[4] = { 0, 0, 0, 0 };
883 ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
884 return ggml_get_i32_nd(tensor, id[0], id[1], id[2], id[3]);
885 }
886 switch (tensor->type) {
887 case GGML_TYPE_I8:
888 {
889 GGML_ASSERT(tensor->nb[0] == sizeof(int8_t))if (!(tensor->nb[0] == sizeof(int8_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 889, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int8_t)"
);
890 return ((int8_t *)(tensor->data))[i];
891 }
892 case GGML_TYPE_I16:
893 {
894 GGML_ASSERT(tensor->nb[0] == sizeof(int16_t))if (!(tensor->nb[0] == sizeof(int16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 894, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int16_t)"
);
895 return ((int16_t *)(tensor->data))[i];
896 }
897 case GGML_TYPE_I32:
898 {
899 GGML_ASSERT(tensor->nb[0] == sizeof(int32_t))if (!(tensor->nb[0] == sizeof(int32_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 899, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int32_t)"
);
900 return ((int32_t *)(tensor->data))[i];
901 }
902 case GGML_TYPE_F16:
903 {
904 GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t))if (!(tensor->nb[0] == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 904, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(ggml_fp16_t)"
);
905 return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i])ggml_lookup_fp16_to_fp32(((ggml_fp16_t *)(tensor->data))[i
]);
906 }
907 case GGML_TYPE_BF16:
908 {
909 GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t))if (!(tensor->nb[0] == sizeof(ggml_bf16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 909, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(ggml_bf16_t)"
);
910 return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i])ggml_compute_bf16_to_fp32(((ggml_bf16_t *)(tensor->data))[
i]);
911 }
912 case GGML_TYPE_F32:
913 {
914 GGML_ASSERT(tensor->nb[0] == sizeof(float))if (!(tensor->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 914, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(float)"
);
915 return ((float *)(tensor->data))[i];
916 }
917 default:
918 {
919 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 919, "fatal error");
920 }
921 }
922}
923
924void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value) {
925 if (!ggml_is_contiguous(tensor)) {
926 int64_t id[4] = { 0, 0, 0, 0 };
927 ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
928 ggml_set_i32_nd(tensor, id[0], id[1], id[2], id[3], value);
929 return;
930 }
931 switch (tensor->type) {
932 case GGML_TYPE_I8:
933 {
934 GGML_ASSERT(tensor->nb[0] == sizeof(int8_t))if (!(tensor->nb[0] == sizeof(int8_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 934, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int8_t)"
);
935 ((int8_t *)(tensor->data))[i] = value;
936 } break;
937 case GGML_TYPE_I16:
938 {
939 GGML_ASSERT(tensor->nb[0] == sizeof(int16_t))if (!(tensor->nb[0] == sizeof(int16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 939, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int16_t)"
);
940 ((int16_t *)(tensor->data))[i] = value;
941 } break;
942 case GGML_TYPE_I32:
943 {
944 GGML_ASSERT(tensor->nb[0] == sizeof(int32_t))if (!(tensor->nb[0] == sizeof(int32_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 944, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int32_t)"
);
945 ((int32_t *)(tensor->data))[i] = value;
946 } break;
947 case GGML_TYPE_F16:
948 {
949 GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t))if (!(tensor->nb[0] == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 949, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(ggml_fp16_t)"
);
950 ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value);
951 } break;
952 case GGML_TYPE_BF16:
953 {
954 GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t))if (!(tensor->nb[0] == sizeof(ggml_bf16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 954, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(ggml_bf16_t)"
);
955 ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value);
956 } break;
957 case GGML_TYPE_F32:
958 {
959 GGML_ASSERT(tensor->nb[0] == sizeof(float))if (!(tensor->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 959, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(float)"
);
960 ((float *)(tensor->data))[i] = value;
961 } break;
962 default:
963 {
964 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 964, "fatal error");
965 }
966 }
967}
968
969int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {
970 void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
971 switch (tensor->type) {
972 case GGML_TYPE_I8:
973 return ((int8_t *) data)[0];
974 case GGML_TYPE_I16:
975 return ((int16_t *) data)[0];
976 case GGML_TYPE_I32:
977 return ((int32_t *) data)[0];
978 case GGML_TYPE_F16:
979 return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0])ggml_lookup_fp16_to_fp32(((ggml_fp16_t *) data)[0]);
980 case GGML_TYPE_BF16:
981 return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0])ggml_compute_bf16_to_fp32(((ggml_bf16_t *) data)[0]);
982 case GGML_TYPE_F32:
983 return ((float *) data)[0];
984 default:
985 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 985, "fatal error");
986 }
987}
988
989void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value) {
990 void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
991 switch (tensor->type) {
992 case GGML_TYPE_I8:
993 {
994 ((int8_t *)(data))[0] = value;
995 } break;
996 case GGML_TYPE_I16:
997 {
998 ((int16_t *)(data))[0] = value;
999 } break;
1000 case GGML_TYPE_I32:
1001 {
1002 ((int32_t *)(data))[0] = value;
1003 } break;
1004 case GGML_TYPE_F16:
1005 {
1006 ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value);
1007 } break;
1008 case GGML_TYPE_BF16:
1009 {
1010 ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value);
1011 } break;
1012 case GGML_TYPE_F32:
1013 {
1014 ((float *)(data))[0] = value;
1015 } break;
1016 default:
1017 {
1018 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1018, "fatal error");
1019 }
1020 }
1021}
1022
1023float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i) {
1024 if (!ggml_is_contiguous(tensor)) {
1025 int64_t id[4] = { 0, 0, 0, 0 };
1026 ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
1027 return ggml_get_f32_nd(tensor, id[0], id[1], id[2], id[3]);
1028 }
1029 switch (tensor->type) {
1030 case GGML_TYPE_I8:
1031 {
1032 return ((int8_t *)(tensor->data))[i];
1033 }
1034 case GGML_TYPE_I16:
1035 {
1036 return ((int16_t *)(tensor->data))[i];
1037 }
1038 case GGML_TYPE_I32:
1039 {
1040 return ((int32_t *)(tensor->data))[i];
1041 }
1042 case GGML_TYPE_F16:
1043 {
1044 return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i])ggml_lookup_fp16_to_fp32(((ggml_fp16_t *)(tensor->data))[i
]);
1045 }
1046 case GGML_TYPE_BF16:
1047 {
1048 return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i])ggml_compute_bf16_to_fp32(((ggml_bf16_t *)(tensor->data))[
i]);
1049 }
1050 case GGML_TYPE_F32:
1051 {
1052 return ((float *)(tensor->data))[i];
1053 }
1054 default:
1055 {
1056 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1056, "fatal error");
1057 }
1058 }
1059}
1060
1061void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value) {
1062 if (!ggml_is_contiguous(tensor)) {
1063 int64_t id[4] = { 0, 0, 0, 0 };
1064 ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
1065 ggml_set_f32_nd(tensor, id[0], id[1], id[2], id[3], value);
1066 return;
1067 }
1068 switch (tensor->type) {
1069 case GGML_TYPE_I8:
1070 {
1071 ((int8_t *)(tensor->data))[i] = value;
1072 } break;
1073 case GGML_TYPE_I16:
1074 {
1075 ((int16_t *)(tensor->data))[i] = value;
1076 } break;
1077 case GGML_TYPE_I32:
1078 {
1079 ((int32_t *)(tensor->data))[i] = value;
1080 } break;
1081 case GGML_TYPE_F16:
1082 {
1083 ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value);
1084 } break;
1085 case GGML_TYPE_BF16:
1086 {
1087 ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value);
1088 } break;
1089 case GGML_TYPE_F32:
1090 {
1091 ((float *)(tensor->data))[i] = value;
1092 } break;
1093 default:
1094 {
1095 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1095, "fatal error");
1096 }
1097 }
1098}
1099
1100float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {
1101 void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
1102 switch (tensor->type) {
1103 case GGML_TYPE_I8:
1104 return ((int8_t *) data)[0];
1105 case GGML_TYPE_I16:
1106 return ((int16_t *) data)[0];
1107 case GGML_TYPE_I32:
1108 return ((int32_t *) data)[0];
1109 case GGML_TYPE_F16:
1110 return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0])ggml_lookup_fp16_to_fp32(((ggml_fp16_t *) data)[0]);
1111 case GGML_TYPE_BF16:
1112 return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0])ggml_compute_bf16_to_fp32(((ggml_bf16_t *) data)[0]);
1113 case GGML_TYPE_F32:
1114 return ((float *) data)[0];
1115 default:
1116 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1116, "fatal error");
1117 }
1118}
1119
1120void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value) {
1121 void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
1122 switch (tensor->type) {
1123 case GGML_TYPE_I8:
1124 {
1125 ((int8_t *)(data))[0] = value;
1126 } break;
1127 case GGML_TYPE_I16:
1128 {
1129 ((int16_t *)(data))[0] = value;
1130 } break;
1131 case GGML_TYPE_I32:
1132 {
1133 ((int32_t *)(data))[0] = value;
1134 } break;
1135 case GGML_TYPE_F16:
1136 {
1137 ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value);
1138 } break;
1139 case GGML_TYPE_BF16:
1140 {
1141 ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value);
1142 } break;
1143 case GGML_TYPE_F32:
1144 {
1145 ((float *)(data))[0] = value;
1146 } break;
1147 default:
1148 {
1149 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1149, "fatal error");
1150 }
1151 }
1152}
1153
1154////////////////////////////////////////////////////////////////////////////////
1155
1156// ggml_compute_forward_mul_mat
1157
1158static void ggml_compute_forward_mul_mat_one_chunk(
1159 const struct ggml_compute_params * params,
1160 struct ggml_tensor * dst,
1161 const enum ggml_type type,
1162 const int64_t num_rows_per_vec_dot,
1163 const int64_t ir0_start,
1164 const int64_t ir0_end,
1165 const int64_t ir1_start,
1166 const int64_t ir1_end) {
1167
1168 const struct ggml_tensor * src0 = dst->src[0];
1169 const struct ggml_tensor * src1 = dst->src[1];
1170
1171 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
1172
1173 const bool_Bool src1_cont = ggml_is_contiguous(src1);
1174
1175 ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;
1176 enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
1177
1178 // broadcast factors
1179 const int64_t r2 = ne12 / ne02;
1180 const int64_t r3 = ne13 / ne03;
1181
1182 //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end);
1183
1184 // threads with no work simply yield (not sure if it helps)
1185 if (ir0_start >= ir0_end || ir1_start >= ir1_end) {
1186 return;
1187 }
1188
1189 const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
1190 const size_t row_size = ggml_row_size(vec_dot_type, ne10);
1191
1192 assert(ne12 % ne02 == 0)((void) sizeof (__assert_single_arg (ne12 % ne02 == 0)), __extension__
({ if (ne12 % ne02 == 0) ; else __assert_fail ("ne12 % ne02 == 0"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1192, __extension__ __PRETTY_FUNCTION__); }));
1193 assert(ne13 % ne03 == 0)((void) sizeof (__assert_single_arg (ne13 % ne03 == 0)), __extension__
({ if (ne13 % ne03 == 0) ; else __assert_fail ("ne13 % ne03 == 0"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1193, __extension__ __PRETTY_FUNCTION__); }));
1194
1195 // block-tiling attempt
1196 const int64_t blck_0 = 16;
1197 const int64_t blck_1 = 16;
1198
1199 const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11;
1200
1201 // attempt to reduce false-sharing (does not seem to make a difference)
1202 // 16 * 2, accounting for mmla kernels
1203 float tmp[32];
1204
1205 for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {
1206 for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {
1207 for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) {
1208 const int64_t i13 = (ir1 / (ne12 * ne1));
1209 const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1;
1210 const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1);
1211
1212 // broadcast src0 into src1
1213 const int64_t i03 = i13 / r3;
1214 const int64_t i02 = i12 / r2;
1215
1216 const int64_t i1 = i11;
1217 const int64_t i2 = i12;
1218 const int64_t i3 = i13;
1219
1220 const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03);
1221
1222 // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides
1223 // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using
1224 // the original src1 data pointer, so we should index using the indices directly
1225 // TODO: this is a bit of a hack, we should probably have a better way to handle this
1226 const char * src1_col = (const char*)wdata +
1227 (src1_cont || src1->type != vec_dot_type
1228 ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size
1229 : (i11 * nb11 + i12 * nb12 + i13 * nb13));
1230 float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3));
1231
1232 //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) {
1233 // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col);
1234 //}
1235
1236 for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) {
1237 vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot);
1238 }
1239
1240 for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) {
1241 memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), (MIN(iir0 + blck_0, ir0_end)((iir0 + blck_0) < (ir0_end) ? (iir0 + blck_0) : (ir0_end)
) - iir0) * sizeof(float));
1242 }
1243 }
1244 }
1245 }
1246}
1247
1248void ggml_compute_forward_mul_mat(
1249 const struct ggml_compute_params * params,
1250 struct ggml_tensor * dst) {
1251
1252 const struct ggml_tensor * src0 = dst->src[0];
1253 const struct ggml_tensor * src1 = dst->src[1];
1254
1255 const int32_t hint = ggml_get_op_params_i32(dst, 1);
1256 if (hint == GGML_HINT_SRC0_IS_HADAMARD && !params->use_ref) {
1257 ggml_compute_forward_fwht(params, dst);
1258 return;
1259 }
1260
1261 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
1262
1263 const int ith = params->ith;
1264 const int nth = params->nth;
1265
1266 enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;
1267 ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
1268 int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows;
1269
1270 GGML_ASSERT(ne0 == ne01)if (!(ne0 == ne01)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1270, "GGML_ASSERT(%s) failed", "ne0 == ne01");
1271 GGML_ASSERT(ne1 == ne11)if (!(ne1 == ne11)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1271, "GGML_ASSERT(%s) failed", "ne1 == ne11");
1272 GGML_ASSERT(ne2 == ne12)if (!(ne2 == ne12)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1272, "GGML_ASSERT(%s) failed", "ne2 == ne12");
1273 GGML_ASSERT(ne3 == ne13)if (!(ne3 == ne13)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1273, "GGML_ASSERT(%s) failed", "ne3 == ne13");
1274
1275 // we don't support permuted src0 or src1
1276 GGML_ASSERT(nb00 == ggml_type_size(src0->type))if (!(nb00 == ggml_type_size(src0->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1276, "GGML_ASSERT(%s) failed", "nb00 == ggml_type_size(src0->type)"
);
1277 GGML_ASSERT(nb10 == ggml_type_size(src1->type))if (!(nb10 == ggml_type_size(src1->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1277, "GGML_ASSERT(%s) failed", "nb10 == ggml_type_size(src1->type)"
);
1278
1279 // dst cannot be transposed or permuted
1280 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1280, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
1281 GGML_ASSERT(nb0 <= nb1)if (!(nb0 <= nb1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1281, "GGML_ASSERT(%s) failed", "nb0 <= nb1");
1282 GGML_ASSERT(nb1 <= nb2)if (!(nb1 <= nb2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1282, "GGML_ASSERT(%s) failed", "nb1 <= nb2");
1283 GGML_ASSERT(nb2 <= nb3)if (!(nb2 <= nb3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1283, "GGML_ASSERT(%s) failed", "nb2 <= nb3");
1284
1285 // nb01 >= nb00 - src0 is not transposed
1286 // compute by src0 rows
1287
1288 // TODO: extract to "extra_op"
1289#if GGML_USE_LLAMAFILE
1290 // broadcast factors
1291 const int64_t r2 = ne12 / ne02;
1292 const int64_t r3 = ne13 / ne03;
1293
1294 const bool_Bool src1_cont = ggml_is_contiguous(src1);
1295
1296 if (src1_cont) {
1297 for (int64_t i13 = 0; i13 < ne13; i13++)
1298 for (int64_t i12 = 0; i12 < ne12; i12++)
1299 if (!llamafile_sgemm(params,
1300 ne01, ne11, ne00/ggml_blck_size(src0->type),
1301 (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,
1302 nb01/ggml_type_size(src0->type),
1303 (const char *)src1->data + i12*nb12 + i13*nb13,
1304 nb11/ggml_type_size(src1->type),
1305 (char *)dst->data + i12*nb2 + i13*nb3,
1306 nb1/ggml_type_size(dst->type),
1307 src0->type,
1308 src1->type,
1309 dst->type))
1310 goto UseGgmlGemm1;
1311 return;
1312 }
1313UseGgmlGemm1:;
1314#endif
1315
1316 if (src1->type != vec_dot_type) {
1317 char * wdata = params->wdata;
1318
1319 const size_t nbw0 = ggml_type_size(vec_dot_type);
1320 const size_t nbw1 = ggml_row_size(vec_dot_type, ne10);
1321 const size_t nbw2 = nbw1*ne11;
1322 const size_t nbw3 = nbw2*ne12;
1323
1324 assert(params->wsize >= ne13*nbw3)((void) sizeof (__assert_single_arg (params->wsize >= ne13
*nbw3)), __extension__ ({ if (params->wsize >= ne13*nbw3
) ; else __assert_fail ("params->wsize >= ne13*nbw3", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1324, __extension__ __PRETTY_FUNCTION__); }));
1325 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1325, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
1326
1327 #if 0
1328 for (int64_t i13 = 0; i13 < ne13; ++i13) {
1329 for (int64_t i12 = 0; i12 < ne12; ++i12) {
1330 for (int64_t i11 = ith; i11 < ne11; i11 += nth) {
1331 from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),
1332 (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),
1333 ne10);
1334 }
1335 }
1336 }
1337 #else
1338 for (int64_t i13 = 0; i13 < ne13; ++i13) {
1339 for (int64_t i12 = 0; i12 < ne12; ++i12) {
1340 for (int64_t i11 = 0; i11 < ne11; ++i11) {
1341 size_t bs = ggml_blck_size(vec_dot_type);
1342 int64_t ne10_block_start = (ith * ne10/bs) / nth;
1343 int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth;
1344 from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),
1345 (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),
1346 (ne10_block_end - ne10_block_start) * bs);
1347 }
1348 }
1349 }
1350 #endif
1351 }
1352
1353 if (ith == 0) {
1354 // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.
1355 atomic_store_explicit__c11_atomic_store(&params->threadpool->current_chunk, nth, memory_order_relaxed);
1356 }
1357
1358 ggml_barrier(params->threadpool);
1359
1360#if GGML_USE_LLAMAFILE
1361 if (src1->type != vec_dot_type) {
1362 const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
1363 const size_t row_size = ggml_row_size(vec_dot_type, ne10);
1364
1365 for (int64_t i13 = 0; i13 < ne13; i13++)
1366 for (int64_t i12 = 0; i12 < ne12; i12++)
1367 if (!llamafile_sgemm(params,
1368 ne01, ne11, ne00/ggml_blck_size(src0->type),
1369 (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,
1370 nb01/ggml_type_size(src0->type),
1371 (const char *)wdata + (i12*ne11 + i13*ne12*ne11)*row_size,
1372 row_size/ggml_type_size(vec_dot_type),
1373 (char *)dst->data + i12*nb2 + i13*nb3,
1374 nb1/ggml_type_size(dst->type),
1375 src0->type,
1376 vec_dot_type,
1377 dst->type))
1378 goto UseGgmlGemm2;
1379 return;
1380 }
1381UseGgmlGemm2:;
1382#endif
1383
1384 // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers)
1385 const int64_t nr0 = ne0;
1386
1387 // This is the size of the rest of the dimensions of the result
1388 const int64_t nr1 = ne1 * ne2 * ne3;
1389
1390 // Now select a reasonable chunk size.
1391 int chunk_size = 16;
1392
1393 // We need to step up the size if it's small
1394 if (nr0 == 1 || nr1 == 1) {
1395 chunk_size = 64;
1396 }
1397
1398 // distribute the work across the inner or outer loop based on which one is larger
1399 // The number of chunks in the 0/1 dim.
1400 // CEIL(nr0/chunk_size)
1401 int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;
1402 int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;
1403
1404 // If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread.
1405 // Also, chunking by thread was measured to have perform better on NUMA systems. See https://github.com/ggml-org/llama.cpp/pull/6915
1406 // In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that.
1407 if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) {
1408 // distribute the thread work across the inner or outer loop based on which one is larger
1409 nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows
1410 nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows
1411 }
1412
1413 // The number of elements in each chunk
1414 const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;
1415 const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;
1416
1417 // The first chunk comes from our thread_id, the rest will get auto-assigned.
1418 int current_chunk = ith;
1419
1420 while (current_chunk < nchunk0 * nchunk1) {
1421 const int64_t ith0 = current_chunk % nchunk0;
1422 const int64_t ith1 = current_chunk / nchunk0;
1423
1424 const int64_t ir0_start = dr0 * ith0;
1425 const int64_t ir0_end = MIN(ir0_start + dr0, nr0)((ir0_start + dr0) < (nr0) ? (ir0_start + dr0) : (nr0));
1426
1427 const int64_t ir1_start = dr1 * ith1;
1428 const int64_t ir1_end = MIN(ir1_start + dr1, nr1)((ir1_start + dr1) < (nr1) ? (ir1_start + dr1) : (nr1));
1429
1430 // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols
1431 int64_t num_rows_per_vec_dot = vec_dot_num_rows;
1432
1433 // these checks are needed to avoid crossing dim1 boundaries
1434 // can be optimized, but the logic would become more complicated, so keeping it like this for simplicity
1435 if ((nr0 % 2 != 0) || (ne11 % 2 != 0) || ((ir0_end - ir0_start) % 2 != 0) || ((ir1_end - ir1_start) % 2 != 0)) {
1436 num_rows_per_vec_dot = 1;
1437 }
1438 ggml_compute_forward_mul_mat_one_chunk(params, dst, src0->type, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end);
1439
1440 if (nth >= nchunk0 * nchunk1) {
1441 break;
1442 }
1443
1444 current_chunk = atomic_fetch_add_explicit__c11_atomic_fetch_add(&params->threadpool->current_chunk, 1, memory_order_relaxed);
1445 }
1446}
1447
1448// ggml_compute_forward_mul_mat_id
1449
1450#define MMID_MATRIX_ROW(row_id, i1)matrix_rows[(row_id)*ids->ne[0]*ids->ne[1] + (i1)] matrix_rows[(row_id)*ids->ne[0]*ids->ne[1] + (i1)]
1451
1452struct mmid_row_mapping {
1453 int32_t i1;
1454 int32_t i2;
1455};
1456
1457static void ggml_compute_forward_mul_mat_id_one_chunk(
1458 struct ggml_tensor * dst,
1459 const struct ggml_tensor * src0,
1460 const struct ggml_tensor * src1,
1461 const struct ggml_tensor * ids,
1462 const int64_t cur_a,
1463 const int64_t ir0_start,
1464 const int64_t ir0_end,
1465 const int64_t ir1_start,
1466 const int64_t ir1_end,
1467 const char * src0_cur,
1468 const struct mmid_row_mapping * matrix_rows,
1469 const size_t row_size,
1470 const bool_Bool src1_cont,
1471 const void * wdata) {
1472
1473 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
1474
1475 const enum ggml_type type = src0->type;
1476
1477 ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;
1478 enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
1479
1480 const int64_t blck_0 = 16;
1481 const int64_t blck_1 = 16;
1482
1483 float tmp[16];
42
←
'tmp' declared without an initial value
→
1484
1485 for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {
43
←
Assuming 'iir1' is < 'ir1_end'
→
44
←
Loop condition is true. Entering loop body
→
1486 for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {
45
←
Assuming 'iir0' is < 'ir0_end'
→
46
←
Loop condition is true. Entering loop body
→
1487 for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1
47.1
'ir1' is < 'ir1_end'
 < ir1_end; ++ir1) {
47
←
Assuming the condition is true
→
48
←
Loop condition is true. Entering loop body
→
1488 const int64_t _i12 = ir1; // logical row index for this expert
1489
1490 struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, _i12)matrix_rows[(cur_a)*ids->ne[0]*ids->ne[1] + (_i12)];
1491 const int id = row_mapping.i1; // selected expert index
1492
1493 const int64_t i11 = id % ne11;
1494 const int64_t i12 = row_mapping.i2; // row index in src1
1495
1496 const int64_t i1 = id; // selected expert index
1497 const int64_t i2 = i12; // row
1498
1499 // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides
1500 // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using
1501 // the original src1 data pointer, so we should index using the indices directly
1502 // TODO: this is a bit of a hack, we should probably have a better way to handle this
1503 const char * src1_col = (const char *) wdata +
1504 (src1_cont || src1->type != vec_dot_type
49
←
Assuming 'src1_cont' is true
→
1505 ? (i11 + i12*ne11)*row_size
1506 : (i11*nb11 + i12*nb12));
1507
1508 float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2));
1509
1510 for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) {
50
←
Assuming the condition is false
→
1511 vec_dot(ne00, &tmp[ir0 - iir0], 0, src0_cur + ir0*nb01, 0, src1_col, 0, 1);
1512 }
1513
1514 memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir0_end)((iir0 + blck_0) < (ir0_end) ? (iir0 + blck_0) : (ir0_end)
) - iir0)*sizeof(float));
Other elements might also be undefined
51
←
Assuming the condition is false
→
52
←
'?' condition is false
→
53
←
The first element of the 2nd argument is undefined
1515 }
1516 }
1517 }
1518}
1519
1520static void * incr_ptr_aligned(void ** p, size_t size, size_t align) {
1521
1522 void * ptr = *p;
1523 ptr = (void *) GGML_PAD((uintptr_t) ptr, align)((((uintptr_t) ptr) + (align) - 1) & ~((align) - 1));
1524 *p = (void *) ((char *) ptr + size);
1525 return ptr;
1526}
1527
1528static void ggml_compute_forward_mul_mat_id(
1529 const struct ggml_compute_params * params,
1530 struct ggml_tensor * dst) {
1531
1532 const struct ggml_tensor * src0 = dst->src[0];
1533 const struct ggml_tensor * src1 = dst->src[1];
1534 const struct ggml_tensor * ids = dst->src[2];
1535
1536 GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00
); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void)
(ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; (
void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] :
0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[
0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)->
nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)->
nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)->
nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1)
->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? (
src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1
) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = (
src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10
= (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11
= (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12
= (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13
= (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t
ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t
ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t
ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t
ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t
nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t
nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t
nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t
nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3);
1
Assuming 'src0' is non-null
→
2
←
Assuming 'src1' is non-null
→
1537
1538 const int ith = params->ith;
1539 const int nth = params->nth;
1540
1541 const enum ggml_type type = src0->type;
1542
1543 const bool_Bool src1_cont = ggml_is_contiguous(src1);
1544
1545 enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
1546 ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
1547
1548 // we don't support permuted src0 or src1
1549 GGML_ASSERT(nb00 == ggml_type_size(type))if (!(nb00 == ggml_type_size(type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1549, "GGML_ASSERT(%s) failed", "nb00 == ggml_type_size(type)"
);
3
←
Assuming the condition is true
→
4
←
Taking false branch
→
1550 GGML_ASSERT(nb10 == ggml_type_size(src1->type))if (!(nb10 == ggml_type_size(src1->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1550, "GGML_ASSERT(%s) failed", "nb10 == ggml_type_size(src1->type)"
);
5
←
Assuming the condition is true
→
6
←
Taking false branch
→
1551
1552 // dst cannot be transposed or permuted
1553 GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1553, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)");
7
←
Assuming the condition is true
→
8
←
Taking false branch
→
1554 GGML_ASSERT(nb0 <= nb1)if (!(nb0 <= nb1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1554, "GGML_ASSERT(%s) failed", "nb0 <= nb1");
9
←
Assuming 'nb0' is <= 'nb1'
→
10
←
Taking false branch
→
1555 GGML_ASSERT(nb1 <= nb2)if (!(nb1 <= nb2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1555, "GGML_ASSERT(%s) failed", "nb1 <= nb2");
11
←
Assuming 'nb1' is <= 'nb2'
→
12
←
Taking false branch
→
1556 GGML_ASSERT(nb2 <= nb3)if (!(nb2 <= nb3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1556, "GGML_ASSERT(%s) failed", "nb2 <= nb3");
13
←
Assuming 'nb2' is <= 'nb3'
→
14
←
Taking false branch
→
1557
1558 // row groups
1559 const int n_ids = ids->ne[0]; // n_expert_used
1560 const int n_as = ne02; // n_expert
1561
1562 void * wdata_cur = params->wdata;
1563
1564 if (src1->type != vec_dot_type) {
15
←
Assuming 'vec_dot_type' is equal to field 'type'
→
16
←
Taking false branch
→
1565 incr_ptr_aligned(&wdata_cur, ggml_row_size(vec_dot_type, ggml_nelements(src1)), sizeof(int64_t));
1566 }
1567
1568 int64_t * matrix_row_counts = // [n_as]
1569 incr_ptr_aligned(&wdata_cur, n_as*sizeof(int64_t), sizeof(int64_t));
1570
1571 struct mmid_row_mapping * matrix_rows = // [n_as][ids->ne[0]*ids->ne[1]]
1572 incr_ptr_aligned(&wdata_cur, n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping), sizeof(int64_t));
1573
1574 char (*atomic_current_chunk)[CACHE_LINE_SIZE64] = // [n_as]
1575 incr_ptr_aligned(&wdata_cur, CACHE_LINE_SIZE64 * n_as, CACHE_LINE_SIZE64);
1576
1577 GGML_ASSERT(params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata))if (!(params->wsize >= (size_t)((char *) wdata_cur - (char
*) params->wdata))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1577, "GGML_ASSERT(%s) failed", "params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata)"
);
17
←
Assuming the condition is false
→
18
←
Taking false branch
→
1578
1579 if (src1->type != vec_dot_type
18.1
'vec_dot_type' is equal to field 'type'
) {
19
←
Taking false branch
→
1580 char * wdata = params->wdata;
1581
1582 const size_t nbw0 = ggml_type_size(vec_dot_type);
1583 const size_t nbw1 = ggml_row_size(vec_dot_type, ne10);
1584 const size_t nbw2 = nbw1*ne11;
1585 const size_t nbw3 = nbw2*ne12;
1586
1587 assert(params->wsize >= ne13*nbw3)((void) sizeof (__assert_single_arg (params->wsize >= ne13
*nbw3)), __extension__ ({ if (params->wsize >= ne13*nbw3
) ; else __assert_fail ("params->wsize >= ne13*nbw3", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1587, __extension__ __PRETTY_FUNCTION__); }));
1588 GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1588, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32"
);
1589
1590#if 0
1591 for (int64_t i13 = 0; i13 < ne13; ++i13) {
1592 for (int64_t i12 = ith; i12 < ne12; i12 += nth) {
1593 for (int64_t i11 = 0; i11 < ne11; ++i11) {
1594 from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),
1595 (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),
1596 ne10);
1597 }
1598 }
1599 }
1600#else
1601 for (int64_t i13 = 0; i13 < ne13; ++i13) {
1602 for (int64_t i12 = 0; i12 < ne12; ++i12) {
1603 for (int64_t i11 = 0; i11 < ne11; ++i11) {
1604 size_t bs = ggml_blck_size(vec_dot_type);
1605 int64_t ne10_block_start = (ith * ne10/bs) / nth;
1606 int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth;
1607 from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),
1608 (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),
1609 (ne10_block_end - ne10_block_start) * bs);
1610 }
1611 }
1612 }
1613#endif
1614 }
1615
1616 if (ith == 0) {
20
←
Assuming 'ith' is not equal to 0
→
21
←
Taking false branch
→
1617 // initialize matrix_row_counts
1618 memset(matrix_row_counts, 0, n_as*sizeof(int64_t));
1619
1620 // group rows by src0 matrix
1621 for (int64_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
1622 for (int id = 0; id < n_ids; ++id) {
1623 const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1*ids->nb[1] + id*ids->nb[0]);
1624
1625 assert(i02 >= 0 && i02 < n_as)((void) sizeof (__assert_single_arg (i02 >= 0 && i02
< n_as)), __extension__ ({ if (i02 >= 0 && i02
< n_as) ; else __assert_fail ("i02 >= 0 && i02 < n_as"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1625, __extension__ __PRETTY_FUNCTION__); }));
1626
1627 MMID_MATRIX_ROW(i02, matrix_row_counts[i02])matrix_rows[(i02)*ids->ne[0]*ids->ne[1] + (matrix_row_counts
[i02])] = (struct mmid_row_mapping) {id, iid1};
1628 matrix_row_counts[i02] += 1;
1629 }
1630 }
1631 }
1632
1633 // reset current_chunk
1634 for (int cur_a = ith; cur_a < n_as; cur_a += nth) {
22
←
Assuming 'cur_a' is >= 'n_as'
→
23
←
Loop condition is false. Execution continues on line 1639
→
1635 atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a);
1636 *current_chunk_ctr = nth;
1637 }
1638
1639 ggml_barrier(params->threadpool);
1640
1641 for (int cur_a = 0; cur_a < n_as; ++cur_a) {
24
←
Assuming 'cur_a' is < 'n_as'
→
25
←
Loop condition is true. Entering loop body
→
1642 const int64_t cne1 = matrix_row_counts[cur_a];
1643
1644 if (cne1 == 0) {
26
←
Assuming 'cne1' is not equal to 0
→
27
←
Taking false branch
→
1645 continue;
1646 }
1647
1648 const char * src0_cur = (const char *) src0->data + cur_a * nb02;
1649 const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
28
←
Assuming 'vec_dot_type' is equal to field 'type'
→
29
←
'?' condition is true
→
1650 const size_t row_size = ggml_row_size(vec_dot_type, ne10);
1651
1652 const int64_t nr0 = ne01;
1653 const int64_t nr1 = cne1;
1654
1655 int chunk_size = 16;
1656 if (nr0 == 1 || nr1 == 1) {
30
←
Assuming 'nr0' is not equal to 1
→
31
←
Assuming 'nr1' is not equal to 1
→
32
←
Taking false branch
→
1657 chunk_size = 64;
1658 }
1659
1660 // disable for NUMA
1661 const bool_Bool disable_chunking = ggml_is_numa();
1662
1663 int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;
1664 int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;
1665
1666 if (nchunk0 * nchunk1 < nth * 4 || disable_chunking
33.1
'disable_chunking' is false
) {
33
←
Assuming the condition is false
→
34
←
Taking false branch
→
1667 nchunk0 = nr0 > nr1 ? nth : 1;
1668 nchunk1 = nr0 > nr1 ? 1 : nth;
1669 }
1670
1671 const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;
1672 const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;
1673
1674 int current_chunk = ith;
1675
1676 atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a);
1677
1678 while (current_chunk < nchunk0 * nchunk1) {
35
←
Assuming the condition is true
→
36
←
Loop condition is true. Entering loop body
→
1679 const int64_t ith0 = current_chunk % nchunk0;
1680 const int64_t ith1 = current_chunk / nchunk0;
1681
1682 const int64_t ir0_start = dr0 * ith0;
1683 const int64_t ir0_end = MIN(ir0_start + dr0, nr0)((ir0_start + dr0) < (nr0) ? (ir0_start + dr0) : (nr0));
37
←
Assuming the condition is false
→
38
←
'?' condition is false
→
1684
1685 const int64_t ir1_start = dr1 * ith1;
1686 const int64_t ir1_end = MIN(ir1_start + dr1, nr1)((ir1_start + dr1) < (nr1) ? (ir1_start + dr1) : (nr1));
39
←
Assuming the condition is false
→
40
←
'?' condition is false
→
1687
1688 ggml_compute_forward_mul_mat_id_one_chunk(
41
←
Calling 'ggml_compute_forward_mul_mat_id_one_chunk'
→
1689 dst, src0, src1, ids, cur_a,
1690 ir0_start, ir0_end, ir1_start, ir1_end,
1691 src0_cur, matrix_rows, row_size, src1_cont, wdata
1692 );
1693
1694 if (nth >= nchunk0 * nchunk1) {
1695 break;
1696 }
1697
1698 current_chunk = atomic_fetch_add_explicit__c11_atomic_fetch_add(current_chunk_ctr, 1, memory_order_relaxed);
1699 }
1700 }
1701}
1702
1703/////////////////////////////////
1704
1705static void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) {
1706 GGML_ASSERT(params)if (!(params)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 1706, "GGML_ASSERT(%s) failed", "params");
1707
1708 if (tensor->op == GGML_OP_NONE || ggml_is_empty(tensor)) {
1709 return;
1710 }
1711
1712 // extra_buffer op?
1713 if (ggml_cpu_extra_compute_forward(params, tensor)) {
1714 return;
1715 }
1716
1717 switch (tensor->op) {
1718 case GGML_OP_DUP:
1719 {
1720 ggml_compute_forward_dup(params, tensor);
1721 } break;
1722 case GGML_OP_ADD:
1723 {
1724 ggml_compute_forward_add(params, tensor);
1725 } break;
1726 case GGML_OP_ADD_ID:
1727 {
1728 ggml_compute_forward_add_id(params, tensor);
1729 } break;
1730 case GGML_OP_ADD1:
1731 {
1732 ggml_compute_forward_add1(params, tensor);
1733 } break;
1734 case GGML_OP_ACC:
1735 {
1736 ggml_compute_forward_acc(params, tensor);
1737 } break;
1738 case GGML_OP_SUB:
1739 {
1740 ggml_compute_forward_sub(params, tensor);
1741 } break;
1742 case GGML_OP_MUL:
1743 {
1744 ggml_compute_forward_mul(params, tensor);
1745 } break;
1746 case GGML_OP_DIV:
1747 {
1748 ggml_compute_forward_div(params, tensor);
1749 } break;
1750 case GGML_OP_SQR:
1751 {
1752 ggml_compute_forward_sqr(params, tensor);
1753 } break;
1754 case GGML_OP_SQRT:
1755 {
1756 ggml_compute_forward_sqrt(params, tensor);
1757 } break;
1758 case GGML_OP_LOG:
1759 {
1760 ggml_compute_forward_log(params, tensor);
1761 } break;
1762 case GGML_OP_SIN:
1763 {
1764 ggml_compute_forward_sin(params, tensor);
1765 } break;
1766 case GGML_OP_COS:
1767 {
1768 ggml_compute_forward_cos(params, tensor);
1769 } break;
1770 case GGML_OP_SUM:
1771 {
1772 ggml_compute_forward_sum(params, tensor);
1773 } break;
1774 case GGML_OP_SUM_ROWS:
1775 {
1776 ggml_compute_forward_sum_rows(params, tensor);
1777 } break;
1778 case GGML_OP_CUMSUM:
1779 {
1780 ggml_compute_forward_cumsum(params, tensor);
1781 } break;
1782 case GGML_OP_MEAN:
1783 {
1784 ggml_compute_forward_mean(params, tensor);
1785 } break;
1786 case GGML_OP_ARGMAX:
1787 {
1788 ggml_compute_forward_argmax(params, tensor);
1789 } break;
1790 case GGML_OP_COUNT_EQUAL:
1791 {
1792 ggml_compute_forward_count_equal(params, tensor);
1793 } break;
1794 case GGML_OP_REPEAT:
1795 {
1796 ggml_compute_forward_repeat(params, tensor);
1797 } break;
1798 case GGML_OP_REPEAT_BACK:
1799 {
1800 ggml_compute_forward_repeat_back(params, tensor);
1801 } break;
1802 case GGML_OP_CONCAT:
1803 {
1804 ggml_compute_forward_concat(params, tensor);
1805 } break;
1806 case GGML_OP_SILU_BACK:
1807 {
1808 ggml_compute_forward_silu_back(params, tensor);
1809 } break;
1810 case GGML_OP_NORM:
1811 {
1812 ggml_compute_forward_norm(params, tensor);
1813 } break;
1814 case GGML_OP_RMS_NORM:
1815 {
1816 ggml_compute_forward_rms_norm(params, tensor);
1817 } break;
1818 case GGML_OP_RMS_NORM_BACK:
1819 {
1820 ggml_compute_forward_rms_norm_back(params, tensor);
1821 } break;
1822 case GGML_OP_GROUP_NORM:
1823 {
1824 ggml_compute_forward_group_norm(params, tensor);
1825 } break;
1826 case GGML_OP_L2_NORM:
1827 {
1828 ggml_compute_forward_l2_norm(params, tensor);
1829 } break;
1830 case GGML_OP_MUL_MAT:
1831 {
1832 ggml_compute_forward_mul_mat(params, tensor);
1833 } break;
1834 case GGML_OP_MUL_MAT_ID:
1835 {
1836 ggml_compute_forward_mul_mat_id(params, tensor);
1837 } break;
1838 case GGML_OP_OUT_PROD:
1839 {
1840 ggml_compute_forward_out_prod(params, tensor);
1841 } break;
1842 case GGML_OP_SCALE:
1843 {
1844 ggml_compute_forward_scale(params, tensor);
1845 } break;
1846 case GGML_OP_SET:
1847 {
1848 ggml_compute_forward_set(params, tensor);
1849 } break;
1850 case GGML_OP_CPY:
1851 {
1852 ggml_compute_forward_cpy(params, tensor);
1853 } break;
1854 case GGML_OP_CONT:
1855 {
1856 ggml_compute_forward_cont(params, tensor);
1857 } break;
1858 case GGML_OP_GET_ROWS:
1859 {
1860 ggml_compute_forward_get_rows(params, tensor);
1861 } break;
1862 case GGML_OP_GET_ROWS_BACK:
1863 {
1864 ggml_compute_forward_get_rows_back(params, tensor);
1865 } break;
1866 case GGML_OP_SET_ROWS:
1867 {
1868 ggml_compute_forward_set_rows(params, tensor);
1869 } break;
1870 case GGML_OP_DIAG:
1871 {
1872 ggml_compute_forward_diag(params, tensor);
1873 } break;
1874 case GGML_OP_DIAG_MASK_INF:
1875 {
1876 ggml_compute_forward_diag_mask_inf(params, tensor);
1877 } break;
1878 case GGML_OP_DIAG_MASK_ZERO:
1879 {
1880 ggml_compute_forward_diag_mask_zero(params, tensor);
1881 } break;
1882 case GGML_OP_SOFT_MAX:
1883 {
1884 ggml_compute_forward_soft_max(params, tensor);
1885 } break;
1886 case GGML_OP_SOFT_MAX_BACK:
1887 {
1888 ggml_compute_forward_soft_max_ext_back(params, tensor);
1889 } break;
1890 case GGML_OP_ROPE:
1891 {
1892 ggml_compute_forward_rope(params, tensor);
1893 } break;
1894 case GGML_OP_ROPE_BACK:
1895 {
1896 ggml_compute_forward_rope_back(params, tensor);
1897 } break;
1898 case GGML_OP_CLAMP:
1899 {
1900 ggml_compute_forward_clamp(params, tensor);
1901 } break;
1902 case GGML_OP_CONV_TRANSPOSE_1D:
1903 {
1904 ggml_compute_forward_conv_transpose_1d(params, tensor);
1905 } break;
1906 case GGML_OP_IM2COL:
1907 {
1908 ggml_compute_forward_im2col(params, tensor);
1909 } break;
1910 case GGML_OP_IM2COL_BACK:
1911 {
1912 ggml_compute_forward_im2col_back_f32(params, tensor);
1913 } break;
1914 case GGML_OP_IM2COL_3D:
1915 {
1916 ggml_compute_forward_im2col_3d(params, tensor);
1917 } break;
1918 case GGML_OP_COL2IM_1D:
1919 {
1920 ggml_compute_forward_col2im_1d(params, tensor);
1921 } break;
1922 case GGML_OP_CONV_2D:
1923 {
1924 ggml_compute_forward_conv_2d(params, tensor);
1925 } break;
1926 case GGML_OP_CONV_3D:
1927 {
1928 ggml_compute_forward_conv_3d(params, tensor);
1929 } break;
1930 case GGML_OP_CONV_2D_DW:
1931 {
1932 ggml_compute_forward_conv_2d_dw(params, tensor);
1933 } break;
1934 case GGML_OP_CONV_TRANSPOSE_2D:
1935 {
1936 ggml_compute_forward_conv_transpose_2d(params, tensor);
1937 } break;
1938 case GGML_OP_POOL_1D:
1939 {
1940 ggml_compute_forward_pool_1d(params, tensor);
1941 } break;
1942 case GGML_OP_POOL_2D:
1943 {
1944 ggml_compute_forward_pool_2d(params, tensor);
1945 } break;
1946 case GGML_OP_POOL_2D_BACK:
1947 {
1948 ggml_compute_forward_pool_2d_back(params, tensor);
1949 } break;
1950 case GGML_OP_UPSCALE:
1951 {
1952 ggml_compute_forward_upscale(params, tensor);
1953 } break;
1954 case GGML_OP_PAD:
1955 {
1956 ggml_compute_forward_pad(params, tensor);
1957 } break;
1958 case GGML_OP_PAD_REFLECT_1D:
1959 {
1960 ggml_compute_forward_pad_reflect_1d(params, tensor);
1961 } break;
1962 case GGML_OP_ROLL:
1963 {
1964 ggml_compute_forward_roll(params, tensor);
1965 } break;
1966 case GGML_OP_ARANGE:
1967 {
1968 ggml_compute_forward_arange(params, tensor);
1969 } break;
1970 case GGML_OP_TIMESTEP_EMBEDDING:
1971 {
1972 ggml_compute_forward_timestep_embedding(params, tensor);
1973 } break;
1974 case GGML_OP_ARGSORT:
1975 {
1976 ggml_compute_forward_argsort(params, tensor);
1977 } break;
1978 case GGML_OP_TOP_K:
1979 {
1980 ggml_compute_forward_top_k(params, tensor);
1981 } break;
1982 case GGML_OP_LEAKY_RELU:
1983 {
1984 ggml_compute_forward_leaky_relu(params, tensor);
1985 } break;
1986 case GGML_OP_TRI:
1987 {
1988 ggml_compute_forward_tri(params, tensor);
1989 } break;
1990 case GGML_OP_FILL:
1991 {
1992 ggml_compute_forward_fill(params, tensor);
1993 } break;
1994 case GGML_OP_FLASH_ATTN_EXT:
1995 {
1996 ggml_compute_forward_flash_attn_ext(params, tensor);
1997 } break;
1998 case GGML_OP_FLASH_ATTN_BACK:
1999 {
2000 int32_t t = ggml_get_op_params_i32(tensor, 0);
2001 GGML_ASSERT(t == 0 || t == 1)if (!(t == 0 || t == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2001, "GGML_ASSERT(%s) failed", "t == 0 || t == 1");
2002 bool_Bool masked = t != 0;
2003 ggml_compute_forward_flash_attn_back(params, masked, tensor);
2004 } break;
2005 case GGML_OP_SSM_CONV:
2006 {
2007 ggml_compute_forward_ssm_conv(params, tensor);
2008 } break;
2009 case GGML_OP_SSM_SCAN:
2010 {
2011 ggml_compute_forward_ssm_scan(params, tensor);
2012 } break;
2013 case GGML_OP_WIN_PART:
2014 {
2015 ggml_compute_forward_win_part(params, tensor);
2016 } break;
2017 case GGML_OP_WIN_UNPART:
2018 {
2019 ggml_compute_forward_win_unpart(params, tensor);
2020 } break;
2021 case GGML_OP_UNARY:
2022 {
2023 ggml_compute_forward_unary(params, tensor);
2024 } break;
2025 case GGML_OP_GLU:
2026 {
2027 ggml_compute_forward_glu(params, tensor);
2028 } break;
2029 case GGML_OP_GET_REL_POS:
2030 {
2031 ggml_compute_forward_get_rel_pos(params, tensor);
2032 } break;
2033 case GGML_OP_ADD_REL_POS:
2034 {
2035 ggml_compute_forward_add_rel_pos(params, tensor);
2036 } break;
2037 case GGML_OP_RWKV_WKV6:
2038 {
2039 ggml_compute_forward_rwkv_wkv6(params, tensor);
2040 } break;
2041 case GGML_OP_GATED_LINEAR_ATTN:
2042 {
2043 ggml_compute_forward_gla(params, tensor);
2044 } break;
2045 case GGML_OP_RWKV_WKV7:
2046 {
2047 ggml_compute_forward_rwkv_wkv7(params, tensor);
2048 } break;
2049 case GGML_OP_SOLVE_TRI:
2050 {
2051 ggml_compute_forward_solve_tri(params, tensor);
2052 } break;
2053 case GGML_OP_GATED_DELTA_NET:
2054 {
2055 ggml_compute_forward_gated_delta_net(params, tensor);
2056 } break;
2057 case GGML_OP_MAP_CUSTOM1:
2058 {
2059 ggml_compute_forward_map_custom1(params, tensor);
2060 }
2061 break;
2062 case GGML_OP_MAP_CUSTOM2:
2063 {
2064 ggml_compute_forward_map_custom2(params, tensor);
2065 }
2066 break;
2067 case GGML_OP_MAP_CUSTOM3:
2068 {
2069 ggml_compute_forward_map_custom3(params, tensor);
2070 }
2071 break;
2072 case GGML_OP_CUSTOM:
2073 {
2074 ggml_compute_forward_custom(params, tensor);
2075 }
2076 break;
2077 case GGML_OP_CROSS_ENTROPY_LOSS:
2078 {
2079 ggml_compute_forward_cross_entropy_loss(params, tensor);
2080 }
2081 break;
2082 case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
2083 {
2084 ggml_compute_forward_cross_entropy_loss_back(params, tensor);
2085 }
2086 break;
2087 case GGML_OP_OPT_STEP_ADAMW:
2088 {
2089 ggml_compute_forward_opt_step_adamw(params, tensor);
2090 }
2091 break;
2092 case GGML_OP_OPT_STEP_SGD:
2093 {
2094 ggml_compute_forward_opt_step_sgd(params, tensor);
2095 }
2096 break;
2097 case GGML_OP_NONE:
2098 {
2099 // nop
2100 } break;
2101 case GGML_OP_RESHAPE:
2102 {
2103 // nop
2104 } break;
2105 case GGML_OP_PERMUTE:
2106 {
2107 // nop
2108 } break;
2109 case GGML_OP_VIEW:
2110 {
2111 // nop
2112 } break;
2113 case GGML_OP_TRANSPOSE:
2114 {
2115 // nop
2116 } break;
2117 case GGML_OP_COUNT:
2118 {
2119 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2119, "fatal error");
2120 }
2121 }
2122}
2123
2124// Android's libc implementation "bionic" does not support setting affinity
2125#if defined(__gnu_linux__1)
2126static void set_numa_thread_affinity(int thread_n) {
2127 if (!ggml_is_numa()) {
2128 return;
2129 }
2130
2131 int node_num;
2132 int rv;
2133 size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus)((((g_state.numa.total_cpus) + (8 * sizeof (__cpu_mask)) - 1)
/ (8 * sizeof (__cpu_mask))) * sizeof (__cpu_mask));
2134
2135 switch(g_state.numa.numa_strategy) {
2136 case GGML_NUMA_STRATEGY_DISTRIBUTE:
2137 // run thread on node_num thread_n / (threads per node)
2138 node_num = thread_n % g_state.numa.n_nodes;
2139 break;
2140 case GGML_NUMA_STRATEGY_ISOLATE:
2141 // run thread on current_node
2142 node_num = g_state.numa.current_node;
2143 break;
2144 case GGML_NUMA_STRATEGY_NUMACTL:
2145 // use the cpuset that numactl gave us
2146 rv = pthread_setaffinity_np(pthread_self(), setsize, &g_state.numa.cpuset);
2147 if (rv) {
2148 fprintf(stderrstderr, "warning: pthread_setaffinity_np() failed: %s\n",strerror(rv));
2149 }
2150 return;
2151 default:
2152 return;
2153 }
2154
2155 struct ggml_numa_node * node = &g_state.numa.nodes[node_num];
2156
2157 cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus)__sched_cpualloc (g_state.numa.total_cpus);
2158 CPU_ZERO_S(setsize, cpus)do __builtin_memset (cpus, '\0', setsize); while (0);
2159 for (size_t i = 0; i < node->n_cpus; ++i) {
2160 CPU_SET_S(node->cpus[i], setsize, cpus)(__extension__ ({ size_t __cpu = (node->cpus[i]); __cpu / 8
< (setsize) ? (((__cpu_mask *) ((cpus)->__bits))[((__cpu
) / (8 * sizeof (__cpu_mask)))] |= ((__cpu_mask) 1 << (
(__cpu) % (8 * sizeof (__cpu_mask))))) : 0; }));
2161 }
2162
2163 rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);
2164 if (rv) {
2165 fprintf(stderrstderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv));
2166 }
2167
2168 CPU_FREE(cpus)__sched_cpufree (cpus);
2169}
2170
2171static void clear_numa_thread_affinity(void) {
2172 if (!ggml_is_numa()) {
2173 return;
2174 }
2175
2176 size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus)((((g_state.numa.total_cpus) + (8 * sizeof (__cpu_mask)) - 1)
/ (8 * sizeof (__cpu_mask))) * sizeof (__cpu_mask));
2177
2178 cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus)__sched_cpualloc (g_state.numa.total_cpus);
2179 CPU_ZERO_S(setsize, cpus)do __builtin_memset (cpus, '\0', setsize); while (0);
2180 for (unsigned i = 0; i < g_state.numa.total_cpus; ++i) {
2181 CPU_SET_S(i, setsize, cpus)(__extension__ ({ size_t __cpu = (i); __cpu / 8 < (setsize
) ? (((__cpu_mask *) ((cpus)->__bits))[((__cpu) / (8 * sizeof
(__cpu_mask)))] |= ((__cpu_mask) 1 << ((__cpu) % (8 * sizeof
(__cpu_mask))))) : 0; }));
2182 }
2183
2184 int rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);
2185 if (rv) {
2186 fprintf(stderrstderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv));
2187 }
2188
2189 CPU_FREE(cpus)__sched_cpufree (cpus);
2190}
2191#else
2192// TODO: Windows etc.
2193// (the linux implementation may also work on BSD, someone should test)
2194static void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n)(void)(thread_n); }
2195static void clear_numa_thread_affinity(void) {}
2196#endif
2197
2198static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
2199 int n_tasks = 0;
2200
2201 if (ggml_is_empty(node)) {
2202 // no need to multi-thread a no-op
2203 n_tasks = 1;
2204 return n_tasks;
2205 }
2206
2207 switch (node->op) {
2208 case GGML_OP_CPY:
2209 case GGML_OP_DUP:
2210 case GGML_OP_CONT:
2211 case GGML_OP_ADD:
2212 case GGML_OP_ADD_ID:
2213 case GGML_OP_ADD1:
2214 case GGML_OP_ACC:
2215 case GGML_OP_CUMSUM:
2216 case GGML_OP_TRI:
2217 case GGML_OP_FILL:
2218 {
2219 n_tasks = n_threads;
2220 } break;
2221 case GGML_OP_SUB:
2222 case GGML_OP_SQR:
2223 case GGML_OP_SQRT:
2224 case GGML_OP_LOG:
2225 case GGML_OP_SIN:
2226 case GGML_OP_COS:
2227 case GGML_OP_SUM:
2228 case GGML_OP_SUM_ROWS:
2229 case GGML_OP_MEAN:
2230 case GGML_OP_ARGMAX:
2231 {
2232 n_tasks = 1;
2233 } break;
2234 case GGML_OP_COUNT_EQUAL:
2235 case GGML_OP_SOLVE_TRI:
2236 case GGML_OP_GATED_DELTA_NET:
2237 {
2238 n_tasks = n_threads;
2239 } break;
2240 case GGML_OP_REPEAT:
2241 case GGML_OP_REPEAT_BACK:
2242 case GGML_OP_LEAKY_RELU:
2243 {
2244 n_tasks = 1;
2245 } break;
2246 case GGML_OP_UNARY:
2247 switch (ggml_get_unary_op(node)) {
2248 case GGML_UNARY_OP_ABS:
2249 case GGML_UNARY_OP_SGN:
2250 case GGML_UNARY_OP_NEG:
2251 case GGML_UNARY_OP_STEP:
2252 case GGML_UNARY_OP_TANH:
2253 case GGML_UNARY_OP_ELU:
2254 case GGML_UNARY_OP_RELU:
2255 case GGML_UNARY_OP_SIGMOID:
2256 case GGML_UNARY_OP_HARDSWISH:
2257 case GGML_UNARY_OP_HARDSIGMOID:
2258 case GGML_UNARY_OP_EXP:
2259 case GGML_UNARY_OP_SOFTPLUS:
2260 case GGML_UNARY_OP_EXPM1:
2261 case GGML_UNARY_OP_FLOOR:
2262 case GGML_UNARY_OP_CEIL:
2263 case GGML_UNARY_OP_ROUND:
2264 case GGML_UNARY_OP_TRUNC:
2265 {
2266 n_tasks = 1;
2267 } break;
2268
2269 case GGML_UNARY_OP_GELU:
2270 case GGML_UNARY_OP_GELU_ERF:
2271 case GGML_UNARY_OP_GELU_QUICK:
2272 case GGML_UNARY_OP_SILU:
2273 case GGML_UNARY_OP_XIELU:
2274 {
2275 n_tasks = n_threads;
2276 } break;
2277 default:
2278 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2278, "fatal error");
2279 }
2280 break;
2281 case GGML_OP_GLU:
2282 switch (ggml_get_glu_op(node)) {
2283 case GGML_GLU_OP_REGLU:
2284 case GGML_GLU_OP_GEGLU:
2285 case GGML_GLU_OP_SWIGLU:
2286 case GGML_GLU_OP_SWIGLU_OAI:
2287 case GGML_GLU_OP_GEGLU_ERF:
2288 case GGML_GLU_OP_GEGLU_QUICK:
2289 {
2290 n_tasks = n_threads;
2291 } break;
2292 default:
2293 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2293, "fatal error");
2294 }
2295 break;
2296 case GGML_OP_SILU_BACK:
2297 case GGML_OP_MUL:
2298 case GGML_OP_DIV:
2299 case GGML_OP_NORM:
2300 case GGML_OP_RMS_NORM:
2301 case GGML_OP_RMS_NORM_BACK:
2302 case GGML_OP_L2_NORM:
2303 case GGML_OP_GROUP_NORM:
2304 case GGML_OP_CONCAT:
2305 case GGML_OP_MUL_MAT:
2306 case GGML_OP_MUL_MAT_ID:
2307 case GGML_OP_OUT_PROD:
2308 {
2309 n_tasks = n_threads;
2310 } break;
2311 case GGML_OP_GET_ROWS:
2312 case GGML_OP_SET_ROWS:
2313 {
2314 // FIXME: get_rows can use additional threads, but the cost of launching additional threads
2315 // decreases performance with GPU offloading
2316 //n_tasks = n_threads;
2317 n_tasks = 1;
2318 } break;
2319 case GGML_OP_SCALE:
2320 case GGML_OP_SET:
2321 case GGML_OP_RESHAPE:
2322 case GGML_OP_VIEW:
2323 case GGML_OP_PERMUTE:
2324 case GGML_OP_TRANSPOSE:
2325 case GGML_OP_GET_ROWS_BACK:
2326 case GGML_OP_DIAG:
2327 {
2328 n_tasks = 1;
2329 } break;
2330 case GGML_OP_DIAG_MASK_ZERO:
2331 case GGML_OP_DIAG_MASK_INF:
2332 case GGML_OP_SOFT_MAX_BACK:
2333 case GGML_OP_ROPE:
2334 case GGML_OP_ROPE_BACK:
2335 case GGML_OP_ADD_REL_POS:
2336 {
2337 n_tasks = n_threads;
2338 } break;
2339 case GGML_OP_CLAMP:
2340 {
2341 n_tasks = 1; //TODO
2342 } break;
2343 case GGML_OP_SOFT_MAX:
2344 {
2345 n_tasks = MIN(n_threads, ggml_nrows(node->src[0]))((n_threads) < (ggml_nrows(node->src[0])) ? (n_threads)
: (ggml_nrows(node->src[0])));
2346 } break;
2347 case GGML_OP_IM2COL:
2348 case GGML_OP_IM2COL_BACK:
2349 case GGML_OP_IM2COL_3D:
2350 case GGML_OP_CONV_2D:
2351 case GGML_OP_CONV_3D:
2352 case GGML_OP_CONV_2D_DW:
2353 case GGML_OP_COL2IM_1D:
2354 case GGML_OP_CONV_TRANSPOSE_1D:
2355 case GGML_OP_CONV_TRANSPOSE_2D:
2356 {
2357 n_tasks = n_threads;
2358 } break;
2359 case GGML_OP_POOL_1D:
2360 case GGML_OP_POOL_2D:
2361 case GGML_OP_POOL_2D_BACK:
2362 {
2363 n_tasks = 1;
2364 } break;
2365 case GGML_OP_UPSCALE:
2366 case GGML_OP_PAD:
2367 case GGML_OP_PAD_REFLECT_1D:
2368 case GGML_OP_ROLL:
2369 case GGML_OP_ARANGE:
2370 case GGML_OP_TIMESTEP_EMBEDDING:
2371 case GGML_OP_ARGSORT:
2372 case GGML_OP_TOP_K:
2373 case GGML_OP_FLASH_ATTN_EXT:
2374 case GGML_OP_FLASH_ATTN_BACK:
2375 case GGML_OP_SSM_CONV:
2376 case GGML_OP_SSM_SCAN:
2377 {
2378 n_tasks = n_threads;
2379 } break;
2380 case GGML_OP_RWKV_WKV6:
2381 case GGML_OP_GATED_LINEAR_ATTN:
2382 case GGML_OP_RWKV_WKV7:
2383 {
2384 const int64_t n_heads = node->src[1]->ne[1];
2385 n_tasks = MIN(n_threads, n_heads)((n_threads) < (n_heads) ? (n_threads) : (n_heads));
2386 } break;
2387 case GGML_OP_WIN_PART:
2388 case GGML_OP_WIN_UNPART:
2389 case GGML_OP_GET_REL_POS:
2390 {
2391 n_tasks = 1;
2392 } break;
2393 case GGML_OP_MAP_CUSTOM1:
2394 {
2395 struct ggml_map_custom1_op_params p;
2396 memcpy(&p, node->op_params, sizeof(p));
2397 if (p.n_tasks == GGML_N_TASKS_MAX(-1)) {
2398 n_tasks = n_threads;
2399 } else {
2400 n_tasks = MIN(p.n_tasks, n_threads)((p.n_tasks) < (n_threads) ? (p.n_tasks) : (n_threads));
2401 }
2402 } break;
2403 case GGML_OP_MAP_CUSTOM2:
2404 {
2405 struct ggml_map_custom2_op_params p;
2406 memcpy(&p, node->op_params, sizeof(p));
2407 if (p.n_tasks == GGML_N_TASKS_MAX(-1)) {
2408 n_tasks = n_threads;
2409 } else {
2410 n_tasks = MIN(p.n_tasks, n_threads)((p.n_tasks) < (n_threads) ? (p.n_tasks) : (n_threads));
2411 }
2412 } break;
2413 case GGML_OP_MAP_CUSTOM3:
2414 {
2415 struct ggml_map_custom3_op_params p;
2416 memcpy(&p, node->op_params, sizeof(p));
2417 if (p.n_tasks == GGML_N_TASKS_MAX(-1)) {
2418 n_tasks = n_threads;
2419 } else {
2420 n_tasks = MIN(p.n_tasks, n_threads)((p.n_tasks) < (n_threads) ? (p.n_tasks) : (n_threads));
2421 }
2422 } break;
2423 case GGML_OP_CUSTOM:
2424 {
2425 struct ggml_custom_op_params p;
2426 memcpy(&p, node->op_params, sizeof(p));
2427 if (p.n_tasks == GGML_N_TASKS_MAX(-1)) {
2428 n_tasks = n_threads;
2429 } else {
2430 n_tasks = MIN(p.n_tasks, n_threads)((p.n_tasks) < (n_threads) ? (p.n_tasks) : (n_threads));
2431 }
2432 } break;
2433 case GGML_OP_CROSS_ENTROPY_LOSS:
2434 case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
2435 case GGML_OP_OPT_STEP_ADAMW:
2436 case GGML_OP_OPT_STEP_SGD:
2437 {
2438 n_tasks = n_threads;
2439 } break;
2440 case GGML_OP_NONE:
2441 {
2442 n_tasks = 1;
2443 } break;
2444 case GGML_OP_COUNT:
2445 {
2446 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2446, "fatal error");
2447 }
2448 default:
2449 {
2450 fprintf(stderrstderr, "%s: op not implemented: ", __func__);
2451 if (node->op < GGML_OP_COUNT) {
2452 fprintf(stderrstderr, "%s\n", ggml_op_name(node->op));
2453 } else {
2454 fprintf(stderrstderr, "%d\n", node->op);
2455 }
2456 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2456, "fatal error");
2457 }
2458 }
2459
2460 assert(n_tasks > 0)((void) sizeof (__assert_single_arg (n_tasks > 0)), __extension__
({ if (n_tasks > 0) ; else __assert_fail ("n_tasks > 0"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2460, __extension__ __PRETTY_FUNCTION__); }));
2461
2462 return n_tasks;
2463}
2464
2465static thread_ret_t ggml_graph_compute_secondary_thread(void* data);
2466
2467#if defined(_WIN32)
2468#include "windows.h"
2469
2470// TODO: support > 64 CPUs
2471static bool_Bool ggml_thread_apply_affinity(bool_Bool * mask) {
2472 HANDLE h = GetCurrentThread();
2473 uint64_t bitmask = 0ULL;
2474
2475 assert(GGML_MAX_N_THREADS >= 64)((void) sizeof (__assert_single_arg (512 >= 64)), __extension__
({ if (512 >= 64) ; else __assert_fail ("GGML_MAX_N_THREADS >= 64"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2475, __extension__ __PRETTY_FUNCTION__); }));
2476
2477 for (int32_t i = 0; i < 8; i++) {
2478 int32_t idx = i * 8;
2479 uint8_t val = 0;
2480 val |= mask[idx + 0] << 0;
2481 val |= mask[idx + 1] << 1;
2482 val |= mask[idx + 2] << 2;
2483 val |= mask[idx + 3] << 3;
2484 val |= mask[idx + 4] << 4;
2485 val |= mask[idx + 5] << 5;
2486 val |= mask[idx + 6] << 6;
2487 val |= mask[idx + 7] << 7;
2488 bitmask |= (uint64_t)val << idx;
2489 }
2490
2491 for (int32_t i = 64; i < GGML_MAX_N_THREADS512; i++) {
2492 if (mask[i]) {
2493 fprintf(stderrstderr, "warn: setting thread-affinity for > 64 CPUs isn't supported on windows!\n");
2494 break;
2495 }
2496 }
2497
2498 DWORD_PTR m = (DWORD_PTR)bitmask;
2499
2500 m = SetThreadAffinityMask(h, m);
2501
2502 return m != 0;
2503}
2504
2505static bool_Bool ggml_thread_apply_priority(int32_t prio) {
2506 // Note that on Windows the Process Priority Class must be updated in order to set Thread priority.
2507 // This is up to the applications.
2508 DWORD p = THREAD_PRIORITY_NORMAL;
2509 switch (prio) {
2510 case GGML_SCHED_PRIO_LOW: p = THREAD_PRIORITY_BELOW_NORMAL; break;
2511 case GGML_SCHED_PRIO_NORMAL: p = THREAD_PRIORITY_NORMAL; break;
2512 case GGML_SCHED_PRIO_MEDIUM: p = THREAD_PRIORITY_ABOVE_NORMAL; break;
2513 case GGML_SCHED_PRIO_HIGH: p = THREAD_PRIORITY_HIGHEST; break;
2514 case GGML_SCHED_PRIO_REALTIME: p = THREAD_PRIORITY_TIME_CRITICAL; break;
2515 }
2516
2517 if (prio != GGML_SCHED_PRIO_LOW) {
2518 // Tell Windows that this thread should not be throttled (needs its own CPU core).
2519 // Newer Windows 11 versions aggressively park (offline) CPU cores and often place
2520 // all our threads onto the first 4 cores which results in terrible performance with
2521 // n_threads > 4
2522 #if _WIN32_WINNT >= 0x0602
2523 THREAD_POWER_THROTTLING_STATE t;
2524 ZeroMemory(&t, sizeof(t));
2525 t.Version = THREAD_POWER_THROTTLING_CURRENT_VERSION;
2526 t.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED;
2527 t.StateMask = 0;
2528
2529 if (!SetThreadInformation(GetCurrentThread(), ThreadPowerThrottling, &t, sizeof(t))) {
2530 GGML_LOG_DEBUG("failed to disable thread power throttling %d : (%d)\n", prio, (int) GetLastError())ggml_log_internal(GGML_LOG_LEVEL_DEBUG, "failed to disable thread power throttling %d : (%d)\n"
, prio, (int) GetLastError());
2531 return false0;
2532 }
2533 #endif
2534 }
2535
2536 if (prio == GGML_SCHED_PRIO_NORMAL) {
2537 // Keep inherited policy/priority
2538 return true1;
2539 }
2540
2541 if (!SetThreadPriority(GetCurrentThread(), p)) {
2542 fprintf(stderrstderr, "warn: failed to set thread priority %d : (%d)\n", prio, (int) GetLastError());
2543 return false0;
2544 }
2545
2546 return true1;
2547}
2548
2549#elif defined(__APPLE__)
2550#include <sys/types.h>
2551#include <sys/resource.h>
2552
2553static bool_Bool ggml_thread_apply_affinity(const bool_Bool * mask) {
2554 // Not supported on Apple platforms
2555 UNUSED(mask)(void)(mask);
2556 return true1;
2557}
2558
2559static bool_Bool ggml_thread_apply_priority(int32_t prio) {
2560 struct sched_param p;
2561 int32_t policy = SCHED_OTHER0;
2562 switch (prio) {
2563 // TODO: there seems to be no way to set lower prio on Apple platforms
2564 case GGML_SCHED_PRIO_LOW: policy = SCHED_OTHER0; p.sched_prioritysched_priority = 0; break;
2565 case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER0; p.sched_prioritysched_priority = 0; break;
2566 case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 40; break;
2567 case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 80; break;
2568 case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 90; break;
2569 }
2570
2571 if (prio == GGML_SCHED_PRIO_NORMAL) {
2572 // Keep inherited policy/priority
2573 return true1;
2574 }
2575
2576 int32_t err = pthread_setschedparam(pthread_self(), policy, &p);
2577 if (err != 0) {
2578 fprintf(stderrstderr, "warn: failed to set thread priority %d : %s (%d)\n", prio, strerror(err), err);
2579 return false0;
2580 }
2581
2582 return true1;
2583}
2584
2585#elif defined(__gnu_linux__1)
2586// TODO: this may not work on BSD, to be verified
2587
2588static bool_Bool ggml_thread_apply_affinity(const bool_Bool * mask) {
2589 cpu_set_t cpuset;
2590 int err;
2591
2592 CPU_ZERO(&cpuset)do __builtin_memset (&cpuset, '\0', sizeof (cpu_set_t)); while
(0);
2593
2594 for (uint32_t i = 0; i < GGML_MAX_N_THREADS512; i++) {
2595 if (mask[i]) {
2596 GGML_PRINT_DEBUG("Thread %lx: adding %d to cpuset\n", pthread_self(), i);
2597 CPU_SET(i, &cpuset)(__extension__ ({ size_t __cpu = (i); __cpu / 8 < (sizeof (
cpu_set_t)) ? (((__cpu_mask *) ((&cpuset)->__bits))[((
__cpu) / (8 * sizeof (__cpu_mask)))] |= ((__cpu_mask) 1 <<
((__cpu) % (8 * sizeof (__cpu_mask))))) : 0; }));
2598 }
2599 }
2600
2601#ifdef __ANDROID__
2602 err = sched_setaffinity(0, sizeof(cpuset), &cpuset);
2603 if (err < 0) {
2604 err = errno(*__errno_location ());
2605 }
2606#else
2607 err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
2608#endif
2609 if (err != 0) {
2610 fprintf(stderrstderr, "warn: failed to set affinity mask 0x%llx : %s (%d)\n", (unsigned long long)mask, strerror(err), err);
2611 return false0;
2612 }
2613
2614 return true1;
2615}
2616
2617static bool_Bool ggml_thread_apply_priority(int32_t prio) {
2618 struct sched_param p;
2619 int32_t policy = SCHED_OTHER0;
2620 switch (prio) {
2621 case GGML_SCHED_PRIO_LOW: policy = SCHED_BATCH3; p.sched_prioritysched_priority = 0; break;
2622 case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER0; p.sched_prioritysched_priority = 0; break;
2623 case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 40; break;
2624 case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 80; break;
2625 case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 90; break;
2626 }
2627
2628 if (prio == GGML_SCHED_PRIO_NORMAL) {
2629 // Keep inherited policy/priority
2630 return true1;
2631 }
2632
2633 int32_t err = pthread_setschedparam(pthread_self(), policy, &p);
2634 if (err != 0) {
2635 fprintf(stderrstderr, "warn: failed to set thread priority %d : %s (%d)\n", prio, strerror(err), err);
2636 return false0;
2637 }
2638
2639 return true1;
2640}
2641
2642#else // unsupported platforms
2643
2644static bool_Bool ggml_thread_apply_affinity(const bool_Bool * mask) {
2645 UNUSED(mask)(void)(mask);
2646 return true1;
2647}
2648
2649static bool_Bool ggml_thread_apply_priority(int32_t prio) {
2650 UNUSED(prio)(void)(prio);
2651 return true1;
2652}
2653
2654#endif
2655
2656static bool_Bool ggml_thread_cpumask_is_valid(const bool_Bool * mask) {
2657 for (int i = 0; i < GGML_MAX_N_THREADS512; i++) {
2658 if (mask[i]) { return true1; }
2659 }
2660 return false0;
2661}
2662
2663static void ggml_thread_cpumask_next(const bool_Bool * global_mask, bool_Bool * local_mask, bool_Bool strict, int32_t* iter) {
2664 if (!strict) {
2665 memcpy(local_mask, global_mask, GGML_MAX_N_THREADS512);
2666 return;
2667 } else {
2668 memset(local_mask, 0, GGML_MAX_N_THREADS512);
2669 int32_t base_idx = *iter;
2670 for (int32_t i = 0; i < GGML_MAX_N_THREADS512; i++) {
2671 int32_t idx = base_idx + i;
2672 if (idx >= GGML_MAX_N_THREADS512) {
2673 // Just a cheaper modulo
2674 idx -= GGML_MAX_N_THREADS512;
2675 }
2676 if (global_mask[idx]) {
2677 local_mask[idx] = 1;
2678 *iter = idx + 1;
2679 return;
2680 }
2681 }
2682 }
2683}
2684
2685void ggml_threadpool_free(struct ggml_threadpool* threadpool) {
2686 if (!threadpool) return;
2687
2688 const int n_threads = threadpool->n_threads;
2689
2690#ifndef GGML_USE_OPENMP
2691 struct ggml_compute_state* workers = threadpool->workers;
2692
2693 ggml_mutex_lock(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex);
2694
2695 threadpool->stop = true1;
2696 threadpool->pause = false0;
2697
2698 ggml_cond_broadcast(&threadpool->cond)pthread_cond_broadcast(&threadpool->cond);
2699 ggml_mutex_unlock(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex);
2700
2701 for (int j = 1; j < n_threads; j++) {
2702 int32_t rc = ggml_thread_joinpthread_join(workers[j].thrd, NULL((void*)0));
2703 GGML_ASSERT(rc == GGML_EXIT_SUCCESS || rc == GGML_EXIT_ABORTED)if (!(rc == 0 || rc == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2703, "GGML_ASSERT(%s) failed", "rc == GGML_EXIT_SUCCESS || rc == GGML_EXIT_ABORTED"
);
2704 UNUSED(rc)(void)(rc);
2705 }
2706
2707 ggml_mutex_destroy(&threadpool->mutex)pthread_mutex_destroy(&threadpool->mutex);
2708 ggml_cond_destroy(&threadpool->cond)pthread_cond_destroy(&threadpool->cond);
2709#endif // GGML_USE_OPENMP
2710
2711 const size_t workers_size = sizeof(struct ggml_compute_state) * n_threads;
2712 ggml_aligned_free(threadpool->workers, workers_size);
2713 ggml_aligned_free(threadpool, sizeof(struct ggml_threadpool));
2714}
2715
2716#ifndef GGML_USE_OPENMP
2717// pause/resume must be called under mutex
2718static void ggml_threadpool_pause_locked(struct ggml_threadpool * threadpool) {
2719 GGML_PRINT_DEBUG("Pausing threadpool\n");
2720 threadpool->pause = true1;
2721 ggml_cond_broadcast(&threadpool->cond)pthread_cond_broadcast(&threadpool->cond);
2722}
2723
2724static void ggml_threadpool_resume_locked(struct ggml_threadpool * threadpool) {
2725 GGML_PRINT_DEBUG("Resuming threadpool\n");
2726 threadpool->pause = false0;
2727 ggml_cond_broadcast(&threadpool->cond)pthread_cond_broadcast(&threadpool->cond);
2728}
2729#endif
2730
2731void ggml_threadpool_pause(struct ggml_threadpool * threadpool) {
2732#ifndef GGML_USE_OPENMP
2733 ggml_mutex_lock(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex);
2734 if (!threadpool->pause) {
2735 ggml_threadpool_pause_locked(threadpool);
2736 }
2737 ggml_mutex_unlock(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex);
2738#else
2739 UNUSED(threadpool)(void)(threadpool);
2740#endif
2741}
2742
2743void ggml_threadpool_resume(struct ggml_threadpool * threadpool) {
2744#ifndef GGML_USE_OPENMP
2745 ggml_mutex_lock(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex);
2746 if (threadpool->pause) {
2747 ggml_threadpool_resume_locked(threadpool);
2748 }
2749 ggml_mutex_unlock(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex);
2750#else
2751 UNUSED(threadpool)(void)(threadpool);
2752#endif
2753}
2754
2755struct ggml_cplan ggml_graph_plan(
2756 const struct ggml_cgraph * cgraph,
2757 int n_threads,
2758 struct ggml_threadpool * threadpool) {
2759
2760 if (threadpool == NULL((void*)0)) {
2761 //GGML_PRINT_DEBUG("Threadpool is not specified. Will create a disposable threadpool : n_threads %d\n", n_threads);
2762 }
2763 if (n_threads <= 0) {
2764 n_threads = threadpool ? threadpool->n_threads : GGML_DEFAULT_N_THREADS4;
2765 }
2766
2767#if defined(__EMSCRIPTEN__) && !defined(__EMSCRIPTEN_PTHREADS__)
2768 // Emscripten without pthreads support can only use a single thread
2769 n_threads = 1;
2770#endif
2771
2772 size_t work_size = 0;
2773
2774 struct ggml_cplan cplan;
2775 memset(&cplan, 0, sizeof(struct ggml_cplan));
2776
2777 int max_tasks = 1;
2778
2779 // thread scheduling for the different operations + work buffer size estimation
2780 for (int i = 0; i < cgraph->n_nodes; i++) {
2781 struct ggml_tensor * node = cgraph->nodes[i];
2782
2783 const int n_tasks = ggml_get_n_tasks(node, n_threads);
2784
2785 max_tasks = MAX(max_tasks, n_tasks)((max_tasks) > (n_tasks) ? (max_tasks) : (n_tasks));
2786
2787 size_t cur = 0;
2788
2789 if (!ggml_cpu_extra_work_size(n_threads, node, &cur)) {
2790 switch (node->op) {
2791 case GGML_OP_CPY:
2792 case GGML_OP_DUP:
2793 {
2794 if (ggml_is_quantized(node->type) ||
2795 // F16 -> BF16 and BF16 -> F16 copies go through intermediate F32
2796 (node->src[0]->type == GGML_TYPE_F16 && node->src[1] && node->src[1]->type == GGML_TYPE_BF16) ||
2797 (node->src[0]->type == GGML_TYPE_BF16 && node->src[1] && node->src[1]->type == GGML_TYPE_F16) ||
2798 // conversion between F32 and I32
2799 (node->src[0]->type == GGML_TYPE_F32 && node->src[1] && node->src[1]->type == GGML_TYPE_I32) ||
2800 (node->src[0]->type == GGML_TYPE_I32 && node->src[1] && node->src[1]->type == GGML_TYPE_F32)) {
2801 cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;
2802 }
2803 } break;
2804 case GGML_OP_ADD:
2805 case GGML_OP_ADD_ID:
2806 case GGML_OP_ADD1:
2807 {
2808 if (ggml_is_quantized(node->src[0]->type)) {
2809 cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
2810 }
2811 } break;
2812 case GGML_OP_ACC:
2813 {
2814 if (ggml_is_quantized(node->src[0]->type)) {
2815 cur = ggml_type_size(GGML_TYPE_F32) * node->src[1]->ne[0] * n_tasks;
2816 }
2817 } break;
2818 case GGML_OP_COUNT_EQUAL:
2819 {
2820 cur = ggml_type_size(node->type)*n_tasks;
2821 } break;
2822 case GGML_OP_MUL_MAT:
2823 {
2824 const enum ggml_type vec_dot_type = type_traits_cpu[node->src[0]->type].vec_dot_type;
2825
2826 if (node->src[1]->type != vec_dot_type) {
2827 cur = ggml_row_size(vec_dot_type, ggml_nelements(node->src[1]));
2828 }
2829 } break;
2830 case GGML_OP_MUL_MAT_ID:
2831 {
2832 cur = 0;
2833 const struct ggml_tensor * src0 = node->src[0];
2834 const struct ggml_tensor * src1 = node->src[1];
2835 const struct ggml_tensor * ids = node->src[2];
2836 const enum ggml_type vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;
2837 const int n_as = src0->ne[2];
2838 // src1
2839 if (src1->type != vec_dot_type) {
2840 cur += ggml_row_size(vec_dot_type, ggml_nelements(src1)) + sizeof(int64_t);
2841 }
2842 // matrix_row_counts
2843 cur += n_as * sizeof(int64_t) + sizeof(int64_t);
2844 // matrix_rows
2845 cur += n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping) + sizeof(int64_t);
2846 // atomic_current_chunk
2847 cur += CACHE_LINE_SIZE64*n_as + CACHE_LINE_SIZE64;
2848 } break;
2849 case GGML_OP_OUT_PROD:
2850 {
2851 if (ggml_is_quantized(node->src[0]->type)) {
2852 cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
2853 }
2854 } break;
2855 case GGML_OP_SOFT_MAX:
2856 case GGML_OP_ROPE:
2857 case GGML_OP_ROPE_BACK:
2858 {
2859 cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;
2860 } break;
2861 case GGML_OP_CONV_TRANSPOSE_1D:
2862 {
2863 GGML_ASSERT(node->src[0]->ne[3] == 1)if (!(node->src[0]->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2863, "GGML_ASSERT(%s) failed", "node->src[0]->ne[3] == 1"
);
2864 GGML_ASSERT(node->src[1]->ne[2] == 1)if (!(node->src[1]->ne[2] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2864, "GGML_ASSERT(%s) failed", "node->src[1]->ne[2] == 1"
);
2865 GGML_ASSERT(node->src[1]->ne[3] == 1)if (!(node->src[1]->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2865, "GGML_ASSERT(%s) failed", "node->src[1]->ne[3] == 1"
);
2866
2867 const int64_t ne00 = node->src[0]->ne[0]; // K
2868 const int64_t ne01 = node->src[0]->ne[1]; // Cout
2869 const int64_t ne02 = node->src[0]->ne[2]; // Cin
2870 const int64_t ne10 = node->src[1]->ne[0]; // L
2871 const int64_t ne11 = node->src[1]->ne[1]; // Cin
2872
2873 if ((node->src[0]->type == GGML_TYPE_F16 ||
2874 node->src[0]->type == GGML_TYPE_BF16) &&
2875 node->src[1]->type == GGML_TYPE_F32) {
2876 cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02;
2877 cur += sizeof(ggml_fp16_t)*ne10*ne11;
2878 } else if (node->src[0]->type == GGML_TYPE_F32 &&
2879 node->src[1]->type == GGML_TYPE_F32) {
2880 cur += sizeof(float)*ne00*ne01*ne02;
2881 cur += sizeof(float)*ne10*ne11;
2882 } else {
2883 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2883, "fatal error");
2884 }
2885 } break;
2886 case GGML_OP_CONV_2D:
2887 case GGML_OP_CONV_3D:
2888 {
2889 cur = GGML_IM2COL_WORK_SIZE(16 * 1024 * 1024);
2890 } break;
2891 case GGML_OP_CONV_TRANSPOSE_2D:
2892 {
2893 const int64_t ne00 = node->src[0]->ne[0]; // W
2894 const int64_t ne01 = node->src[0]->ne[1]; // H
2895 const int64_t ne02 = node->src[0]->ne[2]; // Channels Out
2896 const int64_t ne03 = node->src[0]->ne[3]; // Channels In
2897
2898 const int64_t ne10 = node->src[1]->ne[0]; // W
2899 const int64_t ne11 = node->src[1]->ne[1]; // H
2900 const int64_t ne12 = node->src[1]->ne[2]; // Channels In
2901
2902 GGML_ASSERT(node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32)if (!(node->src[0]->type == GGML_TYPE_F16 || node->src
[0]->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2902, "GGML_ASSERT(%s) failed", "node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32"
);
2903 GGML_ASSERT(node->src[1]->type == GGML_TYPE_F32)if (!(node->src[1]->type == GGML_TYPE_F32)) ggml_abort(
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2903, "GGML_ASSERT(%s) failed", "node->src[1]->type == GGML_TYPE_F32"
);
2904
2905 cur += ggml_type_size(node->src[0]->type) * ne00 * ne01 * ne02 * ne03;
2906 cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12;
2907
2908 } break;
2909 case GGML_OP_TOP_K:
2910 {
2911 cur += sizeof(int32_t)*node->src[0]->ne[0]*n_tasks;
2912 } break;
2913 case GGML_OP_FLASH_ATTN_EXT:
2914 {
2915 const int64_t neq2 = node->src[0]->ne[2]; // number of query heads
2916 const int64_t DK = node->src[1]->ne[0];
2917 const int64_t DV = node->src[2]->ne[0];
2918
2919 // Tiled flash attention scratch (tile sizes defined in common.h)
2920 // Per-thread: Q_q + KQ + mask + VKQ32 + V32 + K_f32 + padding
2921 size_t prefill = sizeof(float)*(GGML_FA_TILE_Q64*DK + 2*GGML_FA_TILE_Q64*GGML_FA_TILE_KV64 + GGML_FA_TILE_Q64*DV + GGML_FA_TILE_KV64*DV + GGML_FA_TILE_KV64*DK)*n_tasks;
2922
2923 // Decode path: n_kv_chunks = n_tasks (one chunk per thread)
2924 // Per-thread: VKQ accmulator (DV), partial M, partial S + intra-thread scratch for V, Q and VKQ
2925 size_t n_chunks = n_tasks;
2926 size_t decode = sizeof(float)*(neq2*n_chunks*(2+DV) + n_tasks*(DK + 2*DV));
2927
2928 cur += MAX(prefill, decode)((prefill) > (decode) ? (prefill) : (decode));
2929 } break;
2930 case GGML_OP_FLASH_ATTN_BACK:
2931 {
2932 const int64_t D = node->src[0]->ne[0];
2933 const int64_t ne11 = ggml_up(node->src[1]->ne[1], GGML_SOFT_MAX_UNROLL4);
2934 const int64_t mxDn = MAX(D, ne11)((D) > (ne11) ? (D) : (ne11)) * 2; // *2 because of S and SM in ggml_compute_forward_flash_attn_back
2935 if (node->src[1]->type == GGML_TYPE_F32) {
2936 cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
2937 cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
2938 } else if (node->src[1]->type == GGML_TYPE_F16) {
2939 cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
2940 cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
2941 } else if (node->src[1]->type == GGML_TYPE_BF16) {
2942 cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
2943 cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
2944 }
2945 } break;
2946
2947 case GGML_OP_CROSS_ENTROPY_LOSS:
2948 {
2949 cur = ggml_type_size(node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks);
2950 } break;
2951 case GGML_OP_GATED_DELTA_NET:
2952 {
2953 const int64_t S_v = node->src[2]->ne[0];
2954 const int64_t K = ggml_get_op_params_i32(node, 0);
2955 const int64_t per_thread = S_v + (K > 1 ? S_v * S_v : 0);
2956 cur = per_thread * sizeof(float) * n_tasks;
2957 } break;
2958 case GGML_OP_COUNT:
2959 {
2960 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 2960, "fatal error");
2961 }
2962 default:
2963 break;
2964 }
2965 }
2966
2967 work_size = MAX(work_size, cur)((work_size) > (cur) ? (work_size) : (cur));
2968 }
2969
2970 if (work_size > 0) {
2971 work_size += CACHE_LINE_SIZE64*(n_threads);
2972 }
2973
2974 cplan.threadpool = threadpool;
2975 cplan.n_threads = MIN(max_tasks, n_threads)((max_tasks) < (n_threads) ? (max_tasks) : (n_threads));
2976 cplan.work_size = work_size;
2977 cplan.work_data = NULL((void*)0);
2978
2979 return cplan;
2980}
2981
2982
2983// Try to fuse the current node with subsequent nodes for better performance.
2984// Returns the number of nodes skipped by fusion (>=1), or 0 if no fusion was applied.
2985static bool_Bool ggml_cpu_disable_fusion = false0; // initialized once in ggml_cpu_init(), read-only afterwards
2986
2987static int ggml_cpu_try_fuse_ops(
2988 const struct ggml_cgraph * cgraph,
2989 const int node_n,
2990 const struct ggml_compute_params * params,
2991 const struct ggml_cplan * cplan) {
2992
2993 if (ggml_cpu_disable_fusion || cplan->use_ref) {
2994 return 0;
2995 }
2996
2997 struct ggml_tensor * node = cgraph->nodes[node_n];
2998
2999 if (node->op == GGML_OP_RMS_NORM) {
3000 // RMS_NORM + MUL fusion
3001 const enum ggml_op fuse_ops[] = { GGML_OP_RMS_NORM, GGML_OP_MUL };
3002 if (ggml_can_fuse(cgraph, node_n, fuse_ops, 2)) {
3003 struct ggml_tensor * mul_node = cgraph->nodes[node_n + 1];
3004 const struct ggml_tensor * mul_w = (mul_node->src[0] == node)
3005 ? mul_node->src[1] : mul_node->src[0];
3006 if (node->src[0]->type == GGML_TYPE_F32 &&
3007 mul_node->type == GGML_TYPE_F32 &&
3008 mul_w->type == GGML_TYPE_F32 &&
3009 mul_w->ne[0] == node->ne[0] &&
3010 mul_w->nb[0] == sizeof(float)) {
3011
3012 ggml_compute_forward_rms_norm_mul_fused(params, node, mul_node);
3013 return 1;
3014 }
3015 }
3016 }
3017
3018 return 0;
3019}
3020
3021static thread_ret_t ggml_graph_compute_thread(void * data) {
3022 struct ggml_compute_state * state = (struct ggml_compute_state *) data;
3023 struct ggml_threadpool * tp = state->threadpool;
3024
3025 const struct ggml_cgraph * cgraph = tp->cgraph;
3026 const struct ggml_cplan * cplan = tp->cplan;
3027
3028#ifdef GGML_USE_CPU_RISCV64_SPACEMIT
3029 ggml_backend_cpu_riscv64_spacemit_set_numa_thread_affinity(state->ith);
3030#else
3031 set_numa_thread_affinity(state->ith);
3032#endif
3033
3034 struct ggml_compute_params params = {
3035 /*.ith =*/ state->ith,
3036 /*.nth =*/ atomic_load_explicit__c11_atomic_load(&tp->n_graph, memory_order_relaxed) & GGML_THREADPOOL_N_THREADS_MASK(0xffffU),
3037 /*.wsize =*/ cplan->work_size,
3038 /*.wdata =*/ cplan->work_data,
3039 /*.threadpool =*/ tp,
3040 /*.use_ref =*/ cplan->use_ref,
3041 };
3042
3043#ifdef GGML_USE_OPENMP
3044 GGML_PRINT_DEBUG("thread #%d compute-start cplan %p\n", state->ith, (const void *)cplan);
3045#else
3046 GGML_PRINT_DEBUG("thread #%d compute-start cplan %p last-graph %d\n", state->ith, (const void *)cplan, state->last_graph);
3047#endif
3048
3049 for (int node_n = 0; node_n < cgraph->n_nodes && atomic_load_explicit__c11_atomic_load(&tp->abort, memory_order_relaxed) != node_n; node_n++) {
3050 struct ggml_tensor * node = cgraph->nodes[node_n];
3051
3052 if (ggml_op_is_empty(node->op)) {
3053 // skip NOPs
3054 continue;
3055 }
3056
3057 if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {
3058 continue;
3059 }
3060
3061 // TODO: move fused-op detection into ggml_graph_plan so fusion decisions are made once at planning time
3062 // Try fused ops, fall back to normal compute
3063 const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, &params, cplan);
3064 if (n_fused > 0) {
3065 node_n += n_fused;
3066 } else {
3067 ggml_compute_forward(&params, node);
3068 }
3069
3070 if (state->ith == 0 && cplan->abort_callback &&
3071 cplan->abort_callback(cplan->abort_callback_data)) {
3072 atomic_store_explicit__c11_atomic_store(&tp->abort, node_n + 1, memory_order_relaxed);
3073 tp->ec = GGML_STATUS_ABORTED;
3074 }
3075
3076 if (node_n + 1 < cgraph->n_nodes) {
3077 ggml_barrier(state->threadpool);
3078 }
3079 }
3080
3081#ifdef GGML_USE_OPENMP
3082 GGML_PRINT_DEBUG("thread #%d compute-done cplan %p\n", state->ith, (const void *)cplan);
3083#else
3084 GGML_PRINT_DEBUG("thread #%d compute-done cplan %p last-graph %d\n", state->ith, (const void *)cplan, state->last_graph);
3085#endif
3086
3087 ggml_barrier(state->threadpool);
3088
3089#ifdef GGML_USE_CPU_RISCV64_SPACEMIT
3090 ggml_backend_cpu_riscv64_spacemit_clear_numa_thread_affinity_threaded(state->ith);
3091#endif
3092
3093 return 0;
3094}
3095
3096#ifndef GGML_USE_OPENMP
3097
3098// check if thread is ready to proceed (exit from polling or sleeping)
3099// returns true if loops should exit, sets state->pending to indicate new work
3100static inline bool_Bool ggml_graph_compute_thread_ready(struct ggml_compute_state * state) {
3101 struct ggml_threadpool * threadpool = state->threadpool;
3102
3103 if (state->pending || threadpool->stop || threadpool->pause) { return true1; }
3104
3105 // check for new graph/work
3106 int n_graph = atomic_load_explicit__c11_atomic_load(&threadpool->n_graph, memory_order_relaxed);
3107 int n_threads = n_graph & GGML_THREADPOOL_N_THREADS_MASK(0xffffU);
3108 if (n_graph != state->last_graph) {
3109 state->pending = (state->ith < n_threads);
3110 state->last_graph = n_graph;
3111 return true1;
3112 }
3113
3114 return false0;
3115}
3116
3117// sync thread state after polling
3118static inline void ggml_graph_compute_thread_sync(struct ggml_compute_state * state) {
3119 // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead
3120 #ifdef GGML_TSAN_ENABLED
3121 atomic_fetch_add_explicit__c11_atomic_fetch_add(&state->threadpool->n_graph, 0, memory_order_seq_cst);
3122 #else
3123 atomic_thread_fence(memory_order_seq_cst)__c11_atomic_thread_fence(memory_order_seq_cst);
3124 #endif
3125 UNUSED(state)(void)(state);
3126}
3127
3128static inline bool_Bool ggml_graph_compute_poll_for_work(struct ggml_compute_state * state) {
3129 struct ggml_threadpool * threadpool = state->threadpool;
3130
3131 // This seems to make 0 ... 100 a decent range for polling level across modern processors.
3132 // Perhaps, we can adjust it dynamically based on load and things.
3133 const uint64_t n_rounds = 1024UL * 128 * threadpool->poll;
3134
3135 for (uint64_t i=0; !ggml_graph_compute_thread_ready(state) && i < n_rounds; i++) {
3136 // No new work. Keep polling.
3137 ggml_thread_cpu_relax();
3138 }
3139
3140 return state->pending;
3141}
3142
3143static inline bool_Bool ggml_graph_compute_check_for_work(struct ggml_compute_state * state) {
3144 struct ggml_threadpool * threadpool = state->threadpool;
3145
3146 if (ggml_graph_compute_poll_for_work(state)) {
3147 ggml_graph_compute_thread_sync(state);
3148 return state->pending;
3149 }
3150
3151 ggml_mutex_lock_shared(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex);
3152 while (!ggml_graph_compute_thread_ready(state)) {
3153 // No new work. Wait for the signal.
3154 GGML_PRINT_DEBUG("thread #%d waiting for work (sleeping)\n", state->ith);
3155 ggml_cond_wait(&threadpool->cond, &threadpool->mutex)pthread_cond_wait(&threadpool->cond, &threadpool->
mutex);
3156 }
3157 ggml_mutex_unlock_shared(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex);
3158
3159 return state->pending;
3160}
3161
3162static thread_ret_t ggml_graph_compute_secondary_thread(void* data) {
3163 struct ggml_compute_state * state = (struct ggml_compute_state *) data;
3164 struct ggml_threadpool * threadpool = state->threadpool;
3165
3166 if (threadpool->thread_create_callback) {
3167 threadpool->thread_create_callback();
3168 }
3169
3170 ggml_thread_apply_priority(threadpool->prio);
3171 if (ggml_thread_cpumask_is_valid(state->cpumask)) {
3172 ggml_thread_apply_affinity(state->cpumask);
3173 }
3174
3175 while (true1) {
3176 // Check if we need to sleep
3177 while (threadpool->pause) {
3178 GGML_PRINT_DEBUG("thread #%d inside pause loop\n", state->ith);
3179 ggml_mutex_lock_shared(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex);
3180 if (threadpool->pause) {
3181 ggml_cond_wait(&threadpool->cond, &threadpool->mutex)pthread_cond_wait(&threadpool->cond, &threadpool->
mutex);
3182 }
3183 GGML_PRINT_DEBUG("thread #%d resuming after wait\n", state->ith);
3184 ggml_mutex_unlock_shared(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex);
3185 }
3186
3187 // This needs to be checked for after the cond_wait
3188 if (threadpool->stop) break;
3189
3190 // Check if there is new work
3191 // The main thread is the only one that can dispatch new work
3192
3193 ggml_graph_compute_check_for_work(state);
3194 if (state->pending) {
3195 state->pending = false0;
3196 ggml_graph_compute_thread(state);
3197 }
3198 }
3199
3200 if (threadpool->thread_destroy_callback) {
3201 threadpool->thread_destroy_callback();
3202 }
3203
3204 return (thread_ret_t) 0;
3205}
3206
3207// Start processing new graph
3208static void ggml_graph_compute_kickoff(struct ggml_threadpool * threadpool, int n_threads)
3209{
3210 // Always take the mutex here because the worker threads are doing hybrid poll/wait
3211
3212 ggml_mutex_lock(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex);
3213
3214 // Update the number of active threads and the graph count
3215 int n_graph = atomic_load_explicit__c11_atomic_load(&threadpool->n_graph, memory_order_relaxed) >> GGML_THREADPOOL_N_THREADS_BITS(16);
3216 n_graph = ((n_graph + 1) << GGML_THREADPOOL_N_THREADS_BITS(16)) | (n_threads & GGML_THREADPOOL_N_THREADS_MASK(0xffffU));
3217
3218 GGML_PRINT_DEBUG("compute-kickoff: n_threads %d n_graph %d\n", n_threads, n_graph);
3219
3220 // Indicate the graph is ready to be processed
3221 // We need the full seq-cst fence here because of the polling threads (used in thread_sync)
3222 atomic_store_explicit__c11_atomic_store(&threadpool->n_graph, n_graph, memory_order_seq_cst);
3223
3224 if (threadpool->pause) {
3225 // Update main thread prio and affinity to match the threadpool settings
3226 ggml_thread_apply_priority(threadpool->prio);
3227 if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {
3228 ggml_thread_apply_affinity(threadpool->workers[0].cpumask);
3229 }
3230
3231 // resume does cond broadcast
3232 ggml_threadpool_resume_locked(threadpool);
3233 } else {
3234 ggml_cond_broadcast(&threadpool->cond)pthread_cond_broadcast(&threadpool->cond);
3235 }
3236
3237 ggml_mutex_unlock(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex);
3238}
3239
3240#endif // GGML_USE_OPENMP
3241
3242static struct ggml_threadpool * ggml_threadpool_new_impl(
3243 struct ggml_threadpool_params * tpp,
3244 struct ggml_cgraph * cgraph,
3245 struct ggml_cplan * cplan) {
3246
3247 struct ggml_threadpool * threadpool =
3248 ggml_aligned_malloc(sizeof(struct ggml_threadpool));
3249 {
3250 threadpool->cgraph = cgraph;
3251 threadpool->cplan = cplan;
3252 threadpool->n_graph = 0;
3253 threadpool->n_barrier = 0;
3254 threadpool->n_barrier_passed = 0;
3255 threadpool->current_chunk = 0;
3256 threadpool->stop = false0;
3257 threadpool->pause = tpp->paused;
3258 threadpool->abort = -1;
3259 threadpool->workers = NULL((void*)0);
3260 threadpool->n_threads = tpp->n_threads;
3261 threadpool->poll = tpp->poll;
3262 threadpool->prio = tpp->prio;
3263 threadpool->thread_create_callback = tpp->thread_create_callback;
3264 threadpool->thread_destroy_callback = tpp->thread_destroy_callback;
3265 threadpool->ec = GGML_STATUS_SUCCESS;
3266 }
3267
3268 // Allocate and init workers state
3269 const size_t workers_size = sizeof(struct ggml_compute_state) * tpp->n_threads;
3270 struct ggml_compute_state * workers = ggml_aligned_malloc(workers_size);
3271
3272 memset(workers, 0, workers_size);
3273 for (int j = 0; j < tpp->n_threads; j++) {
3274 workers[j].threadpool = threadpool;
3275 workers[j].ith = j;
3276 }
3277
3278 threadpool->workers = workers;
3279
3280#ifdef GGML_USE_OPENMP
3281 int32_t cpumask_iter = 0;
3282
3283 // Compute CPU masks for each thread
3284 for (int j = 0; j < tpp->n_threads; j++) {
3285 ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter);
3286 }
3287#else // GGML_USE_OPENMP
3288 ggml_mutex_init(&threadpool->mutex)pthread_mutex_init(&threadpool->mutex, ((void*)0));
3289 ggml_cond_init(&threadpool->cond)pthread_cond_init(&threadpool->cond, ((void*)0));
3290
3291 // Spin the threads for all workers, and update CPU placements.
3292 // Place the main thread last (towards the higher numbered CPU cores).
3293
3294 int32_t cpumask_iter = 0;
3295
3296 for (int j = 1; j < tpp->n_threads; j++) {
3297 ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter);
3298
3299 int32_t rc = ggml_thread_createpthread_create(&workers[j].thrd, NULL((void*)0), ggml_graph_compute_secondary_thread, &workers[j]);
3300 GGML_ASSERT(rc == 0)if (!(rc == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 3300, "GGML_ASSERT(%s) failed", "rc == 0");
3301 }
3302
3303 ggml_thread_cpumask_next(tpp->cpumask, workers[0].cpumask, tpp->strict_cpu, &cpumask_iter);
3304
3305 if (!threadpool->pause) {
3306 // Update main thread prio and affinity at the start, otherwise we'll do it in resume
3307 ggml_thread_apply_priority(threadpool->prio);
3308 if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {
3309 ggml_thread_apply_affinity(threadpool->workers[0].cpumask);
3310 }
3311 }
3312#endif // GGML_USE_OPENMP
3313
3314 return threadpool;
3315}
3316
3317struct ggml_threadpool * ggml_threadpool_new(struct ggml_threadpool_params * tpp) {
3318 return ggml_threadpool_new_impl(tpp, NULL((void*)0), NULL((void*)0));
3319}
3320
3321enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {
3322 ggml_cpu_init();
3323
3324 GGML_ASSERT(cplan)if (!(cplan)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 3324, "GGML_ASSERT(%s) failed", "cplan");
3325 GGML_ASSERT(cplan->n_threads > 0)if (!(cplan->n_threads > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 3325, "GGML_ASSERT(%s) failed", "cplan->n_threads > 0"
);
3326 GGML_ASSERT(cplan->work_size == 0 || cplan->work_data != NULL)if (!(cplan->work_size == 0 || cplan->work_data != ((void
*)0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c"
, 3326, "GGML_ASSERT(%s) failed", "cplan->work_size == 0 || cplan->work_data != NULL"
);
3327
3328 int n_threads = cplan->n_threads;
3329 struct ggml_threadpool * threadpool = cplan->threadpool;
3330
3331 bool_Bool disposable_threadpool = false0;
3332
3333 if (threadpool == NULL((void*)0)) {
3334 //GGML_PRINT_DEBUG("Threadpool is not specified. Will create a disposable threadpool : n_threads %d\n", n_threads);
3335 disposable_threadpool = true1;
3336
3337 struct ggml_threadpool_params ttp = ggml_threadpool_params_default(n_threads);
3338 threadpool = ggml_threadpool_new_impl(&ttp, cgraph, cplan);
3339 } else {
3340 // Reset some of the parameters that need resetting
3341 // No worker threads should be accessing the parameters below at this stage
3342 threadpool->cgraph = cgraph;
3343 threadpool->cplan = cplan;
3344 threadpool->current_chunk = 0;
3345 threadpool->abort = -1;
3346 threadpool->ec = GGML_STATUS_SUCCESS;
3347 }
3348
3349#ifdef GGML_USE_OPENMP
3350 if (n_threads > 1) {
3351 #pragma omp parallel num_threads(n_threads)
3352 {
3353 #pragma omp single
3354 {
3355 // update the number of threads from the actual number of threads that we got from OpenMP
3356 n_threads = omp_get_num_threads();
3357 atomic_store_explicit__c11_atomic_store(&threadpool->n_graph, n_threads, memory_order_relaxed);
3358 }
3359
3360 // Apply thread CPU mask and priority
3361 int ith = omp_get_thread_num();
3362
3363 ggml_thread_apply_priority(threadpool->prio);
3364 if (ggml_thread_cpumask_is_valid(threadpool->workers[ith].cpumask)) {
3365 ggml_thread_apply_affinity(threadpool->workers[ith].cpumask);
3366 }
3367 ggml_graph_compute_thread(&threadpool->workers[ith]);
3368 }
3369 } else {
3370 atomic_store_explicit__c11_atomic_store(&threadpool->n_graph, 1, memory_order_relaxed);
3371 ggml_graph_compute_thread(&threadpool->workers[0]);
3372 }
3373#else
3374 if (n_threads > threadpool->n_threads) {
3375 GGML_LOG_WARN("cplan requested more threads (%d) than available (%d)\n", n_threads, threadpool->n_threads)ggml_log_internal(GGML_LOG_LEVEL_WARN , "cplan requested more threads (%d) than available (%d)\n"
, n_threads, threadpool->n_threads);
3376 n_threads = threadpool->n_threads;
3377 }
3378
3379 // Kick all threads to start the new graph
3380 ggml_graph_compute_kickoff(threadpool, n_threads);
3381
3382 // This is a work thread too
3383 ggml_graph_compute_thread(&threadpool->workers[0]);
3384#endif
3385
3386 // don't leave affinity set on the main thread
3387 clear_numa_thread_affinity();
3388
3389 enum ggml_status ret = threadpool->ec;
3390
3391 if (disposable_threadpool) {
3392 ggml_threadpool_free(threadpool);
3393 }
3394
3395 return ret;
3396}
3397
3398enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) {
3399 struct ggml_cplan cplan = ggml_graph_plan(cgraph, n_threads, NULL((void*)0));
3400
3401 cplan.work_data = (uint8_t *)ggml_new_buffer(ctx, cplan.work_size);
3402
3403 return ggml_graph_compute(cgraph, &cplan);
3404}
3405
3406void ggml_cpu_fp32_to_fp32(const float * x, float * y, int64_t n) {
3407 memcpy(y, x, n * sizeof(float));
3408}
3409
3410void ggml_cpu_fp32_to_fp16(const float * x, ggml_fp16_t * y, int64_t n) {
3411 int64_t i = 0;
3412#if defined(__F16C__1)
3413#if defined(__AVX512F__)
3414 for (; i + 15 < n; i += 16) {
3415 __m512 x_vec = _mm512_loadu_ps(x + i);
3416 __m256i y_vec = _mm512_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT)((__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(__m512)(
x_vec), (int)(0x00), (__v16hi)_mm256_undefined_si256(), (__mmask16
)-1));
3417 _mm256_storeu_si256((__m256i *)(y + i), y_vec);
3418 }
3419#endif
3420 for (; i + 7 < n; i += 8) {
3421 __m256 x_vec = _mm256_loadu_ps(x + i);
3422 __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT)((__m128i)__builtin_ia32_vcvtps2ph256((__v8sf)(__m256)(x_vec)
, (0x00)));
3423 _mm_storeu_si128((__m128i *)(y + i), y_vec);
3424 }
3425 for (; i + 3 < n; i += 4) {
3426 __m128 x_vec = _mm_loadu_ps(x + i);
3427 __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT)((__m128i)__builtin_ia32_vcvtps2ph((__v4sf)(__m128)(x_vec), (
0x00)));
3428 _mm_storel_epi64((__m128i *)(y + i), y_vec);
3429 }
3430#elif defined(__riscv_zvfh)
3431 for (int vl; i < n; i += vl) {
3432 vl = __riscv_vsetvl_e32m2(n - i);
3433 vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);
3434 vfloat16m1_t vy = __riscv_vfncvt_f_f_w_f16m1(vx, vl);
3435 __riscv_vse16_v_f16m1((_Float16 *)&y[i], vy, vl);
3436 }
3437#endif
3438 for (; i < n; ++i) {
3439 y[i] = GGML_CPU_FP32_TO_FP16(x[i])ggml_compute_fp32_to_fp16(x[i]);
3440 }
3441}
3442
3443void ggml_cpu_fp16_to_fp32(const ggml_fp16_t * x, float * y, int64_t n) {
3444 int64_t i = 0;
3445#if defined(__F16C__1)
3446#if defined(__AVX512F__)
3447 for (; i + 15 < n; i += 16) {
3448 __m256i x_vec = _mm256_loadu_si256((const __m256i *)(x + i));
3449 __m512 y_vec = _mm512_cvtph_ps(x_vec);
3450 _mm512_storeu_ps(y + i, y_vec);
3451 }
3452#endif
3453 for (; i + 7 < n; i += 8) {
3454 __m128i x_vec = _mm_loadu_si128((const __m128i *)(x + i));
3455 __m256 y_vec = _mm256_cvtph_ps(x_vec);
3456 _mm256_storeu_ps(y + i, y_vec);
3457 }
3458 for (; i + 3 < n; i += 4) {
3459 __m128i x_vec = _mm_loadl_epi64((const __m128i *)(x + i));
3460 __m128 y_vec = _mm_cvtph_ps(x_vec);
3461 _mm_storeu_ps(y + i, y_vec);
3462 }
3463
3464#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfhmin)
3465 // calculate step size
3466 const int epr = __riscv_vsetvlmax_e16m2();
3467 const int step = epr * 2;
3468 const int np = (n & ~(step - 1));
3469
3470 // unroll by 2
3471 for (; i < np; i += step) {
3472 vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16*)x + i, epr);
3473 vfloat32m4_t ay0 = __riscv_vfwcvt_f_f_v_f32m4(ax0, epr);
3474 __riscv_vse32_v_f32m4(y + i, ay0, epr);
3475
3476 vfloat16m2_t ax1 = __riscv_vle16_v_f16m2((const _Float16*)x + i + epr, epr);
3477 vfloat32m4_t ay1 = __riscv_vfwcvt_f_f_v_f32m4(ax1, epr);
3478 __riscv_vse32_v_f32m4(y + i + epr, ay1, epr);
3479 }
3480
3481 // leftovers
3482 int vl;
3483 for (i = np; i < n; i += vl) {
3484 vl = __riscv_vsetvl_e16m2(n - i);
3485 vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16*)x + i, vl);
3486 vfloat32m4_t ay0 = __riscv_vfwcvt_f_f_v_f32m4(ax0, vl);
3487 __riscv_vse32_v_f32m4(y + i, ay0, vl);
3488 }
3489
3490#endif
3491
3492 for (; i < n; ++i) {
3493 y[i] = GGML_CPU_FP16_TO_FP32(x[i])ggml_lookup_fp16_to_fp32(x[i]);
3494 }
3495}
3496
3497void ggml_cpu_fp32_to_bf16(const float * x, ggml_bf16_t * y, int64_t n) {
3498 int64_t i = 0;
3499 for (; i < n; ++i) {
3500 y[i] = GGML_FP32_TO_BF16(x[i])ggml_compute_fp32_to_bf16(x[i]);
3501 }
3502}
3503
3504void ggml_cpu_fp32_to_i32(const float * x, int32_t * y, int64_t n) {
3505 int64_t i = 0;
3506 for (; i < n; ++i) {
3507 y[i] = x[i];
3508 }
3509}
3510
3511void ggml_cpu_bf16_to_fp32(const ggml_bf16_t * x, float * y, int64_t n) {
3512 int64_t i = 0;
3513#if defined(__AVX2__1)
3514#if defined(__AVX512F__)
3515 for (; i + 15 < n; i += 16) {
3516 _mm512_storeu_ps(y + i,
3517 _mm512_castsi512_ps(
3518 _mm512_slli_epi32(
3519 _mm512_cvtepu16_epi32(
3520 _mm256_loadu_si256(
3521 (const __m256i *)(x + i))),
3522 16)));
3523 }
3524#endif
3525 for (; i + 7 < n; i += 8) {
3526 _mm256_storeu_ps(y + i,
3527 _mm256_castsi256_ps(
3528 _mm256_slli_epi32(
3529 _mm256_cvtepu16_epi32(
3530 _mm_loadu_si128(
3531 (const __m128i *)(x + i))),
3532 16)));
3533 }
3534#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfmin)
3535 // calculate step size
3536 const int epr = __riscv_vsetvlmax_e16m2();
3537 const int step = epr * 2;
3538 const int np = (n & ~(step - 1));
3539
3540 // unroll by 2
3541 for (; i < np; i += step) {
3542 vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16*)x + i, epr);
3543 vfloat32m4_t ay0 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax0, epr);
3544 __riscv_vse32_v_f32m4(y + i, ay0, epr);
3545
3546 vbfloat16m2_t ax1 = __riscv_vle16_v_bf16m2((const __bf16*)x + i + epr, epr);
3547 vfloat32m4_t ay1 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax1, epr);
3548 __riscv_vse32_v_f32m4(y + i + epr, ay1, epr);
3549 }
3550
3551 // leftovers
3552 int vl;
3553 for (i = np; i < n; i += vl) {
3554 vl = __riscv_vsetvl_e16m2(n - i);
3555 vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16*)x + i, vl);
3556 vfloat32m4_t ay0 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax0, vl);
3557 __riscv_vse32_v_f32m4(y + i, ay0, vl);
3558 }
3559#endif
3560 for (; i < n; i++) {
3561 y[i] = GGML_BF16_TO_FP32(x[i])ggml_compute_bf16_to_fp32(x[i]);
3562 }
3563}
3564
3565int ggml_cpu_has_avx(void) {
3566#if defined(__AVX__1)
3567 return 1;
3568#else
3569 return 0;
3570#endif
3571}
3572
3573int ggml_cpu_has_avx_vnni(void) {
3574#if defined(__AVXVNNI__)
3575 return 1;
3576#else
3577 return 0;
3578#endif
3579}
3580
3581int ggml_cpu_has_avx2(void) {
3582#if defined(__AVX2__1)
3583 return 1;
3584#else
3585 return 0;
3586#endif
3587}
3588
3589int ggml_cpu_has_avx512(void) {
3590#if defined(__AVX512F__)
3591 return 1;
3592#else
3593 return 0;
3594#endif
3595}
3596
3597int ggml_cpu_has_avx512_vbmi(void) {
3598#if defined(__AVX512VBMI__)
3599 return 1;
3600#else
3601 return 0;
3602#endif
3603}
3604
3605int ggml_cpu_has_avx512_vnni(void) {
3606#if defined(__AVX512VNNI__)
3607 return 1;
3608#else
3609 return 0;
3610#endif
3611}
3612
3613int ggml_cpu_has_avx512_bf16(void) {
3614#if defined(__AVX512BF16__)
3615 return 1;
3616#else
3617 return 0;
3618#endif
3619}
3620
3621int ggml_cpu_has_amx_int8(void) {
3622#if defined(__AMX_INT8__)
3623 return 1;
3624#else
3625 return 0;
3626#endif
3627}
3628
3629int ggml_cpu_has_bmi2(void) {
3630#if defined(__BMI2__1)
3631 return 1;
3632#else
3633 return 0;
3634#endif
3635}
3636
3637int ggml_cpu_has_fma(void) {
3638#if defined(__FMA__1)
3639 return 1;
3640#else
3641 return 0;
3642#endif
3643}
3644
3645int ggml_cpu_has_arm_fma(void) {
3646#if defined(__ARM_FEATURE_FMA)
3647 return 1;
3648#else
3649 return 0;
3650#endif
3651}
3652
3653int ggml_cpu_has_riscv_v(void) {
3654#if defined(__riscv_v_intrinsic)
3655 return 1;
3656#else
3657 return 0;
3658#endif
3659}
3660
3661int ggml_cpu_get_rvv_vlen(void) {
3662#if defined(__riscv) && defined(__riscv_v_intrinsic)
3663 return ggml_riscv_arch_features.rvv_vlen;
3664#else
3665 return 0;
3666#endif
3667}
3668
3669int ggml_cpu_has_f16c(void) {
3670#if defined(__F16C__1)
3671 return 1;
3672#else
3673 return 0;
3674#endif
3675}
3676
3677int ggml_cpu_has_fp16_va(void) {
3678#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
3679 return 1;
3680#else
3681 return 0;
3682#endif
3683}
3684
3685int ggml_cpu_has_wasm_simd(void) {
3686#if defined(__wasm_simd128__)
3687 return 1;
3688#else
3689 return 0;
3690#endif
3691}
3692
3693int ggml_cpu_has_llamafile(void) {
3694#if defined(GGML_USE_LLAMAFILE)
3695 return 1;
3696#else
3697 return 0;
3698#endif
3699}
3700
3701int ggml_cpu_has_sse3(void) {
3702#if defined(__SSE3__1)
3703 return 1;
3704#else
3705 return 0;
3706#endif
3707}
3708
3709int ggml_cpu_has_ssse3(void) {
3710#if defined(__SSSE3__1)
3711 return 1;
3712#else
3713 return 0;
3714#endif
3715}
3716
3717int ggml_cpu_has_vsx(void) {
3718#if defined(__POWER9_VECTOR__)
3719 return 1;
3720#else
3721 return 0;
3722#endif
3723}
3724
3725int ggml_cpu_has_vxe(void) {
3726#if defined(__VXE__) || defined(__VXE2__)
3727 return 1;
3728#else
3729 return 0;
3730#endif
3731}
3732
3733int ggml_cpu_has_neon(void) {
3734#if defined(__ARM_ARCH) && defined(__ARM_NEON)
3735 return 1;
3736#else
3737 return 0;
3738#endif
3739}
3740
3741int ggml_cpu_has_dotprod(void) {
3742#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_DOTPROD)
3743 return 1;
3744#else
3745 return 0;
3746#endif
3747}
3748
3749int ggml_cpu_has_sve(void) {
3750#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)
3751 return 1;
3752#else
3753 return 0;
3754#endif
3755}
3756
3757int ggml_cpu_has_matmul_int8(void) {
3758#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_MATMUL_INT8)
3759 return 1;
3760#else
3761 return 0;
3762#endif
3763}
3764
3765int ggml_cpu_get_sve_cnt(void) {
3766#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)
3767 return ggml_arm_arch_features.sve_cnt;
3768#else
3769 return 0;
3770#endif
3771}
3772
3773int ggml_cpu_has_sme(void) {
3774#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SME)
3775 return 1;
3776#else
3777 return 0;
3778#endif
3779}
3780
3781void ggml_cpu_init(void) {
3782 // needed to initialize ggml_time
3783 {
3784 struct ggml_init_params params = { 0, NULL((void*)0), false0 };
3785 struct ggml_context * ctx = ggml_init(params);
3786 ggml_free(ctx);
3787 }
3788
3789 ggml_critical_section_start();
3790
3791 static bool_Bool is_first_call = true1;
3792
3793 if (is_first_call) {
3794 // initialize GELU, Quick GELU, SILU and EXP F32 tables
3795 {
3796 const uint64_t t_start = ggml_time_us(); UNUSED(t_start)(void)(t_start);
3797
3798 for (int i = 0; i < (1 << 16); ++i) {
3799 union {
3800 uint16_t u16;
3801 ggml_fp16_t fp16;
3802 } u = {i};
3803 float f = GGML_COMPUTE_FP16_TO_FP32(u.fp16)ggml_compute_fp16_to_fp32(u.fp16);
3804 ggml_table_f32_f16[i] = f;
3805 ggml_table_gelu_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_f32(f))ggml_compute_fp32_to_fp16(ggml_gelu_f32(f));
3806 ggml_table_gelu_quick_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_quick_f32(f))ggml_compute_fp32_to_fp16(ggml_gelu_quick_f32(f));
3807 }
3808
3809 // initialize E8M0 half table (256 entries)
3810 for (int i = 0; i < (1 << 8); ++i) {
3811 ggml_table_f32_e8m0_half[i] = GGML_E8M0_TO_FP32_HALF(i)ggml_e8m0_to_fp32_half(i);
3812 }
3813
3814 const uint64_t t_end = ggml_time_us(); UNUSED(t_end)(void)(t_end);
3815
3816 GGML_PRINT_DEBUG("%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\n", __func__, (t_end - t_start)/1000.0);
3817
3818#ifdef GGML_USE_OPENMP
3819 //if (!getenv("OMP_WAIT_POLICY")) {
3820 // // set the wait policy to active, so that OpenMP threads don't sleep
3821 // setenv("OMP_WAIT_POLICY", "active", 0)
3822 //}
3823
3824 if (!getenv("KMP_BLOCKTIME")) {
3825 // set the time to wait before sleeping a thread
3826 // this is less aggressive than setting the wait policy to active, but should achieve similar results in most cases
3827#ifdef _WIN32
3828 _putenv_s("KMP_BLOCKTIME", "200"); // 200ms
3829#else
3830 setenv("KMP_BLOCKTIME", "200", 0); // 200ms
3831#endif
3832 }
3833#endif
3834 }
3835
3836#if defined(__ARM_ARCH)
3837 ggml_init_arm_arch_features();
3838#endif
3839
3840#if defined(__riscv)
3841 ggml_init_riscv_arch_features();
3842#endif
3843
3844 {
3845 const char * env = getenv("GGML_CPU_DISABLE_FUSION");
3846 ggml_cpu_disable_fusion = (env != NULL((void*)0) && atoi(env) == 1);
3847 }
3848
3849 is_first_call = false0;
3850 }
3851
3852 ggml_critical_section_end();
3853}