Bug Summary

File:root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c
Warning:line 3324, column 13
Array access (via field 'ne') results in a null pointer dereference

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O2 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ggml-c.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +avx -target-feature +avx2 -target-feature +bmi2 -target-feature +f16c -target-feature +fma -target-feature +sse4.2 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -resource-dir /usr/lib/llvm-23/lib/clang/23 -include /root/firefox-clang/config/gcc_hidden.h -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/system_wrappers -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG=1 -D _GNU_SOURCE=1 -D GGML_USE_CPU=1 -D GGML_VERSION="GGML_VERSION" -D GGML_COMMIT="GGML_COMMIT" -D GGML_SHARED=1 -D LLAMA_SHARED=1 -D GGML_BUILD=1 -D LLAMA_BUILD=1 -D GGML_BACKEND_SHARED=1 -D GGML_BACKEND_BUILD=1 -D MOZ_HAS_MOZGLUE -I /root/firefox-clang/third_party/llama.cpp -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -I /root/firefox-clang/third_party/llama.cpp/ggml -I /root/firefox-clang/third_party/llama.cpp/ggml/include -I /root/firefox-clang/third_party/llama.cpp/ggml/src -I /root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu -I /root/firefox-clang/third_party/llama.cpp/include -I /root/firefox-clang/third_party/llama.cpp/src -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/llvm-23/lib/clang/23/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=tautological-type-limit-compare -Wno-range-loop-analysis -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-unknown-warning-option -Wno-character-conversion -Wno-sign-compare -Wno-unused-function -Wno-unreachable-code -Wno-tautological-unsigned-enum-zero-compare -Wno-absolute-value -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -fdiagnostics-absolute-paths -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -mllvm -dwarf-linkage-names=Abstract -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-09-01-224014-2642839-1 -x c /root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c
1#define _CRT_SECURE_NO_DEPRECATE // Disables "unsafe" warnings on Windows
2#define _USE_MATH_DEFINES // For M_PI on MSVC
3
4#include "ggml-backend.h"
5#include "ggml-impl.h"
6#include "ggml-threading.h"
7#include "ggml-cpu.h"
8#include "ggml.h"
9
10// FIXME: required here for quantization functions
11#include "ggml-quants.h"
12
13#ifdef GGML_USE_CPU_HBM
14#include <hbwmalloc.h>
15#endif
16
17#if defined(_MSC_VER) || defined(__MINGW32__)
18#include <malloc.h> // using malloc.h with MSC/MINGW
19#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
20#include <alloca.h>
21#endif
22
23#include <assert.h>
24#include <errno(*__errno_location ()).h>
25#include <time.h>
26#include <math.h>
27#include <stdlib.h>
28#include <string.h>
29#include <stdint.h>
30#include <inttypes.h>
31#include <stdio.h>
32#include <float.h>
33#include <limits.h>
34#include <stdarg.h>
35#include <signal.h>
36#if defined(__gnu_linux__1)
37#include <syscall.h>
38#endif
39
40#if defined(__APPLE__)
41#include <unistd.h>
42#include <mach/mach.h>
43#include <TargetConditionals.h>
44#endif
45
46#if defined(_WIN32)
47#define WIN32_LEAN_AND_MEAN
48#ifndef NOMINMAX
49 #define NOMINMAX
50#endif
51#include <windows.h>
52#endif
53
54#define UNUSEDGGML_UNUSED GGML_UNUSED
55
56uint64_t ggml_graph_next_uid(void) {
57#ifdef _MSC_VER
58#if defined(_WIN32)
59 static volatile LONG counter = 1;
60 return (uint64_t) InterlockedIncrement(&counter) - 1;
61#else
62 static volatile long long counter = 1;
63 return (uint64_t) _InterlockedIncrement64(&counter) - 1;
64#endif
65#else
66 static uint64_t counter = 1;
67 return __atomic_fetch_add(&counter, 1, __ATOMIC_RELAXED0);
68#endif
69}
70
71// Needed for ggml_fp32_to_bf16_row()
72#if defined(__AVX512BF16__)
73#if defined(_MSC_VER)
74#define m512i(p) p
75#else
76#include <immintrin.h>
77#define m512i(p) (__m512i)(p)
78#endif // defined(_MSC_VER)
79#endif // defined(__AVX512BF16__)
80
81#if defined(__linux__1) || \
82 defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
83 (defined(__APPLE__) && !TARGET_OS_TV && !TARGET_OS_WATCH)
84
85#include <unistd.h>
86#include <sys/types.h>
87#include <sys/stat.h>
88#include <sys/wait.h>
89#if defined(__linux__1)
90#include <sys/prctl.h>
91#endif
92
93#if defined(__ANDROID__)
94#include <unwind.h>
95#include <dlfcn.h>
96#include <stdio.h>
97
98struct backtrace_state {
99 void ** current;
100 void ** end;
101};
102
103static _Unwind_Reason_Code unwind_callback(struct _Unwind_Context* context, void* arg) {
104 struct backtrace_state * state = (struct backtrace_state *)arg;
105 uintptr_t pc = _Unwind_GetIP(context);
106 if (pc) {
107 if (state->current == state->end) {
108 return _URC_END_OF_STACK;
109 } else {
110 *state->current++ = (void*)pc;
111 }
112 }
113 return _URC_NO_REASON;
114}
115
116static void ggml_print_backtrace_symbols(void) {
117 const int max = 100;
118 void* buffer[max];
119
120 struct backtrace_state state = {buffer, buffer + max};
121 _Unwind_Backtrace(unwind_callback, &state);
122
123 int count = state.current - buffer;
124
125 for (int idx = 0; idx < count; ++idx) {
126 const void * addr = buffer[idx];
127 const char * symbol = "";
128
129 Dl_info info;
130 if (dladdr(addr, &info) && info.dli_sname) {
131 symbol = info.dli_sname;
132 }
133
134 fprintf(stderrstderr, "%d: %p %s\n", idx, addr, symbol);
135 }
136}
137#elif defined(__linux__1) && defined(__GLIBC__2)
138#include <execinfo.h>
139static void ggml_print_backtrace_symbols(void) {
140 void * trace[100];
141 int nptrs = backtrace(trace, sizeof(trace)/sizeof(trace[0]));
142 backtrace_symbols_fd(trace, nptrs, STDERR_FILENO2);
143}
144#elif defined(__APPLE__)
145#include <execinfo.h>
146static void ggml_print_backtrace_symbols(void) {
147 void * trace[100];
148 int nptrs = backtrace(trace, sizeof(trace)/sizeof(trace[0]));
149 backtrace_symbols_fd(trace, nptrs, STDERR_FILENO2);
150}
151#else
152static void ggml_print_backtrace_symbols(void) {
153 // platform not supported
154}
155#endif
156
157void ggml_print_backtrace(void) {
158 const char * GGML_NO_BACKTRACE = getenv("GGML_NO_BACKTRACE");
159 if (GGML_NO_BACKTRACE) {
160 return;
161 }
162#if defined(__APPLE__)
163 // On macOS, fork+debugger attachment is problematic due to:
164 // 1. libdispatch "poisons" forked child processes
165 // 2. lldb has issues attaching to parent from forked child
166 // Use simple backtrace() instead to avoid Terminal.app crashes
167 const char * GGML_BACKTRACE_LLDB = getenv("GGML_BACKTRACE_LLDB");
168 if (!GGML_BACKTRACE_LLDB) {
169 fprintf(stderrstderr, "WARNING: Using native backtrace. Set GGML_BACKTRACE_LLDB for more info.\n");
170 fprintf(stderrstderr, "WARNING: GGML_BACKTRACE_LLDB may cause native MacOS Terminal.app to crash.\n");
171 fprintf(stderrstderr, "See: https://github.com/ggml-org/llama.cpp/pull/17869\n");
172 ggml_print_backtrace_symbols();
173 return;
174 }
175#endif
176#if defined(__linux__1)
177 FILE * f = fopen("/proc/self/status", "r");
178 size_t size = 0;
179 char * line = NULL((void*)0);
180 ssize_t length = 0;
181 while ((length = getline(&line, &size, f)) > 0) {
182 if (!strncmp(line, "TracerPid:", sizeof("TracerPid:") - 1) &&
183 (length != sizeof("TracerPid:\t0\n") - 1 || line[length - 2] != '0')) {
184 // Already being debugged, and the breakpoint is the later abort()
185 free(line);
186 fclose(f);
187 return;
188 }
189 }
190 free(line);
191 fclose(f);
192 int lock[2] = { -1, -1 };
193 (void) !pipe(lock); // Don't start gdb until after PR_SET_PTRACER
194#endif
195 const int parent_pid = getpid();
196 const int child_pid = fork();
197 if (child_pid < 0) { // error
198#if defined(__linux__1)
199 close(lock[1]);
200 close(lock[0]);
201#endif
202 return;
203 } else if (child_pid == 0) { // child
204 char attach[32];
205 snprintf(attach, sizeof(attach), "attach %d", parent_pid);
206#if defined(__linux__1)
207 close(lock[1]);
208 (void) !read(lock[0], lock, 1);
209 close(lock[0]);
210#endif
211 // try gdb
212 execlp("gdb", "gdb", "--batch",
213 "-ex", "set style enabled on",
214 "-ex", attach,
215 "-ex", "bt -frame-info source-and-location",
216 "-ex", "detach",
217 "-ex", "quit",
218 (char *) NULL((void*)0));
219 // try lldb
220 execlp("lldb", "lldb", "--batch",
221 "-o", "bt",
222 "-o", "quit",
223 "-p", &attach[sizeof("attach ") - 1],
224 (char *) NULL((void*)0));
225 // gdb failed, fallback to backtrace_symbols
226 ggml_print_backtrace_symbols();
227 _Exit(0);
228 } else { // parent
229#if defined(__linux__1)
230 prctl(PR_SET_PTRACER0x59616d61, child_pid);
231 close(lock[1]);
232 close(lock[0]);
233#endif
234 waitpid(child_pid, NULL((void*)0), 0);
235 }
236}
237#else
238void ggml_print_backtrace(void) {
239 // platform not supported
240}
241#endif
242
243static ggml_abort_callback_t g_abort_callback = NULL((void*)0);
244
245// Set the abort callback (passing null will restore original abort functionality: printing a message to stdout)
246GGML_API__attribute__ ((visibility ("default"))) extern ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback) {
247 ggml_abort_callback_t ret_val = g_abort_callback;
248 g_abort_callback = callback;
249 return ret_val;
250}
251
252void ggml_abort(const char * file, int line, const char * fmt, ...) {
253 fflush(stdoutstdout);
254
255 char message[2048];
256 int offset = snprintf(message, sizeof(message), "%s:%d: ", file, line);
257
258 va_list args;
259 va_start(args, fmt)__builtin_va_start(args, fmt);
260 vsnprintf(message + offset, sizeof(message) - offset, fmt, args);
261 va_end(args)__builtin_va_end(args);
262
263 if (g_abort_callback) {
264 g_abort_callback(message);
265 } else {
266 // default: print error and backtrace to stderr
267 fprintf(stderrstderr, "%s\n", message);
268 ggml_print_backtrace();
269 }
270
271 abort();
272}
273
274// ggml_print_backtrace is registered with std::set_terminate by ggml.cpp
275
276//
277// logging
278//
279
280struct ggml_logger_state {
281 ggml_log_callback log_callback;
282 void * log_callback_user_data;
283};
284static struct ggml_logger_state g_logger_state = {ggml_log_callback_default, NULL((void*)0)};
285
286static void ggml_log_internal_v(enum ggml_log_level level, const char * format, va_list args) {
287 if (format == NULL((void*)0)) {
288 return;
289 }
290 va_list args_copy;
291 va_copy(args_copy, args)__builtin_va_copy(args_copy, args);
292 char buffer[128];
293 int len = vsnprintf(buffer, 128, format, args);
294 if (len < 128) {
295 g_logger_state.log_callback(level, buffer, g_logger_state.log_callback_user_data);
296 } else {
297 char * buffer2 = (char *) calloc(len + 1, sizeof(char));
298 vsnprintf(buffer2, len + 1, format, args_copy);
299 buffer2[len] = 0;
300 g_logger_state.log_callback(level, buffer2, g_logger_state.log_callback_user_data);
301 free(buffer2);
302 }
303 va_end(args_copy)__builtin_va_end(args_copy);
304}
305
306void ggml_log_internal(enum ggml_log_level level, const char * format, ...) {
307 va_list args;
308 va_start(args, format)__builtin_va_start(args, format);
309 ggml_log_internal_v(level, format, args);
310 va_end(args)__builtin_va_end(args);
311}
312
313void ggml_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) {
314 (void) level;
315 (void) user_data;
316 fputs(text, stderrstderr);
317 fflush(stderrstderr);
318}
319
320//
321// end of logging block
322//
323
324#ifdef GGML_USE_ACCELERATE
325// uncomment to use vDSP for soft max computation
326// note: not sure if it is actually faster
327//#define GGML_SOFT_MAX_ACCELERATE
328#endif
329
330
331void * ggml_aligned_malloc(size_t size) {
332#if defined(__s390x__)
333 const int alignment = 256;
334#else
335 const int alignment = 64;
336#endif
337
338#if defined(_MSC_VER) || defined(__MINGW32__)
339 return _aligned_malloc(size, alignment);
340#else
341 if (size == 0) {
342 GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n")ggml_log_internal(GGML_LOG_LEVEL_WARN , "Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n"
);
343 return NULL((void*)0);
344 }
345 void * aligned_memory = NULL((void*)0);
346 #ifdef GGML_USE_CPU_HBM
347 int result = hbw_posix_memalign(&aligned_memory, alignment, size);
348 #elif TARGET_OS_OSX
349 GGML_UNUSED(alignment)(void)(alignment);
350 kern_return_t alloc_status = vm_allocate((vm_map_t) mach_task_self(), (vm_address_t *) &aligned_memory, size, VM_FLAGS_ANYWHERE);
351 int result = EFAULT14;
352 switch (alloc_status) {
353 case KERN_SUCCESS:
354 result = 0;
355 break;
356 case KERN_INVALID_ADDRESS:
357 result = EINVAL22;
358 break;
359 case KERN_NO_SPACE:
360 result = ENOMEM12;
361 break;
362 default:
363 result = EFAULT14;
364 break;
365 }
366 #else
367 int result = posix_memalign(&aligned_memory, alignment, size);
368 #endif
369 if (result != 0) {
370 // Handle allocation failure
371 const char *error_desc = "unknown allocation error";
372 switch (result) {
373 case EINVAL22:
374 error_desc = "invalid alignment value";
375 break;
376 case ENOMEM12:
377 error_desc = "insufficient memory";
378 break;
379 }
380 GGML_LOG_ERROR("%s: %s (attempted to allocate %6.2f MB)\n", __func__, error_desc, size/(1024.0*1024.0))ggml_log_internal(GGML_LOG_LEVEL_ERROR, "%s: %s (attempted to allocate %6.2f MB)\n"
, __func__, error_desc, size/(1024.0*1024.0));
381 return NULL((void*)0);
382 }
383 return aligned_memory;
384#endif
385}
386
387void ggml_aligned_free(void * ptr, size_t size) {
388 GGML_UNUSED(size)(void)(size);
389#if defined(_MSC_VER) || defined(__MINGW32__)
390 _aligned_free(ptr);
391#elif GGML_USE_CPU_HBM
392 if (ptr != NULL((void*)0)) {
393 hbw_free(ptr);
394 }
395#elif TARGET_OS_OSX
396 if (ptr != NULL((void*)0)) {
397 vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ptr, size);
398 }
399#else
400 free(ptr);
401#endif
402}
403
404
405inline static void * ggml_malloc(size_t size) {
406 if (size == 0) {
407 GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_malloc!\n")ggml_log_internal(GGML_LOG_LEVEL_WARN , "Behavior may be unexpected when allocating 0 bytes for ggml_malloc!\n"
);
408 return NULL((void*)0);
409 }
410 void * result = malloc(size);
411 if (result == NULL((void*)0)) {
412 GGML_LOG_ERROR("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0))ggml_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to allocate %6.2f MB\n"
, __func__, size/(1024.0*1024.0));
413 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 413, "fatal error");
414 }
415 return result;
416}
417
418// calloc
419inline static void * ggml_calloc(size_t num, size_t size) {
420 if (num == 0 || size == 0) {
421 GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_calloc!\n")ggml_log_internal(GGML_LOG_LEVEL_WARN , "Behavior may be unexpected when allocating 0 bytes for ggml_calloc!\n"
);
422 return NULL((void*)0);
423 }
424 void * result = calloc(num, size);
425 if (result == NULL((void*)0)) {
426 GGML_LOG_ERROR("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0))ggml_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to allocate %6.2f MB\n"
, __func__, size/(1024.0*1024.0));
427 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 427, "fatal error");
428 }
429 return result;
430}
431
432#define GGML_MALLOC(size)ggml_malloc(size) ggml_malloc(size)
433#define GGML_CALLOC(num, size)ggml_calloc(num, size) ggml_calloc(num, size)
434
435#define GGML_FREE(ptr)free(ptr) free(ptr)
436
437const char * ggml_status_to_string(enum ggml_status status) {
438 switch (status) {
439 case GGML_STATUS_ALLOC_FAILED: return "GGML status: error (failed to allocate memory)";
440 case GGML_STATUS_FAILED: return "GGML status: error (operation failed)";
441 case GGML_STATUS_SUCCESS: return "GGML status: success";
442 case GGML_STATUS_ABORTED: return "GGML status: warning (operation aborted)";
443 }
444
445 return "GGML status: unknown";
446}
447
448float ggml_fp16_to_fp32do_not_use__ggml_fp16_to_fp32__in_ggml(ggml_fp16_t x) {
449#define ggml_fp16_to_fp32do_not_use__ggml_fp16_to_fp32__in_ggml do_not_use__ggml_fp16_to_fp32__in_ggml
450 return GGML_FP16_TO_FP32(x)ggml_compute_fp16_to_fp32(x);
451}
452
453ggml_fp16_t ggml_fp32_to_fp16do_not_use__ggml_fp32_to_fp16__in_ggml(float x) {
454#define ggml_fp32_to_fp16do_not_use__ggml_fp32_to_fp16__in_ggml do_not_use__ggml_fp32_to_fp16__in_ggml
455 return GGML_FP32_TO_FP16(x)ggml_compute_fp32_to_fp16(x);
456}
457
458float ggml_bf16_to_fp32do_not_use__ggml_bf16_to_fp32__in_ggml(ggml_bf16_t x) {
459#define ggml_bf16_to_fp32do_not_use__ggml_bf16_to_fp32__in_ggml do_not_use__ggml_bf16_to_fp32__in_ggml
460 return GGML_BF16_TO_FP32(x)ggml_compute_bf16_to_fp32(x); // it just left shifts
461}
462
463ggml_bf16_t ggml_fp32_to_bf16do_not_use__ggml_fp32_to_bf16__in_ggml(float x) {
464#define ggml_fp32_to_bf16do_not_use__ggml_fp32_to_bf16__in_ggml do_not_use__ggml_fp32_to_bf16__in_ggml
465 return GGML_FP32_TO_BF16(x)ggml_compute_fp32_to_bf16(x);
466}
467
468void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {
469 for (int64_t i = 0; i < n; i++) {
470 y[i] = GGML_FP16_TO_FP32(x[i])ggml_compute_fp16_to_fp32(x[i]);
471 }
472}
473
474void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {
475 int i = 0;
476 for (; i < n; ++i) {
477 y[i] = GGML_FP32_TO_FP16(x[i])ggml_compute_fp32_to_fp16(x[i]);
478 }
479}
480
481void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {
482 int i = 0;
483 for (; i < n; ++i) {
484 y[i] = GGML_BF16_TO_FP32(x[i])ggml_compute_bf16_to_fp32(x[i]);
485 }
486}
487
488void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) {
489 for (int i = 0; i < n; i++) {
490 y[i] = ggml_compute_fp32_to_bf16(x[i]);
491 }
492}
493
494void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {
495 int i = 0;
496#if defined(__AVX512BF16__)
497 // subnormals are flushed to zero on this platform
498 for (; i + 32 <= n; i += 32) {
499 _mm512_storeu_si512(
500 (__m512i *)(y + i),
501 m512i(_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),
502 _mm512_loadu_ps(x + i))));
503 }
504#endif
505 for (; i < n; i++) {
506 y[i] = GGML_FP32_TO_BF16(x[i])ggml_compute_fp32_to_bf16(x[i]);
507 }
508}
509
510bool_Bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {
511 return memcmp(guid_a, guid_b, sizeof(ggml_guid)) == 0;
512}
513
514const char * ggml_version(void) {
515 return GGML_VERSION"GGML_VERSION";
516}
517
518const char * ggml_commit(void) {
519 return GGML_COMMIT"GGML_COMMIT";
520}
521
522//
523// timing
524//
525
526#if defined(_MSC_VER) || defined(__MINGW32__)
527static int64_t timer_freq, timer_start;
528void ggml_time_init(void) {
529 LARGE_INTEGER t;
530 QueryPerformanceFrequency(&t);
531 timer_freq = t.QuadPart;
532
533 // The multiplication by 1000 or 1000000 below can cause an overflow if timer_freq
534 // and the uptime is high enough.
535 // We subtract the program start time to reduce the likelihood of that happening.
536 QueryPerformanceCounter(&t);
537 timer_start = t.QuadPart;
538}
539int64_t ggml_time_ms(void) {
540 LARGE_INTEGER t;
541 QueryPerformanceCounter(&t);
542 return ((t.QuadPart-timer_start) * 1000) / timer_freq;
543}
544int64_t ggml_time_us(void) {
545 LARGE_INTEGER t;
546 QueryPerformanceCounter(&t);
547 return ((t.QuadPart-timer_start) * 1000000) / timer_freq;
548}
549#else
550void ggml_time_init(void) {}
551int64_t ggml_time_ms(void) {
552 struct timespec ts;
553 clock_gettime(CLOCK_MONOTONIC1, &ts);
554 return (int64_t)ts.tv_sec*1000 + (int64_t)ts.tv_nsec/1000000;
555}
556
557int64_t ggml_time_us(void) {
558 struct timespec ts;
559 clock_gettime(CLOCK_MONOTONIC1, &ts);
560 return (int64_t)ts.tv_sec*1000000 + (int64_t)ts.tv_nsec/1000;
561}
562#endif
563
564int64_t ggml_cycles(void) {
565 return clock();
566}
567
568int64_t ggml_cycles_per_ms(void) {
569 return CLOCKS_PER_SEC((__clock_t) 1000000)/1000;
570}
571
572//
573// cross-platform UTF-8 file paths
574//
575
576#ifdef _WIN32
577static wchar_t * ggml_mbstowcs(const char * mbs) {
578 int wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, NULL((void*)0), 0);
579 if (!wlen) {
580 errno(*__errno_location ()) = EINVAL22;
581 return NULL((void*)0);
582 }
583
584 wchar_t * wbuf = GGML_MALLOC(wlen * sizeof(wchar_t))ggml_malloc(wlen * sizeof(wchar_t));
585 wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, wbuf, wlen);
586 if (!wlen) {
587 GGML_FREE(wbuf)free(wbuf);
588 errno(*__errno_location ()) = EINVAL22;
589 return NULL((void*)0);
590 }
591
592 return wbuf;
593}
594#endif
595
596FILE * ggml_fopen(const char * fname, const char * mode) {
597#ifdef _WIN32
598 FILE * file = NULL((void*)0);
599
600 // convert fname (UTF-8)
601 wchar_t * wfname = ggml_mbstowcs(fname);
602 if (wfname) {
603 // convert mode (ANSI)
604 wchar_t * wmode = GGML_MALLOC((strlen(mode) + 1) * sizeof(wchar_t))ggml_malloc((strlen(mode) + 1) * sizeof(wchar_t));
605 wchar_t * wmode_p = wmode;
606 do {
607 *wmode_p++ = (wchar_t)*mode;
608 } while (*mode++);
609
610 // open file
611 file = _wfopen(wfname, wmode);
612
613 GGML_FREE(wfname)free(wfname);
614 GGML_FREE(wmode)free(wmode);
615 }
616
617 return file;
618#else
619 return fopen(fname, mode);
620#endif
621
622}
623
624static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {
625 [GGML_TYPE_I8] = {
626 .type_name = "i8",
627 .blck_size = 1,
628 .type_size = sizeof(int8_t),
629 .is_quantized = false0,
630 },
631 [GGML_TYPE_I16] = {
632 .type_name = "i16",
633 .blck_size = 1,
634 .type_size = sizeof(int16_t),
635 .is_quantized = false0,
636 },
637 [GGML_TYPE_I32] = {
638 .type_name = "i32",
639 .blck_size = 1,
640 .type_size = sizeof(int32_t),
641 .is_quantized = false0,
642 },
643 [GGML_TYPE_I64] = {
644 .type_name = "i64",
645 .blck_size = 1,
646 .type_size = sizeof(int64_t),
647 .is_quantized = false0,
648 },
649 [GGML_TYPE_F64] = {
650 .type_name = "f64",
651 .blck_size = 1,
652 .type_size = sizeof(double),
653 .is_quantized = false0,
654 },
655 [GGML_TYPE_F32] = {
656 .type_name = "f32",
657 .blck_size = 1,
658 .type_size = sizeof(float),
659 .is_quantized = false0,
660 },
661 [GGML_TYPE_F16] = {
662 .type_name = "f16",
663 .blck_size = 1,
664 .type_size = sizeof(ggml_fp16_t),
665 .is_quantized = false0,
666 .to_float = (ggml_to_float_t) ggml_fp16_to_fp32_row,
667 .from_float_ref = (ggml_from_float_t) ggml_fp32_to_fp16_row,
668 },
669 [GGML_TYPE_Q1_0] = {
670 .type_name = "q1_0",
671 .blck_size = QK1_0128,
672 .type_size = sizeof(block_q1_0),
673 .is_quantized = true1,
674 .to_float = (ggml_to_float_t) dequantize_row_q1_0,
675 .from_float_ref = (ggml_from_float_t) quantize_row_q1_0_ref,
676 },
677 [GGML_TYPE_Q4_0] = {
678 .type_name = "q4_0",
679 .blck_size = QK4_032,
680 .type_size = sizeof(block_q4_0),
681 .is_quantized = true1,
682 .to_float = (ggml_to_float_t) dequantize_row_q4_0,
683 .from_float_ref = (ggml_from_float_t) quantize_row_q4_0_ref,
684 },
685 [GGML_TYPE_Q4_1] = {
686 .type_name = "q4_1",
687 .blck_size = QK4_132,
688 .type_size = sizeof(block_q4_1),
689 .is_quantized = true1,
690 .to_float = (ggml_to_float_t) dequantize_row_q4_1,
691 .from_float_ref = (ggml_from_float_t) quantize_row_q4_1_ref,
692 },
693 [4] = { // GGML_TYPE_Q4_2
694 .type_name = "DEPRECATED",
695 .blck_size = 0,
696 .type_size = 0,
697 .is_quantized = false0,
698 },
699 [5] = { // GGML_TYPE_Q4_3
700 .type_name = "DEPRECATED",
701 .blck_size = 0,
702 .type_size = 0,
703 .is_quantized = false0,
704 },
705 [GGML_TYPE_Q5_0] = {
706 .type_name = "q5_0",
707 .blck_size = QK5_032,
708 .type_size = sizeof(block_q5_0),
709 .is_quantized = true1,
710 .to_float = (ggml_to_float_t) dequantize_row_q5_0,
711 .from_float_ref = (ggml_from_float_t) quantize_row_q5_0_ref,
712 },
713 [GGML_TYPE_Q5_1] = {
714 .type_name = "q5_1",
715 .blck_size = QK5_132,
716 .type_size = sizeof(block_q5_1),
717 .is_quantized = true1,
718 .to_float = (ggml_to_float_t) dequantize_row_q5_1,
719 .from_float_ref = (ggml_from_float_t) quantize_row_q5_1_ref,
720 },
721 [GGML_TYPE_Q8_0] = {
722 .type_name = "q8_0",
723 .blck_size = QK8_032,
724 .type_size = sizeof(block_q8_0),
725 .is_quantized = true1,
726 .to_float = (ggml_to_float_t) dequantize_row_q8_0,
727 .from_float_ref = (ggml_from_float_t) quantize_row_q8_0_ref,
728 },
729 [GGML_TYPE_Q8_1] = {
730 .type_name = "q8_1",
731 .blck_size = QK8_132,
732 .type_size = sizeof(block_q8_1),
733 .is_quantized = true1,
734 .from_float_ref = (ggml_from_float_t) quantize_row_q8_1_ref,
735 },
736 [GGML_TYPE_MXFP4] = {
737 .type_name = "mxfp4",
738 .blck_size = QK_MXFP432,
739 .type_size = sizeof(block_mxfp4),
740 .is_quantized = true1,
741 .to_float = (ggml_to_float_t) dequantize_row_mxfp4,
742 .from_float_ref = (ggml_from_float_t)quantize_row_mxfp4_ref,
743 },
744 [GGML_TYPE_NVFP4] = {
745 .type_name = "nvfp4",
746 .blck_size = QK_NVFP464,
747 .type_size = sizeof(block_nvfp4),
748 .is_quantized = true1,
749 .to_float = (ggml_to_float_t) dequantize_row_nvfp4,
750 .from_float_ref = (ggml_from_float_t)quantize_row_nvfp4_ref,
751 },
752 [GGML_TYPE_Q2_K] = {
753 .type_name = "q2_K",
754 .blck_size = QK_K256,
755 .type_size = sizeof(block_q2_K),
756 .is_quantized = true1,
757 .to_float = (ggml_to_float_t) dequantize_row_q2_K,
758 .from_float_ref = (ggml_from_float_t) quantize_row_q2_K_ref,
759 },
760 [GGML_TYPE_Q3_K] = {
761 .type_name = "q3_K",
762 .blck_size = QK_K256,
763 .type_size = sizeof(block_q3_K),
764 .is_quantized = true1,
765 .to_float = (ggml_to_float_t) dequantize_row_q3_K,
766 .from_float_ref = (ggml_from_float_t) quantize_row_q3_K_ref,
767 },
768 [GGML_TYPE_Q4_K] = {
769 .type_name = "q4_K",
770 .blck_size = QK_K256,
771 .type_size = sizeof(block_q4_K),
772 .is_quantized = true1,
773 .to_float = (ggml_to_float_t) dequantize_row_q4_K,
774 .from_float_ref = (ggml_from_float_t) quantize_row_q4_K_ref,
775 },
776 [GGML_TYPE_Q5_K] = {
777 .type_name = "q5_K",
778 .blck_size = QK_K256,
779 .type_size = sizeof(block_q5_K),
780 .is_quantized = true1,
781 .to_float = (ggml_to_float_t) dequantize_row_q5_K,
782 .from_float_ref = (ggml_from_float_t) quantize_row_q5_K_ref,
783 },
784 [GGML_TYPE_Q6_K] = {
785 .type_name = "q6_K",
786 .blck_size = QK_K256,
787 .type_size = sizeof(block_q6_K),
788 .is_quantized = true1,
789 .to_float = (ggml_to_float_t) dequantize_row_q6_K,
790 .from_float_ref = (ggml_from_float_t) quantize_row_q6_K_ref,
791 },
792 [GGML_TYPE_IQ2_XXS] = {
793 .type_name = "iq2_xxs",
794 .blck_size = QK_K256,
795 .type_size = sizeof(block_iq2_xxs),
796 .is_quantized = true1,
797 .to_float = (ggml_to_float_t) dequantize_row_iq2_xxs,
798 .from_float_ref = NULL((void*)0),
799 },
800 [GGML_TYPE_IQ2_XS] = {
801 .type_name = "iq2_xs",
802 .blck_size = QK_K256,
803 .type_size = sizeof(block_iq2_xs),
804 .is_quantized = true1,
805 .to_float = (ggml_to_float_t) dequantize_row_iq2_xs,
806 .from_float_ref = NULL((void*)0),
807 },
808 [GGML_TYPE_IQ3_XXS] = {
809 .type_name = "iq3_xxs",
810 .blck_size = QK_K256,
811 .type_size = sizeof(block_iq3_xxs),
812 .is_quantized = true1,
813 .to_float = (ggml_to_float_t) dequantize_row_iq3_xxs,
814 .from_float_ref = (ggml_from_float_t)quantize_row_iq3_xxs_ref,
815 },
816 [GGML_TYPE_IQ3_S] = {
817 .type_name = "iq3_s",
818 .blck_size = QK_K256,
819 .type_size = sizeof(block_iq3_s),
820 .is_quantized = true1,
821 .to_float = (ggml_to_float_t) dequantize_row_iq3_s,
822 .from_float_ref = (ggml_from_float_t)quantize_row_iq3_s_ref,
823 },
824 [GGML_TYPE_IQ2_S] = {
825 .type_name = "iq2_s",
826 .blck_size = QK_K256,
827 .type_size = sizeof(block_iq2_s),
828 .is_quantized = true1,
829 .to_float = (ggml_to_float_t) dequantize_row_iq2_s,
830 .from_float_ref = (ggml_from_float_t)quantize_row_iq2_s_ref,
831 },
832 [GGML_TYPE_IQ1_S] = {
833 .type_name = "iq1_s",
834 .blck_size = QK_K256,
835 .type_size = sizeof(block_iq1_s),
836 .is_quantized = true1,
837 .to_float = (ggml_to_float_t) dequantize_row_iq1_s,
838 .from_float_ref = NULL((void*)0),
839 },
840 [GGML_TYPE_IQ1_M] = {
841 .type_name = "iq1_m",
842 .blck_size = QK_K256,
843 .type_size = sizeof(block_iq1_m),
844 .is_quantized = true1,
845 .to_float = (ggml_to_float_t) dequantize_row_iq1_m,
846 .from_float_ref = NULL((void*)0),
847 },
848 [GGML_TYPE_IQ4_NL] = {
849 .type_name = "iq4_nl",
850 .blck_size = QK4_NL32,
851 .type_size = sizeof(block_iq4_nl),
852 .is_quantized = true1,
853 .to_float = (ggml_to_float_t) dequantize_row_iq4_nl,
854 .from_float_ref = (ggml_from_float_t)quantize_row_iq4_nl_ref,
855 },
856 [GGML_TYPE_IQ4_XS] = {
857 .type_name = "iq4_xs",
858 .blck_size = QK_K256,
859 .type_size = sizeof(block_iq4_xs),
860 .is_quantized = true1,
861 .to_float = (ggml_to_float_t) dequantize_row_iq4_xs,
862 .from_float_ref = (ggml_from_float_t)quantize_row_iq4_xs_ref,
863 },
864 [GGML_TYPE_Q8_K] = {
865 .type_name = "q8_K",
866 .blck_size = QK_K256,
867 .type_size = sizeof(block_q8_K),
868 .is_quantized = true1,
869 },
870 [GGML_TYPE_BF16] = {
871 .type_name = "bf16",
872 .blck_size = 1,
873 .type_size = sizeof(ggml_bf16_t),
874 .is_quantized = false0,
875 .to_float = (ggml_to_float_t) ggml_bf16_to_fp32_row,
876 .from_float_ref = (ggml_from_float_t) ggml_fp32_to_bf16_row_ref,
877 },
878 [31] = { // GGML_TYPE_Q4_0_4_4
879 .type_name = "TYPE_Q4_0_4_4 REMOVED, use Q4_0 with runtime repacking",
880 .blck_size = 0,
881 .type_size = 0,
882 .is_quantized = false0,
883 },
884 [32] = { // GGML_TYPE_Q4_0_4_8
885 .type_name = "TYPE_Q4_0_4_8 REMOVED, use Q4_0 with runtime repacking",
886 .blck_size = 0,
887 .type_size = 0,
888 .is_quantized = false0,
889 },
890 [33] = { // GGML_TYPE_Q4_0_8_8
891 .type_name = "TYPE_Q4_0_8_8 REMOVED, use Q4_0 with runtime repacking",
892 .blck_size = 0,
893 .type_size = 0,
894 .is_quantized = false0,
895 },
896 [GGML_TYPE_TQ1_0] = {
897 .type_name = "tq1_0",
898 .blck_size = QK_K256,
899 .type_size = sizeof(block_tq1_0),
900 .is_quantized = true1,
901 .to_float = (ggml_to_float_t) dequantize_row_tq1_0,
902 .from_float_ref = (ggml_from_float_t) quantize_row_tq1_0_ref,
903 },
904 [GGML_TYPE_TQ2_0] = {
905 .type_name = "tq2_0",
906 .blck_size = QK_K256,
907 .type_size = sizeof(block_tq2_0),
908 .is_quantized = true1,
909 .to_float = (ggml_to_float_t) dequantize_row_tq2_0,
910 .from_float_ref = (ggml_from_float_t) quantize_row_tq2_0_ref,
911 },
912 [36] = { // GGML_TYPE_IQ4_NL_4_4
913 .type_name = "TYPE_IQ4_NL_4_4 REMOVED, use IQ4_NL with runtime repacking",
914 .blck_size = 0,
915 .type_size = 0,
916 .is_quantized = false0,
917 },
918 [37] = { // GGML_TYPE_IQ4_NL_4_8
919 .type_name = "TYPE_IQ4_NL_4_8 REMOVED, use IQ4_NL with runtime repacking",
920 .blck_size = 0,
921 .type_size = 0,
922 .is_quantized = false0,
923 },
924 [38] = { // GGML_TYPE_IQ4_NL_8_8
925 .type_name = "TYPE_IQ4_NL_8_8 REMOVED, use IQ4_NL with runtime repacking",
926 .blck_size = 0,
927 .type_size = 0,
928 .is_quantized = false0,
929 },
930};
931
932const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type) {
933 assert(type >= 0)((void) sizeof (__assert_single_arg (type >= 0)), __extension__
({ if (type >= 0) ; else __assert_fail ("type >= 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 933, __extension__ __PRETTY_FUNCTION__); }));
934 assert(type < GGML_TYPE_COUNT)((void) sizeof (__assert_single_arg (type < GGML_TYPE_COUNT
)), __extension__ ({ if (type < GGML_TYPE_COUNT) ; else __assert_fail
("type < GGML_TYPE_COUNT", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 934, __extension__ __PRETTY_FUNCTION__); }));
935 return &type_traits[type];
936}
937
938//
939// ggml object
940//
941
942struct ggml_object {
943 size_t offs;
944 size_t size;
945
946 struct ggml_object * next;
947
948 enum ggml_object_type type;
949
950 char padding[4];
951};
952
953static const size_t GGML_OBJECT_SIZE = sizeof(struct ggml_object);
954
955//
956// ggml context
957//
958
959struct ggml_context {
960 size_t mem_size;
961 void * mem_buffer;
962 bool_Bool mem_buffer_owned;
963 bool_Bool no_alloc;
964
965 int n_objects;
966
967 struct ggml_object * objects_begin;
968 struct ggml_object * objects_end;
969};
970
971//
972// data types
973//
974
975static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
976 "NONE",
977
978 "DUP",
979 "ADD",
980 "ADD_ID",
981 "ADD1",
982 "ACC",
983 "SUB",
984 "MUL",
985 "DIV",
986 "SQR",
987 "SQRT",
988 "LOG",
989 "SIN",
990 "COS",
991 "SUM",
992 "SUM_ROWS",
993 "CUMSUM",
994 "MEAN",
995 "ARGMAX",
996 "COUNT_EQUAL",
997 "REPEAT",
998 "REPEAT_BACK",
999 "CONCAT",
1000 "SILU_BACK",
1001 "NORM",
1002 "RMS_NORM",
1003 "RMS_NORM_BACK",
1004 "GROUP_NORM",
1005 "L2_NORM",
1006
1007 "MUL_MAT",
1008 "MUL_MAT_ID",
1009 "OUT_PROD",
1010
1011 "SCALE",
1012 "SET",
1013 "CPY",
1014 "CONT",
1015 "RESHAPE",
1016 "VIEW",
1017 "PERMUTE",
1018 "TRANSPOSE",
1019 "GET_ROWS",
1020 "GET_ROWS_BACK",
1021 "SET_ROWS",
1022 "DIAG",
1023 "DIAG_MASK_INF",
1024 "DIAG_MASK_ZERO",
1025 "SOFT_MAX",
1026 "SOFT_MAX_BACK",
1027 "ROPE",
1028 "ROPE_BACK",
1029 "CLAMP",
1030 "CONV_TRANSPOSE_1D",
1031 "IM2COL",
1032 "IM2COL_BACK",
1033 "IM2COL_3D",
1034 "COL2IM_1D",
1035 "CONV_2D",
1036 "CONV_3D",
1037 "CONV_2D_DW",
1038 "CONV_TRANSPOSE_2D",
1039 "POOL_1D",
1040 "POOL_2D",
1041 "POOL_2D_BACK",
1042 "UPSCALE",
1043 "PAD",
1044 "PAD_REFLECT_1D",
1045 "ROLL",
1046 "ARANGE",
1047 "TIMESTEP_EMBEDDING",
1048 "ARGSORT",
1049 "TOP_K",
1050 "LEAKY_RELU",
1051 "TRI",
1052 "FILL",
1053
1054 "FLASH_ATTN_EXT",
1055 "FLASH_ATTN_BACK",
1056 "SSM_CONV",
1057 "SSM_SCAN",
1058 "WIN_PART",
1059 "WIN_UNPART",
1060 "GET_REL_POS",
1061 "ADD_REL_POS",
1062 "RWKV_WKV6",
1063 "GATED_LINEAR_ATTN",
1064 "RWKV_WKV7",
1065 "SOLVE_TRI",
1066 "GATED_DELTA_NET",
1067
1068 "UNARY",
1069
1070 "MAP_CUSTOM1",
1071 "MAP_CUSTOM2",
1072 "MAP_CUSTOM3",
1073
1074 "CUSTOM",
1075
1076 "CROSS_ENTROPY_LOSS",
1077 "CROSS_ENTROPY_LOSS_BACK",
1078 "OPT_STEP_ADAMW",
1079 "OPT_STEP_SGD",
1080
1081 "GLU",
1082};
1083
1084static_assert_Static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97");
1085
1086static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
1087 "none",
1088
1089 "x",
1090 "x+y",
1091 "x[i]+y",
1092 "x+y",
1093 "view(x,nb,offset)+=y->x",
1094 "x-y",
1095 "x*y",
1096 "x/y",
1097 "x^2",
1098 "√x",
1099 "log(x)",
1100 "sin(x)",
1101 "cos(x)",
1102 "Σx",
1103 "Σx_k",
1104 "cumsum(x)",
1105 "Σx/n",
1106 "argmax(x)",
1107 "count_equal(x)",
1108 "repeat(x)",
1109 "repeat_back(x)",
1110 "concat(x, y)",
1111 "silu_back(x)",
1112 "norm(x)",
1113 "rms_norm(x)",
1114 "rms_norm_back(x)",
1115 "group_norm(x)",
1116 "l2_norm(x)",
1117
1118 "X*Y",
1119 "X[i]*Y",
1120 "X*Y",
1121
1122 "x*v",
1123 "y-\\>view(x)",
1124 "x-\\>y",
1125 "cont(x)",
1126 "reshape(x)",
1127 "view(x)",
1128 "permute(x)",
1129 "transpose(x)",
1130 "get_rows(x)",
1131 "get_rows_back(x)",
1132 "set_rows(x)",
1133 "diag(x)",
1134 "diag_mask_inf(x)",
1135 "diag_mask_zero(x)",
1136 "soft_max(x)",
1137 "soft_max_back(x)",
1138 "rope(x)",
1139 "rope_back(x)",
1140 "clamp(x)",
1141 "conv_transpose_1d(x)",
1142 "im2col(x)",
1143 "im2col_back(x)",
1144 "im2col_3d(x)",
1145 "col2im_1d(x)",
1146 "conv_2d(x)",
1147 "conv_3d(x)",
1148 "conv_2d_dw(x)",
1149 "conv_transpose_2d(x)",
1150 "pool_1d(x)",
1151 "pool_2d(x)",
1152 "pool_2d_back(x)",
1153 "upscale(x)",
1154 "pad(x)",
1155 "pad_reflect_1d(x)",
1156 "roll(x)",
1157 "arange(start, stop, step)",
1158 "timestep_embedding(timesteps, dim, max_period)",
1159 "argsort(x)",
1160 "top_k(x)",
1161 "leaky_relu(x)",
1162 "tri(x)",
1163 "fill(x, c)",
1164
1165 "flash_attn_ext(x)",
1166 "flash_attn_back(x)",
1167 "ssm_conv(x)",
1168 "ssm_scan(x)",
1169 "win_part(x)",
1170 "win_unpart(x)",
1171 "get_rel_pos(x)",
1172 "add_rel_pos(x)",
1173 "rwkv_wkv6(k, v, r, tf, td, s)",
1174 "gated_linear_attn(k, v, q, gate, s)",
1175 "rwkv_wkv7(r, w, k, v, a, b, s)",
1176 "A X = B, A triangular, solve X",
1177 "gated_delta_net(q, k, v, g, beta, s)",
1178
1179 "unary(x)",
1180
1181 "map_custom(x)",
1182 "map_custom(x,y)",
1183 "map_custom(x,y,z)",
1184
1185 "custom(x)",
1186
1187 "cross_entropy_loss(x,y)",
1188 "cross_entropy_loss_back(x,y)",
1189 "adamw(x)",
1190 "sgd(x)",
1191
1192 "glu(x)",
1193};
1194
1195static_assert_Static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97");
1196
1197static_assert_Static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
1198
1199static const char * GGML_UNARY_OP_NAME[GGML_UNARY_OP_COUNT] = {
1200 "ABS",
1201 "SGN",
1202 "NEG",
1203 "STEP",
1204 "TANH",
1205 "ELU",
1206 "RELU",
1207 "SIGMOID",
1208 "GELU",
1209 "GELU_QUICK",
1210 "SILU",
1211 "HARDSWISH",
1212 "HARDSIGMOID",
1213 "EXP",
1214 "EXPM1",
1215 "SOFTPLUS",
1216 "GELU_ERF",
1217 "XIELU",
1218 "FLOOR",
1219 "CEIL",
1220 "ROUND",
1221 "TRUNC",
1222};
1223
1224static_assert_Static_assert(GGML_UNARY_OP_COUNT == 22, "GGML_UNARY_OP_COUNT != 22");
1225
1226static const char * GGML_GLU_OP_NAME[GGML_GLU_OP_COUNT] = {
1227 "REGLU",
1228 "GEGLU",
1229 "SWIGLU",
1230 "SWIGLU_OAI",
1231 "GEGLU_ERF",
1232 "GEGLU_QUICK",
1233};
1234
1235static_assert_Static_assert(GGML_GLU_OP_COUNT == 6, "GGML_GLU_OP_COUNT != 6");
1236
1237
1238static_assert_Static_assert(sizeof(struct ggml_object)%GGML_MEM_ALIGN16 == 0, "ggml_object size must be a multiple of GGML_MEM_ALIGN");
1239static_assert_Static_assert(sizeof(struct ggml_tensor)%GGML_MEM_ALIGN16 == 0, "ggml_tensor size must be a multiple of GGML_MEM_ALIGN");
1240
1241
1242////////////////////////////////////////////////////////////////////////////////
1243
1244void ggml_print_object(const struct ggml_object * obj) {
1245 GGML_LOG_INFO(" - ggml_object: type = %d, offset = %zu, size = %zu, next = %p\n",ggml_log_internal(GGML_LOG_LEVEL_INFO , " - ggml_object: type = %d, offset = %zu, size = %zu, next = %p\n"
, obj->type, obj->offs, obj->size, (const void *) obj
->next)
1246 obj->type, obj->offs, obj->size, (const void *) obj->next)ggml_log_internal(GGML_LOG_LEVEL_INFO , " - ggml_object: type = %d, offset = %zu, size = %zu, next = %p\n"
, obj->type, obj->offs, obj->size, (const void *) obj
->next);
1247}
1248
1249void ggml_print_objects(const struct ggml_context * ctx) {
1250 struct ggml_object * obj = ctx->objects_begin;
1251
1252 GGML_LOG_INFO("%s: objects in context %p:\n", __func__, (const void *) ctx)ggml_log_internal(GGML_LOG_LEVEL_INFO , "%s: objects in context %p:\n"
, __func__, (const void *) ctx);
1253
1254 while (obj != NULL((void*)0)) {
1255 ggml_print_object(obj);
1256 obj = obj->next;
1257 }
1258
1259 GGML_LOG_INFO("%s: --- end ---\n", __func__)ggml_log_internal(GGML_LOG_LEVEL_INFO , "%s: --- end ---\n", __func__
);
1260}
1261
1262int64_t ggml_nelements(const struct ggml_tensor * tensor) {
1263 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
1264
1265 return tensor->ne[0]*tensor->ne[1]*tensor->ne[2]*tensor->ne[3];
1266}
1267
1268int64_t ggml_nrows(const struct ggml_tensor * tensor) {
1269 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
1270
1271 return tensor->ne[1]*tensor->ne[2]*tensor->ne[3];
1272}
1273
1274size_t ggml_nbytes(const struct ggml_tensor * tensor) {
1275 for (int i = 0; i < GGML_MAX_DIMS4; ++i) {
1276 if (tensor->ne[i] <= 0) {
1277 return 0;
1278 }
1279 }
1280
1281 size_t nbytes;
1282 const size_t blck_size = ggml_blck_size(tensor->type);
1283 if (blck_size == 1) {
1284 nbytes = ggml_type_size(tensor->type);
1285 for (int i = 0; i < GGML_MAX_DIMS4; ++i) {
1286 nbytes += (tensor->ne[i] - 1)*tensor->nb[i];
1287 }
1288 }
1289 else {
1290 nbytes = tensor->ne[0]*tensor->nb[0]/blck_size;
1291 for (int i = 1; i < GGML_MAX_DIMS4; ++i) {
1292 nbytes += (tensor->ne[i] - 1)*tensor->nb[i];
1293 }
1294 }
1295
1296 return nbytes;
1297}
1298
1299size_t ggml_nbytes_pad(const struct ggml_tensor * tensor) {
1300 return GGML_PAD(ggml_nbytes(tensor), GGML_MEM_ALIGN)(((ggml_nbytes(tensor)) + (16) - 1) & ~((16) - 1));
1301}
1302
1303int64_t ggml_blck_size(enum ggml_type type) {
1304 assert(type >= 0)((void) sizeof (__assert_single_arg (type >= 0)), __extension__
({ if (type >= 0) ; else __assert_fail ("type >= 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1304, __extension__ __PRETTY_FUNCTION__); }));
1305 assert(type < GGML_TYPE_COUNT)((void) sizeof (__assert_single_arg (type < GGML_TYPE_COUNT
)), __extension__ ({ if (type < GGML_TYPE_COUNT) ; else __assert_fail
("type < GGML_TYPE_COUNT", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1305, __extension__ __PRETTY_FUNCTION__); }));
1306 return type_traits[type].blck_size;
1307}
1308
1309size_t ggml_type_size(enum ggml_type type) {
1310 assert(type >= 0)((void) sizeof (__assert_single_arg (type >= 0)), __extension__
({ if (type >= 0) ; else __assert_fail ("type >= 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1310, __extension__ __PRETTY_FUNCTION__); }));
1311 assert(type < GGML_TYPE_COUNT)((void) sizeof (__assert_single_arg (type < GGML_TYPE_COUNT
)), __extension__ ({ if (type < GGML_TYPE_COUNT) ; else __assert_fail
("type < GGML_TYPE_COUNT", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1311, __extension__ __PRETTY_FUNCTION__); }));
1312 return type_traits[type].type_size;
1313}
1314
1315size_t ggml_row_size(enum ggml_type type, int64_t ne) {
1316 assert(type >= 0)((void) sizeof (__assert_single_arg (type >= 0)), __extension__
({ if (type >= 0) ; else __assert_fail ("type >= 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1316, __extension__ __PRETTY_FUNCTION__); }));
1317 assert(type < GGML_TYPE_COUNT)((void) sizeof (__assert_single_arg (type < GGML_TYPE_COUNT
)), __extension__ ({ if (type < GGML_TYPE_COUNT) ; else __assert_fail
("type < GGML_TYPE_COUNT", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1317, __extension__ __PRETTY_FUNCTION__); }));
1318 assert(ne % ggml_blck_size(type) == 0)((void) sizeof (__assert_single_arg (ne % ggml_blck_size(type
) == 0)), __extension__ ({ if (ne % ggml_blck_size(type) == 0
) ; else __assert_fail ("ne % ggml_blck_size(type) == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1318, __extension__ __PRETTY_FUNCTION__); }));
1319 return ggml_type_size(type)*ne/ggml_blck_size(type);
1320}
1321
1322double ggml_type_sizef(enum ggml_type type) {
1323 assert(type >= 0)((void) sizeof (__assert_single_arg (type >= 0)), __extension__
({ if (type >= 0) ; else __assert_fail ("type >= 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1323, __extension__ __PRETTY_FUNCTION__); }));
1324 assert(type < GGML_TYPE_COUNT)((void) sizeof (__assert_single_arg (type < GGML_TYPE_COUNT
)), __extension__ ({ if (type < GGML_TYPE_COUNT) ; else __assert_fail
("type < GGML_TYPE_COUNT", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1324, __extension__ __PRETTY_FUNCTION__); }));
1325 return ((double)(type_traits[type].type_size))/type_traits[type].blck_size;
1326}
1327
1328const char * ggml_type_name(enum ggml_type type) {
1329 assert(type >= 0)((void) sizeof (__assert_single_arg (type >= 0)), __extension__
({ if (type >= 0) ; else __assert_fail ("type >= 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1329, __extension__ __PRETTY_FUNCTION__); }));
1330 assert(type < GGML_TYPE_COUNT)((void) sizeof (__assert_single_arg (type < GGML_TYPE_COUNT
)), __extension__ ({ if (type < GGML_TYPE_COUNT) ; else __assert_fail
("type < GGML_TYPE_COUNT", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1330, __extension__ __PRETTY_FUNCTION__); }));
1331 return type_traits[type].type_name;
1332}
1333
1334bool_Bool ggml_is_quantized(enum ggml_type type) {
1335 assert(type >= 0)((void) sizeof (__assert_single_arg (type >= 0)), __extension__
({ if (type >= 0) ; else __assert_fail ("type >= 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1335, __extension__ __PRETTY_FUNCTION__); }));
1336 assert(type < GGML_TYPE_COUNT)((void) sizeof (__assert_single_arg (type < GGML_TYPE_COUNT
)), __extension__ ({ if (type < GGML_TYPE_COUNT) ; else __assert_fail
("type < GGML_TYPE_COUNT", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1336, __extension__ __PRETTY_FUNCTION__); }));
1337 return type_traits[type].is_quantized;
1338}
1339
1340const char * ggml_op_name(enum ggml_op op) {
1341 return GGML_OP_NAME[op];
1342}
1343
1344const char * ggml_op_symbol(enum ggml_op op) {
1345 return GGML_OP_SYMBOL[op];
1346}
1347
1348const char * ggml_unary_op_name(enum ggml_unary_op op) {
1349 return GGML_UNARY_OP_NAME[op];
1350}
1351
1352const char * ggml_glu_op_name(enum ggml_glu_op op) {
1353 return GGML_GLU_OP_NAME[op];
1354}
1355
1356const char * ggml_op_desc(const struct ggml_tensor * t) {
1357 if (t->op == GGML_OP_UNARY) {
1358 enum ggml_unary_op uop = ggml_get_unary_op(t);
1359 return ggml_unary_op_name(uop);
1360 }
1361 if (t->op == GGML_OP_GLU) {
1362 enum ggml_glu_op gop = ggml_get_glu_op(t);
1363 return ggml_glu_op_name(gop);
1364 }
1365 return ggml_op_name(t->op);
1366}
1367
1368size_t ggml_element_size(const struct ggml_tensor * tensor) {
1369 return ggml_type_size(tensor->type);
1370}
1371
1372bool_Bool ggml_is_scalar(const struct ggml_tensor * tensor) {
1373 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
1374
1375 return tensor->ne[0] == 1 && tensor->ne[1] == 1 && tensor->ne[2] == 1 && tensor->ne[3] == 1;
1376}
1377
1378bool_Bool ggml_is_vector(const struct ggml_tensor * tensor) {
1379 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
1380
1381 return tensor->ne[1] == 1 && tensor->ne[2] == 1 && tensor->ne[3] == 1;
1382}
1383
1384bool_Bool ggml_is_matrix(const struct ggml_tensor * tensor) {
1385 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
1386
1387 return tensor->ne[2] == 1 && tensor->ne[3] == 1;
1388}
1389
1390bool_Bool ggml_is_3d(const struct ggml_tensor * tensor) {
1391 return tensor->ne[3] == 1;
1392}
1393
1394int ggml_n_dims(const struct ggml_tensor * tensor) {
1395 for (int i = GGML_MAX_DIMS4 - 1; i >= 1; --i) {
1396 if (tensor->ne[i] > 1) {
1397 return i + 1;
1398 }
1399 }
1400 return 1;
1401}
1402
1403enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) {
1404 enum ggml_type wtype = GGML_TYPE_COUNT;
1405
1406 switch (ftype) {
1407 case GGML_FTYPE_ALL_F32: wtype = GGML_TYPE_F32; break;
1408 case GGML_FTYPE_MOSTLY_F16: wtype = GGML_TYPE_F16; break;
1409 case GGML_FTYPE_MOSTLY_BF16: wtype = GGML_TYPE_BF16; break;
1410 case GGML_FTYPE_MOSTLY_Q4_0: wtype = GGML_TYPE_Q4_0; break;
1411 case GGML_FTYPE_MOSTLY_Q4_1: wtype = GGML_TYPE_Q4_1; break;
1412 case GGML_FTYPE_MOSTLY_Q1_0: wtype = GGML_TYPE_Q1_0; break;
1413 case GGML_FTYPE_MOSTLY_Q5_0: wtype = GGML_TYPE_Q5_0; break;
1414 case GGML_FTYPE_MOSTLY_Q5_1: wtype = GGML_TYPE_Q5_1; break;
1415 case GGML_FTYPE_MOSTLY_Q8_0: wtype = GGML_TYPE_Q8_0; break;
1416 case GGML_FTYPE_MOSTLY_MXFP4: wtype = GGML_TYPE_MXFP4; break;
1417 case GGML_FTYPE_MOSTLY_NVFP4: wtype = GGML_TYPE_NVFP4; break;
1418 case GGML_FTYPE_MOSTLY_Q2_K: wtype = GGML_TYPE_Q2_K; break;
1419 case GGML_FTYPE_MOSTLY_Q3_K: wtype = GGML_TYPE_Q3_K; break;
1420 case GGML_FTYPE_MOSTLY_Q4_K: wtype = GGML_TYPE_Q4_K; break;
1421 case GGML_FTYPE_MOSTLY_Q5_K: wtype = GGML_TYPE_Q5_K; break;
1422 case GGML_FTYPE_MOSTLY_Q6_K: wtype = GGML_TYPE_Q6_K; break;
1423 case GGML_FTYPE_MOSTLY_IQ2_XXS: wtype = GGML_TYPE_IQ2_XXS; break;
1424 case GGML_FTYPE_MOSTLY_IQ2_XS: wtype = GGML_TYPE_IQ2_XS; break;
1425 case GGML_FTYPE_MOSTLY_IQ3_XXS: wtype = GGML_TYPE_IQ3_XXS; break;
1426 case GGML_FTYPE_MOSTLY_IQ1_S: wtype = GGML_TYPE_IQ1_S; break;
1427 case GGML_FTYPE_MOSTLY_IQ1_M: wtype = GGML_TYPE_IQ1_M; break;
1428 case GGML_FTYPE_MOSTLY_IQ4_NL: wtype = GGML_TYPE_IQ4_NL; break;
1429 case GGML_FTYPE_MOSTLY_IQ4_XS: wtype = GGML_TYPE_IQ4_XS; break;
1430 case GGML_FTYPE_MOSTLY_IQ3_S: wtype = GGML_TYPE_IQ3_S; break;
1431 case GGML_FTYPE_MOSTLY_IQ2_S: wtype = GGML_TYPE_IQ2_S; break;
1432 case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break;
1433 case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break;
1434 }
1435
1436 GGML_ASSERT(wtype != GGML_TYPE_COUNT)if (!(wtype != GGML_TYPE_COUNT)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1436, "GGML_ASSERT(%s) failed", "wtype != GGML_TYPE_COUNT");
1437
1438 return wtype;
1439}
1440
1441size_t ggml_tensor_overhead(void) {
1442 return GGML_OBJECT_SIZE + GGML_TENSOR_SIZE;
1443}
1444
1445bool_Bool ggml_is_transposed(const struct ggml_tensor * tensor) {
1446 return tensor->nb[0] > tensor->nb[1];
1447}
1448
1449static bool_Bool ggml_is_contiguous_n(const struct ggml_tensor * tensor, int n) {
1450 size_t next_nb = ggml_type_size(tensor->type);
1451 if (tensor->ne[0] != ggml_blck_size(tensor->type) && tensor->nb[0] != next_nb) {
1452 return false0;
1453 }
1454 next_nb *= tensor->ne[0]/ggml_blck_size(tensor->type);
1455 for (int i = 1; i < GGML_MAX_DIMS4; i++) {
1456 if (i > n) {
1457 if (tensor->ne[i] != 1 && tensor->nb[i] != next_nb) {
1458 return false0;
1459 }
1460 next_nb *= tensor->ne[i];
1461 } else {
1462 // this dimension does not need to be contiguous
1463 next_nb = tensor->ne[i]*tensor->nb[i];
1464 }
1465 }
1466 return true1;
1467}
1468
1469bool_Bool ggml_is_contiguous(const struct ggml_tensor * tensor) {
1470 return ggml_is_contiguous_0(tensor);
1471}
1472
1473bool_Bool ggml_is_contiguous_0(const struct ggml_tensor * tensor) {
1474 return ggml_is_contiguous_n(tensor, 0);
1475}
1476
1477bool_Bool ggml_is_contiguous_1(const struct ggml_tensor * tensor) {
1478 return ggml_is_contiguous_n(tensor, 1);
1479}
1480
1481bool_Bool ggml_is_contiguous_2(const struct ggml_tensor * tensor) {
1482 return ggml_is_contiguous_n(tensor, 2);
1483}
1484
1485bool_Bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor) {
1486 return ggml_nbytes(tensor) == ggml_nelements(tensor) * ggml_type_size(tensor->type)/ggml_blck_size(tensor->type);
1487}
1488
1489bool_Bool ggml_is_permuted(const struct ggml_tensor * tensor) {
1490 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
1491
1492 return tensor->nb[0] > tensor->nb[1] || tensor->nb[1] > tensor->nb[2] || tensor->nb[2] > tensor->nb[3];
1493}
1494
1495bool_Bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor) {
1496 return
1497 tensor->nb[0] > tensor->nb[2] &&
1498 tensor->nb[1] > tensor->nb[0] &&
1499 tensor->nb[2] == ggml_type_size(tensor->type);
1500}
1501
1502bool_Bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor) {
1503 return
1504 tensor->ne[0] == ggml_blck_size(tensor->type) ||
1505 tensor->nb[0] == ggml_type_size(tensor->type);
1506}
1507
1508static inline bool_Bool ggml_is_padded_1d(const struct ggml_tensor * tensor) {
1509 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
1510
1511 return
1512 tensor->nb[0] == ggml_type_size(tensor->type) &&
1513 tensor->nb[2] == tensor->nb[1]*tensor->ne[1] &&
1514 tensor->nb[3] == tensor->nb[2]*tensor->ne[2];
1515}
1516
1517bool_Bool ggml_is_empty(const struct ggml_tensor * tensor) {
1518 for (int i = 0; i < GGML_MAX_DIMS4; ++i) {
1519 if (tensor->ne[i] == 0) {
1520 // empty if any dimension has no elements
1521 return true1;
1522 }
1523 }
1524 return false0;
1525}
1526
1527bool_Bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
1528 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
1529
1530 return
1531 (t0->ne[0] == t1->ne[0]) &&
1532 (t0->ne[1] == t1->ne[1]) &&
1533 (t0->ne[2] == t1->ne[2]) &&
1534 (t0->ne[3] == t1->ne[3]);
1535}
1536
1537bool_Bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
1538 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
1539
1540 return
1541 (t0->nb[0] == t1->nb[0]) &&
1542 (t0->nb[1] == t1->nb[1]) &&
1543 (t0->nb[2] == t1->nb[2]) &&
1544 (t0->nb[3] == t1->nb[3]);
1545}
1546
1547bool_Bool ggml_is_view(const struct ggml_tensor * t) {
1548 return ggml_impl_is_view(t);
1549}
1550
1551// check if t1 can be represented as a repetition of t0
1552bool_Bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
1553 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
1554
1555 return ggml_is_empty(t0) ? ggml_is_empty(t1) :
1556 (t1->ne[0]%t0->ne[0] == 0) &&
1557 (t1->ne[1]%t0->ne[1] == 0) &&
1558 (t1->ne[2]%t0->ne[2] == 0) &&
1559 (t1->ne[3]%t0->ne[3] == 0);
1560}
1561
1562static inline bool_Bool ggml_can_repeat_rows(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
1563 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
1564
1565 return (t0->ne[0] == t1->ne[0]) && ggml_can_repeat(t0, t1);
1566}
1567
1568// assert that pointer is aligned to GGML_MEM_ALIGN
1569#define GGML_ASSERT_ALIGNED(ptr)if (!(((uintptr_t) (ptr))%16 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1569, "GGML_ASSERT(%s) failed", "((uintptr_t) (ptr))%GGML_MEM_ALIGN == 0"
) \
1570 GGML_ASSERT(((uintptr_t) (ptr))%GGML_MEM_ALIGN == 0)if (!(((uintptr_t) (ptr))%16 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1570, "GGML_ASSERT(%s) failed", "((uintptr_t) (ptr))%GGML_MEM_ALIGN == 0"
)
1571
1572////////////////////////////////////////////////////////////////////////////////
1573
1574struct ggml_context * ggml_init(struct ggml_init_params params) {
1575 static bool_Bool is_first_call = true1;
1576
1577 ggml_critical_section_start();
1578
1579 if (is_first_call) {
1580 // initialize time system (required on Windows)
1581 ggml_time_init();
1582
1583 is_first_call = false0;
1584 }
1585
1586 ggml_critical_section_end();
1587
1588 struct ggml_context * ctx = GGML_MALLOC(sizeof(struct ggml_context))ggml_malloc(sizeof(struct ggml_context));
1589
1590 // allow to call ggml_init with 0 size
1591 if (params.mem_size == 0) {
1592 params.mem_size = GGML_MEM_ALIGN16;
1593 }
1594
1595 const size_t mem_size = params.mem_buffer ? params.mem_size : GGML_PAD(params.mem_size, GGML_MEM_ALIGN)(((params.mem_size) + (16) - 1) & ~((16) - 1));
1596
1597 *ctx = (struct ggml_context) {
1598 /*.mem_size =*/ mem_size,
1599 /*.mem_buffer =*/ params.mem_buffer ? params.mem_buffer : ggml_aligned_malloc(mem_size),
1600 /*.mem_buffer_owned =*/ params.mem_buffer ? false0 : true1,
1601 /*.no_alloc =*/ params.no_alloc,
1602 /*.n_objects =*/ 0,
1603 /*.objects_begin =*/ NULL((void*)0),
1604 /*.objects_end =*/ NULL((void*)0),
1605 };
1606
1607 GGML_ASSERT(ctx->mem_buffer != NULL)if (!(ctx->mem_buffer != ((void*)0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1607, "GGML_ASSERT(%s) failed", "ctx->mem_buffer != NULL"
);
1608
1609 GGML_ASSERT_ALIGNED(ctx->mem_buffer)if (!(((uintptr_t) (ctx->mem_buffer))%16 == 0)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1609, "GGML_ASSERT(%s) failed", "((uintptr_t) (ctx->mem_buffer))%GGML_MEM_ALIGN == 0"
);
1610
1611 GGML_PRINT_DEBUG("%s: context initialized\n", __func__);
1612
1613 return ctx;
1614}
1615
1616void ggml_reset(struct ggml_context * ctx) {
1617 if (ctx == NULL((void*)0)) {
1618 return;
1619 }
1620
1621 ctx->n_objects = 0;
1622 ctx->objects_begin = NULL((void*)0);
1623 ctx->objects_end = NULL((void*)0);
1624}
1625
1626void ggml_free(struct ggml_context * ctx) {
1627 if (ctx == NULL((void*)0)) {
1628 return;
1629 }
1630
1631 if (ctx->mem_buffer_owned) {
1632 ggml_aligned_free(ctx->mem_buffer, ctx->mem_size);
1633 }
1634
1635 GGML_FREE(ctx)free(ctx);
1636}
1637
1638size_t ggml_used_mem(const struct ggml_context * ctx) {
1639 return ctx->objects_end == NULL((void*)0) ? 0 : ctx->objects_end->offs + ctx->objects_end->size;
1640}
1641
1642bool_Bool ggml_get_no_alloc(struct ggml_context * ctx) {
1643 return ctx->no_alloc;
1644}
1645
1646void ggml_set_no_alloc(struct ggml_context * ctx, bool_Bool no_alloc) {
1647 ctx->no_alloc = no_alloc;
1648}
1649
1650void * ggml_get_mem_buffer(const struct ggml_context * ctx) {
1651 return ctx->mem_buffer;
1652}
1653
1654size_t ggml_get_mem_size(const struct ggml_context * ctx) {
1655 return ctx->mem_size;
1656}
1657
1658size_t ggml_get_max_tensor_size(const struct ggml_context * ctx) {
1659 size_t max_size = 0;
1660
1661 for (struct ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor != NULL((void*)0); tensor = ggml_get_next_tensor(ctx, tensor)) {
1662 size_t bytes = ggml_nbytes(tensor);
1663 max_size = MAX(max_size, bytes)((max_size) > (bytes) ? (max_size) : (bytes));
1664 }
1665
1666 return max_size;
1667}
1668
1669////////////////////////////////////////////////////////////////////////////////
1670
1671static struct ggml_object * ggml_new_object(struct ggml_context * ctx, enum ggml_object_type type, size_t size) {
1672 // always insert objects at the end of the context's memory pool
1673 struct ggml_object * obj_cur = ctx->objects_end;
1674
1675 const size_t cur_offs = obj_cur == NULL((void*)0) ? 0 : obj_cur->offs;
1676 const size_t cur_size = obj_cur == NULL((void*)0) ? 0 : obj_cur->size;
1677 const size_t cur_end = cur_offs + cur_size;
1678
1679 // align to GGML_MEM_ALIGN
1680 GGML_ASSERT(size <= SIZE_MAX - (GGML_MEM_ALIGN - 1))if (!(size <= (18446744073709551615UL) - (16 - 1))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1680, "GGML_ASSERT(%s) failed", "size <= SIZE_MAX - (GGML_MEM_ALIGN - 1)"
);
1681 size_t size_needed = GGML_PAD(size, GGML_MEM_ALIGN)(((size) + (16) - 1) & ~((16) - 1));
1682
1683 char * const mem_buffer = ctx->mem_buffer;
1684 struct ggml_object * const obj_new = (struct ggml_object *)(mem_buffer + cur_end);
1685
1686 // integer overflow checks
1687 if (cur_end > SIZE_MAX(18446744073709551615UL) - size_needed) {
1688 GGML_LOG_WARN("%s: overflow detected in cur_end (%zu) + size_needed (%zu)\n", __func__, cur_end, size_needed)ggml_log_internal(GGML_LOG_LEVEL_WARN , "%s: overflow detected in cur_end (%zu) + size_needed (%zu)\n"
, __func__, cur_end, size_needed);
1689 return NULL((void*)0);
1690 }
1691 if (cur_end + size_needed > SIZE_MAX(18446744073709551615UL) - GGML_OBJECT_SIZE) {
1692 GGML_LOG_WARN("%s: overflow detected in cur_end (%zu) + size_needed (%zu) + GGML_OBJECT_SIZE (%zu)\n", __func__,ggml_log_internal(GGML_LOG_LEVEL_WARN , "%s: overflow detected in cur_end (%zu) + size_needed (%zu) + GGML_OBJECT_SIZE (%zu)\n"
, __func__, cur_end, size_needed, (size_t) GGML_OBJECT_SIZE)
1693 cur_end, size_needed, (size_t) GGML_OBJECT_SIZE)ggml_log_internal(GGML_LOG_LEVEL_WARN , "%s: overflow detected in cur_end (%zu) + size_needed (%zu) + GGML_OBJECT_SIZE (%zu)\n"
, __func__, cur_end, size_needed, (size_t) GGML_OBJECT_SIZE);
1694 return NULL((void*)0);
1695 }
1696
1697 if (cur_end + size_needed + GGML_OBJECT_SIZE > ctx->mem_size) {
1698 GGML_LOG_WARN("%s: not enough space in the context's memory pool (needed %zu, available %zu)\n",ggml_log_internal(GGML_LOG_LEVEL_WARN , "%s: not enough space in the context's memory pool (needed %zu, available %zu)\n"
, __func__, cur_end + size_needed + GGML_OBJECT_SIZE, ctx->
mem_size)
1699 __func__, cur_end + size_needed + GGML_OBJECT_SIZE, ctx->mem_size)ggml_log_internal(GGML_LOG_LEVEL_WARN , "%s: not enough space in the context's memory pool (needed %zu, available %zu)\n"
, __func__, cur_end + size_needed + GGML_OBJECT_SIZE, ctx->
mem_size);
1700#ifndef NDEBUG
1701 GGML_ABORT("not enough space in the context's memory pool")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1701, "not enough space in the context's memory pool");
1702#endif
1703 return NULL((void*)0);
1704 }
1705
1706 *obj_new = (struct ggml_object) {
1707 .offs = cur_end + GGML_OBJECT_SIZE,
1708 .size = size_needed,
1709 .next = NULL((void*)0),
1710 .type = type,
1711 };
1712
1713 GGML_ASSERT_ALIGNED(mem_buffer + obj_new->offs)if (!(((uintptr_t) (mem_buffer + obj_new->offs))%16 == 0))
ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1713, "GGML_ASSERT(%s) failed", "((uintptr_t) (mem_buffer + obj_new->offs))%GGML_MEM_ALIGN == 0"
);
1714
1715 if (obj_cur != NULL((void*)0)) {
1716 obj_cur->next = obj_new;
1717 } else {
1718 // this is the first object in this context
1719 ctx->objects_begin = obj_new;
1720 }
1721
1722 ctx->objects_end = obj_new;
1723
1724 //printf("%s: inserted new object at %zu, size = %zu\n", __func__, cur_end, obj_new->size);
1725
1726 return obj_new;
1727}
1728
1729static struct ggml_tensor * ggml_new_tensor_impl(
1730 struct ggml_context * ctx,
1731 enum ggml_type type,
1732 int n_dims,
1733 const int64_t * ne,
1734 struct ggml_tensor * view_src,
1735 size_t view_offs) {
1736
1737 GGML_ASSERT(type >= 0 && type < GGML_TYPE_COUNT)if (!(type >= 0 && type < GGML_TYPE_COUNT)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1737, "GGML_ASSERT(%s) failed", "type >= 0 && type < GGML_TYPE_COUNT"
);
1738 GGML_ASSERT(n_dims >= 1 && n_dims <= GGML_MAX_DIMS)if (!(n_dims >= 1 && n_dims <= 4)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1738, "GGML_ASSERT(%s) failed", "n_dims >= 1 && n_dims <= GGML_MAX_DIMS"
);
1739
1740 // find the base tensor and absolute offset
1741 if (view_src != NULL((void*)0) && view_src->view_src != NULL((void*)0)) {
1742 view_offs += view_src->view_offs;
1743 view_src = view_src->view_src;
1744 }
1745
1746 size_t data_size = ggml_row_size(type, ne[0]);
1747 for (int i = 1; i < n_dims; i++) {
1748 data_size *= ne[i];
1749 }
1750
1751 GGML_ASSERT(view_src == NULL || data_size == 0 || data_size + view_offs <= ggml_nbytes(view_src))if (!(view_src == ((void*)0) || data_size == 0 || data_size +
view_offs <= ggml_nbytes(view_src))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1751, "GGML_ASSERT(%s) failed", "view_src == NULL || data_size == 0 || data_size + view_offs <= ggml_nbytes(view_src)"
);
1752
1753 void * data = view_src != NULL((void*)0) ? view_src->data : NULL((void*)0);
1754 if (data != NULL((void*)0)) {
1755 data = (char *) data + view_offs;
1756 }
1757
1758 size_t obj_alloc_size = 0;
1759
1760 if (view_src == NULL((void*)0) && !ctx->no_alloc) {
1761 // allocate tensor data in the context's memory pool
1762 obj_alloc_size = data_size;
1763 }
1764
1765 GGML_ASSERT(GGML_TENSOR_SIZE <= SIZE_MAX - obj_alloc_size)if (!(GGML_TENSOR_SIZE <= (18446744073709551615UL) - obj_alloc_size
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1765, "GGML_ASSERT(%s) failed", "GGML_TENSOR_SIZE <= SIZE_MAX - obj_alloc_size"
);
1766
1767 struct ggml_object * const obj_new = ggml_new_object(ctx, GGML_OBJECT_TYPE_TENSOR, GGML_TENSOR_SIZE + obj_alloc_size);
1768 GGML_ASSERT(obj_new)if (!(obj_new)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1768, "GGML_ASSERT(%s) failed", "obj_new");
1769
1770 struct ggml_tensor * const result = (struct ggml_tensor *)((char *)ctx->mem_buffer + obj_new->offs);
1771
1772 *result = (struct ggml_tensor) {
1773 /*.type =*/ type,
1774 /*.buffer =*/ NULL((void*)0),
1775 /*.ne =*/ { 1, 1, 1, 1 },
1776 /*.nb =*/ { 0, 0, 0, 0 },
1777 /*.op =*/ GGML_OP_NONE,
1778 /*.op_params =*/ { 0 },
1779 /*.flags =*/ 0,
1780 /*.src =*/ { NULL((void*)0) },
1781 /*.view_src =*/ view_src,
1782 /*.view_offs =*/ view_offs,
1783 /*.data =*/ obj_alloc_size > 0 ? (void *)(result + 1) : data,
1784 /*.name =*/ { 0 },
1785 /*.extra =*/ NULL((void*)0),
1786 /*.padding =*/ { 0 },
1787 };
1788
1789 // TODO: this should not be needed as long as we don't rely on aligned SIMD loads
1790 //GGML_ASSERT_ALIGNED(result->data);
1791
1792 for (int i = 0; i < n_dims; i++) {
1793 result->ne[i] = ne[i];
1794 }
1795
1796 result->nb[0] = ggml_type_size(type);
1797 result->nb[1] = result->nb[0]*(result->ne[0]/ggml_blck_size(type));
1798 for (int i = 2; i < GGML_MAX_DIMS4; i++) {
1799 result->nb[i] = result->nb[i - 1]*result->ne[i - 1];
1800 }
1801
1802 ctx->n_objects++;
1803
1804 return result;
1805}
1806
1807struct ggml_tensor * ggml_new_tensor(
1808 struct ggml_context * ctx,
1809 enum ggml_type type,
1810 int n_dims,
1811 const int64_t * ne) {
1812 return ggml_new_tensor_impl(ctx, type, n_dims, ne, NULL((void*)0), 0);
1813}
1814
1815struct ggml_tensor * ggml_new_tensor_1d(
1816 struct ggml_context * ctx,
1817 enum ggml_type type,
1818 int64_t ne0) {
1819 return ggml_new_tensor(ctx, type, 1, &ne0);
1820}
1821
1822struct ggml_tensor * ggml_new_tensor_2d(
1823 struct ggml_context * ctx,
1824 enum ggml_type type,
1825 int64_t ne0,
1826 int64_t ne1) {
1827 const int64_t ne[2] = { ne0, ne1 };
1828 return ggml_new_tensor(ctx, type, 2, ne);
1829}
1830
1831struct ggml_tensor * ggml_new_tensor_3d(
1832 struct ggml_context * ctx,
1833 enum ggml_type type,
1834 int64_t ne0,
1835 int64_t ne1,
1836 int64_t ne2) {
1837 const int64_t ne[3] = { ne0, ne1, ne2 };
1838 return ggml_new_tensor(ctx, type, 3, ne);
1839}
1840
1841struct ggml_tensor * ggml_new_tensor_4d(
1842 struct ggml_context * ctx,
1843 enum ggml_type type,
1844 int64_t ne0,
1845 int64_t ne1,
1846 int64_t ne2,
1847 int64_t ne3) {
1848 const int64_t ne[4] = { ne0, ne1, ne2, ne3 };
1849 return ggml_new_tensor(ctx, type, 4, ne);
1850}
1851
1852void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes) {
1853 struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_WORK_BUFFER, nbytes);
1854
1855 return (uint8_t *)ctx->mem_buffer + obj->offs;
1856}
1857
1858struct ggml_tensor * ggml_dup_tensor(struct ggml_context * ctx, const struct ggml_tensor * src) {
1859 return ggml_new_tensor(ctx, src->type, GGML_MAX_DIMS4, src->ne);
1860}
1861
1862void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3) {
1863 const int64_t ne2 = tensor->ne[2];
1864 const int64_t ne1 = tensor->ne[1];
1865 const int64_t ne0 = tensor->ne[0];
1866
1867 const int64_t i3_ = (i/(ne2*ne1*ne0));
1868 const int64_t i2_ = (i - i3_*ne2*ne1*ne0)/(ne1*ne0);
1869 const int64_t i1_ = (i - i3_*ne2*ne1*ne0 - i2_*ne1*ne0)/ne0;
1870 const int64_t i0_ = (i - i3_*ne2*ne1*ne0 - i2_*ne1*ne0 - i1_*ne0);
1871
1872 if (i0) {
1873 * i0 = i0_;
1874 }
1875 if (i1) {
1876 * i1 = i1_;
1877 }
1878 if (i2) {
1879 * i2 = i2_;
1880 }
1881 if (i3) {
1882 * i3 = i3_;
1883 }
1884}
1885
1886void * ggml_get_data(const struct ggml_tensor * tensor) {
1887 return tensor->data;
1888}
1889
1890float * ggml_get_data_f32(const struct ggml_tensor * tensor) {
1891 assert(tensor->type == GGML_TYPE_F32)((void) sizeof (__assert_single_arg (tensor->type == GGML_TYPE_F32
)), __extension__ ({ if (tensor->type == GGML_TYPE_F32) ; else
__assert_fail ("tensor->type == GGML_TYPE_F32", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1891, __extension__ __PRETTY_FUNCTION__); }));
1892 return (float *)(tensor->data);
1893}
1894
1895enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor) {
1896 GGML_ASSERT(tensor->op == GGML_OP_UNARY)if (!(tensor->op == GGML_OP_UNARY)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1896, "GGML_ASSERT(%s) failed", "tensor->op == GGML_OP_UNARY"
);
1897 return (enum ggml_unary_op) ggml_get_op_params_i32(tensor, 0);
1898}
1899
1900enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor) {
1901 GGML_ASSERT(tensor->op == GGML_OP_GLU)if (!(tensor->op == GGML_OP_GLU)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 1901, "GGML_ASSERT(%s) failed", "tensor->op == GGML_OP_GLU"
);
1902 return (enum ggml_glu_op) ggml_get_op_params_i32(tensor, 0);
1903}
1904
1905const char * ggml_get_name(const struct ggml_tensor * tensor) {
1906 return tensor->name;
1907}
1908
1909struct ggml_tensor * ggml_set_name(struct ggml_tensor * tensor, const char * name) {
1910 size_t i;
1911 for (i = 0; i < sizeof(tensor->name) - 1 && name[i] != '\0'; i++) {
1912 tensor->name[i] = name[i];
1913 }
1914 tensor->name[i] = '\0';
1915 return tensor;
1916}
1917
1918struct ggml_tensor * ggml_format_name(struct ggml_tensor * tensor, const char * fmt, ...) {
1919 va_list args;
1920 va_start(args, fmt)__builtin_va_start(args, fmt);
1921 vsnprintf(tensor->name, sizeof(tensor->name), fmt, args);
1922 va_end(args)__builtin_va_end(args);
1923 return tensor;
1924}
1925
1926struct ggml_tensor * ggml_view_tensor(
1927 struct ggml_context * ctx,
1928 struct ggml_tensor * src) {
1929 struct ggml_tensor * result = ggml_new_tensor_impl(ctx, src->type, GGML_MAX_DIMS4, src->ne, src, 0);
1930 ggml_format_name(result, "%s (view)", src->name);
1931
1932 for (int i = 0; i < GGML_MAX_DIMS4; i++) {
1933 result->nb[i] = src->nb[i];
1934 }
1935
1936 return result;
1937}
1938
1939struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx) {
1940 struct ggml_object * obj = ctx->objects_begin;
1941
1942 char * const mem_buffer = ctx->mem_buffer;
1943
1944 while (obj != NULL((void*)0)) {
1945 if (obj->type == GGML_OBJECT_TYPE_TENSOR) {
1946 return (struct ggml_tensor *)(mem_buffer + obj->offs);
1947 }
1948
1949 obj = obj->next;
1950 }
1951
1952 return NULL((void*)0);
1953}
1954
1955struct ggml_tensor * ggml_get_next_tensor(const struct ggml_context * ctx, struct ggml_tensor * tensor) {
1956 struct ggml_object * obj = (struct ggml_object *) ((char *)tensor - GGML_OBJECT_SIZE);
1957 obj = obj->next;
1958
1959 char * const mem_buffer = ctx->mem_buffer;
1960
1961 while (obj != NULL((void*)0)) {
1962 if (obj->type == GGML_OBJECT_TYPE_TENSOR) {
1963 return (struct ggml_tensor *)(mem_buffer + obj->offs);
1964 }
1965
1966 obj = obj->next;
1967 }
1968
1969 return NULL((void*)0);
1970}
1971
1972struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name) {
1973 struct ggml_object * obj = ctx->objects_begin;
1974
1975 char * const mem_buffer = ctx->mem_buffer;
1976
1977 while (obj != NULL((void*)0)) {
1978 if (obj->type == GGML_OBJECT_TYPE_TENSOR) {
1979 struct ggml_tensor * cur = (struct ggml_tensor *)(mem_buffer + obj->offs);
1980 if (strcmp(cur->name, name) == 0) {
1981 return cur;
1982 }
1983 }
1984
1985 obj = obj->next;
1986 }
1987
1988 return NULL((void*)0);
1989}
1990
1991////////////////////////////////////////////////////////////////////////////////
1992
1993// ggml_dup
1994
1995static struct ggml_tensor * ggml_dup_impl(
1996 struct ggml_context * ctx,
1997 struct ggml_tensor * a,
1998 bool_Bool inplace) {
1999 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2000
2001 result->op = GGML_OP_DUP;
2002 result->src[0] = a;
2003
2004 return result;
2005}
2006
2007struct ggml_tensor * ggml_dup(
2008 struct ggml_context * ctx,
2009 struct ggml_tensor * a) {
2010 return ggml_dup_impl(ctx, a, false0);
2011}
2012
2013struct ggml_tensor * ggml_dup_inplace(
2014 struct ggml_context * ctx,
2015 struct ggml_tensor * a) {
2016 return ggml_dup_impl(ctx, a, true1);
2017}
2018
2019// ggml_add
2020
2021static struct ggml_tensor * ggml_add_impl(
2022 struct ggml_context * ctx,
2023 struct ggml_tensor * a,
2024 struct ggml_tensor * b,
2025 bool_Bool inplace) {
2026 GGML_ASSERT(ggml_can_repeat(b, a))if (!(ggml_can_repeat(b, a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2026, "GGML_ASSERT(%s) failed", "ggml_can_repeat(b, a)");
2027
2028 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2029
2030 result->op = GGML_OP_ADD;
2031 result->src[0] = a;
2032 result->src[1] = b;
2033
2034 return result;
2035}
2036
2037struct ggml_tensor * ggml_add(
2038 struct ggml_context * ctx,
2039 struct ggml_tensor * a,
2040 struct ggml_tensor * b) {
2041 return ggml_add_impl(ctx, a, b, false0);
2042}
2043
2044struct ggml_tensor * ggml_add_inplace(
2045 struct ggml_context * ctx,
2046 struct ggml_tensor * a,
2047 struct ggml_tensor * b) {
2048 return ggml_add_impl(ctx, a, b, true1);
2049}
2050
2051// ggml_add_cast
2052
2053static struct ggml_tensor * ggml_add_cast_impl(
2054 struct ggml_context * ctx,
2055 struct ggml_tensor * a,
2056 struct ggml_tensor * b,
2057 enum ggml_type type) {
2058 // TODO: support less-strict constraint
2059 // GGML_ASSERT(ggml_can_repeat(b, a));
2060 GGML_ASSERT(ggml_can_repeat_rows(b, a))if (!(ggml_can_repeat_rows(b, a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2060, "GGML_ASSERT(%s) failed", "ggml_can_repeat_rows(b, a)"
);
2061
2062 // currently only supported for quantized input and f16
2063 GGML_ASSERT(ggml_is_quantized(a->type) ||if (!(ggml_is_quantized(a->type) || a->type == GGML_TYPE_F16
|| a->type == GGML_TYPE_BF16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2065, "GGML_ASSERT(%s) failed", "ggml_is_quantized(a->type) || a->type == GGML_TYPE_F16 || a->type == GGML_TYPE_BF16"
)
2064 a->type == GGML_TYPE_F16 ||if (!(ggml_is_quantized(a->type) || a->type == GGML_TYPE_F16
|| a->type == GGML_TYPE_BF16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2065, "GGML_ASSERT(%s) failed", "ggml_is_quantized(a->type) || a->type == GGML_TYPE_F16 || a->type == GGML_TYPE_BF16"
)
2065 a->type == GGML_TYPE_BF16)if (!(ggml_is_quantized(a->type) || a->type == GGML_TYPE_F16
|| a->type == GGML_TYPE_BF16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2065, "GGML_ASSERT(%s) failed", "ggml_is_quantized(a->type) || a->type == GGML_TYPE_F16 || a->type == GGML_TYPE_BF16"
);
2066
2067 struct ggml_tensor * result = ggml_new_tensor(ctx, type, GGML_MAX_DIMS4, a->ne);
2068
2069 result->op = GGML_OP_ADD;
2070 result->src[0] = a;
2071 result->src[1] = b;
2072
2073 return result;
2074}
2075
2076struct ggml_tensor * ggml_add_cast(
2077 struct ggml_context * ctx,
2078 struct ggml_tensor * a,
2079 struct ggml_tensor * b,
2080 enum ggml_type type) {
2081 return ggml_add_cast_impl(ctx, a, b, type);
2082}
2083
2084struct ggml_tensor * ggml_add_id(
2085 struct ggml_context * ctx,
2086 struct ggml_tensor * a,
2087 struct ggml_tensor * b,
2088 struct ggml_tensor * ids) {
2089
2090 GGML_ASSERT(a->ne[0] == b->ne[0])if (!(a->ne[0] == b->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2090, "GGML_ASSERT(%s) failed", "a->ne[0] == b->ne[0]"
);
2091 GGML_ASSERT(a->ne[1] == ids->ne[0])if (!(a->ne[1] == ids->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2091, "GGML_ASSERT(%s) failed", "a->ne[1] == ids->ne[0]"
);
2092 GGML_ASSERT(a->ne[2] == ids->ne[1])if (!(a->ne[2] == ids->ne[1])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2092, "GGML_ASSERT(%s) failed", "a->ne[2] == ids->ne[1]"
);
2093 GGML_ASSERT(ids->type == GGML_TYPE_I32)if (!(ids->type == GGML_TYPE_I32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2093, "GGML_ASSERT(%s) failed", "ids->type == GGML_TYPE_I32"
);
2094
2095 struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
2096
2097 result->op = GGML_OP_ADD_ID;
2098 result->src[0] = a;
2099 result->src[1] = b;
2100 result->src[2] = ids;
2101
2102 return result;
2103}
2104
2105// ggml_add1
2106
2107static struct ggml_tensor * ggml_add1_impl(
2108 struct ggml_context * ctx,
2109 struct ggml_tensor * a,
2110 struct ggml_tensor * b,
2111 bool_Bool inplace) {
2112 GGML_ASSERT(ggml_is_scalar(b))if (!(ggml_is_scalar(b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2112, "GGML_ASSERT(%s) failed", "ggml_is_scalar(b)");
2113 GGML_ASSERT(ggml_is_padded_1d(a))if (!(ggml_is_padded_1d(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2113, "GGML_ASSERT(%s) failed", "ggml_is_padded_1d(a)");
2114
2115 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2116
2117 result->op = GGML_OP_ADD1;
2118 result->src[0] = a;
2119 result->src[1] = b;
2120
2121 return result;
2122}
2123
2124struct ggml_tensor * ggml_add1(
2125 struct ggml_context * ctx,
2126 struct ggml_tensor * a,
2127 struct ggml_tensor * b) {
2128 return ggml_add1_impl(ctx, a, b, false0);
2129}
2130
2131struct ggml_tensor * ggml_add1_inplace(
2132 struct ggml_context * ctx,
2133 struct ggml_tensor * a,
2134 struct ggml_tensor * b) {
2135 return ggml_add1_impl(ctx, a, b, true1);
2136}
2137
2138// ggml_acc
2139
2140static struct ggml_tensor * ggml_acc_impl(
2141 struct ggml_context * ctx,
2142 struct ggml_tensor * a,
2143 struct ggml_tensor * b,
2144 size_t nb1,
2145 size_t nb2,
2146 size_t nb3,
2147 size_t offset,
2148 bool_Bool inplace) {
2149 GGML_ASSERT(ggml_nelements(b) <= ggml_nelements(a))if (!(ggml_nelements(b) <= ggml_nelements(a))) ggml_abort(
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2149, "GGML_ASSERT(%s) failed", "ggml_nelements(b) <= ggml_nelements(a)"
);
2150 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2150, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
2151 GGML_ASSERT(a->type == GGML_TYPE_F32)if (!(a->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2151, "GGML_ASSERT(%s) failed", "a->type == GGML_TYPE_F32"
);
2152 GGML_ASSERT(b->type == GGML_TYPE_F32)if (!(b->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2152, "GGML_ASSERT(%s) failed", "b->type == GGML_TYPE_F32"
);
2153
2154 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2155
2156 int32_t params[] = { nb1, nb2, nb3, offset, inplace ? 1 : 0 };
2157 ggml_set_op_params(result, params, sizeof(params));
2158
2159 result->op = GGML_OP_ACC;
2160 result->src[0] = a;
2161 result->src[1] = b;
2162
2163 return result;
2164}
2165
2166struct ggml_tensor * ggml_acc(
2167 struct ggml_context * ctx,
2168 struct ggml_tensor * a,
2169 struct ggml_tensor * b,
2170 size_t nb1,
2171 size_t nb2,
2172 size_t nb3,
2173 size_t offset) {
2174 return ggml_acc_impl(ctx, a, b, nb1, nb2, nb3, offset, false0);
2175}
2176
2177struct ggml_tensor * ggml_acc_inplace(
2178 struct ggml_context * ctx,
2179 struct ggml_tensor * a,
2180 struct ggml_tensor * b,
2181 size_t nb1,
2182 size_t nb2,
2183 size_t nb3,
2184 size_t offset) {
2185 return ggml_acc_impl(ctx, a, b, nb1, nb2, nb3, offset, true1);
2186}
2187
2188// ggml_sub
2189
2190static struct ggml_tensor * ggml_sub_impl(
2191 struct ggml_context * ctx,
2192 struct ggml_tensor * a,
2193 struct ggml_tensor * b,
2194 bool_Bool inplace) {
2195 GGML_ASSERT(ggml_can_repeat(b, a))if (!(ggml_can_repeat(b, a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2195, "GGML_ASSERT(%s) failed", "ggml_can_repeat(b, a)");
2196
2197 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2198
2199 result->op = GGML_OP_SUB;
2200 result->src[0] = a;
2201 result->src[1] = b;
2202
2203 return result;
2204}
2205
2206struct ggml_tensor * ggml_sub(
2207 struct ggml_context * ctx,
2208 struct ggml_tensor * a,
2209 struct ggml_tensor * b) {
2210 return ggml_sub_impl(ctx, a, b, false0);
2211}
2212
2213struct ggml_tensor * ggml_sub_inplace(
2214 struct ggml_context * ctx,
2215 struct ggml_tensor * a,
2216 struct ggml_tensor * b) {
2217 return ggml_sub_impl(ctx, a, b, true1);
2218}
2219
2220// ggml_mul
2221
2222static struct ggml_tensor * ggml_mul_impl(
2223 struct ggml_context * ctx,
2224 struct ggml_tensor * a,
2225 struct ggml_tensor * b,
2226 bool_Bool inplace) {
2227 GGML_ASSERT(ggml_can_repeat(b, a))if (!(ggml_can_repeat(b, a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2227, "GGML_ASSERT(%s) failed", "ggml_can_repeat(b, a)");
2228
2229 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2230
2231 result->op = GGML_OP_MUL;
2232 result->src[0] = a;
2233 result->src[1] = b;
2234
2235 return result;
2236}
2237
2238struct ggml_tensor * ggml_mul(
2239 struct ggml_context * ctx,
2240 struct ggml_tensor * a,
2241 struct ggml_tensor * b) {
2242 return ggml_mul_impl(ctx, a, b, false0);
2243}
2244
2245struct ggml_tensor * ggml_mul_inplace(
2246 struct ggml_context * ctx,
2247 struct ggml_tensor * a,
2248 struct ggml_tensor * b) {
2249 return ggml_mul_impl(ctx, a, b, true1);
2250}
2251
2252// ggml_div
2253
2254static struct ggml_tensor * ggml_div_impl(
2255 struct ggml_context * ctx,
2256 struct ggml_tensor * a,
2257 struct ggml_tensor * b,
2258 bool_Bool inplace) {
2259 GGML_ASSERT(ggml_can_repeat(b, a))if (!(ggml_can_repeat(b, a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2259, "GGML_ASSERT(%s) failed", "ggml_can_repeat(b, a)");
2260
2261 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2262
2263 result->op = GGML_OP_DIV;
2264 result->src[0] = a;
2265 result->src[1] = b;
2266
2267 return result;
2268}
2269
2270struct ggml_tensor * ggml_div(
2271 struct ggml_context * ctx,
2272 struct ggml_tensor * a,
2273 struct ggml_tensor * b) {
2274 return ggml_div_impl(ctx, a, b, false0);
2275}
2276
2277struct ggml_tensor * ggml_div_inplace(
2278 struct ggml_context * ctx,
2279 struct ggml_tensor * a,
2280 struct ggml_tensor * b) {
2281 return ggml_div_impl(ctx, a, b, true1);
2282}
2283
2284// ggml_sqr
2285
2286static struct ggml_tensor * ggml_sqr_impl(
2287 struct ggml_context * ctx,
2288 struct ggml_tensor * a,
2289 bool_Bool inplace) {
2290 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2291
2292 result->op = GGML_OP_SQR;
2293 result->src[0] = a;
2294
2295 return result;
2296}
2297
2298struct ggml_tensor * ggml_sqr(
2299 struct ggml_context * ctx,
2300 struct ggml_tensor * a) {
2301 return ggml_sqr_impl(ctx, a, false0);
2302}
2303
2304struct ggml_tensor * ggml_sqr_inplace(
2305 struct ggml_context * ctx,
2306 struct ggml_tensor * a) {
2307 return ggml_sqr_impl(ctx, a, true1);
2308}
2309
2310// ggml_sqrt
2311
2312static struct ggml_tensor * ggml_sqrt_impl(
2313 struct ggml_context * ctx,
2314 struct ggml_tensor * a,
2315 bool_Bool inplace) {
2316 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2317
2318 result->op = GGML_OP_SQRT;
2319 result->src[0] = a;
2320
2321 return result;
2322}
2323
2324struct ggml_tensor * ggml_sqrt(
2325 struct ggml_context * ctx,
2326 struct ggml_tensor * a) {
2327 return ggml_sqrt_impl(ctx, a, false0);
2328}
2329
2330struct ggml_tensor * ggml_sqrt_inplace(
2331 struct ggml_context * ctx,
2332 struct ggml_tensor * a) {
2333 return ggml_sqrt_impl(ctx, a, true1);
2334}
2335
2336// ggml_log
2337
2338static struct ggml_tensor * ggml_log_impl(
2339 struct ggml_context * ctx,
2340 struct ggml_tensor * a,
2341 bool_Bool inplace) {
2342 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2343
2344 result->op = GGML_OP_LOG;
2345 result->src[0] = a;
2346
2347 return result;
2348}
2349
2350struct ggml_tensor * ggml_log(
2351 struct ggml_context * ctx,
2352 struct ggml_tensor * a) {
2353 return ggml_log_impl(ctx, a, false0);
2354}
2355
2356struct ggml_tensor * ggml_log_inplace(
2357 struct ggml_context * ctx,
2358 struct ggml_tensor * a) {
2359 return ggml_log_impl(ctx, a, true1);
2360}
2361
2362struct ggml_tensor * ggml_expm1(
2363 struct ggml_context * ctx,
2364 struct ggml_tensor * a) {
2365 return ggml_unary(ctx, a, GGML_UNARY_OP_EXPM1);
2366}
2367
2368struct ggml_tensor * ggml_expm1_inplace(
2369 struct ggml_context * ctx,
2370 struct ggml_tensor * a) {
2371 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_EXPM1);
2372}
2373
2374struct ggml_tensor * ggml_softplus(
2375 struct ggml_context * ctx,
2376 struct ggml_tensor * a) {
2377 return ggml_unary(ctx, a, GGML_UNARY_OP_SOFTPLUS);
2378}
2379
2380struct ggml_tensor * ggml_softplus_inplace(
2381 struct ggml_context * ctx,
2382 struct ggml_tensor * a) {
2383 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_SOFTPLUS);
2384}
2385
2386// ggml_sin
2387
2388static struct ggml_tensor * ggml_sin_impl(
2389 struct ggml_context * ctx,
2390 struct ggml_tensor * a,
2391 bool_Bool inplace) {
2392 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2393
2394 result->op = GGML_OP_SIN;
2395 result->src[0] = a;
2396
2397 return result;
2398}
2399
2400struct ggml_tensor * ggml_sin(
2401 struct ggml_context * ctx,
2402 struct ggml_tensor * a) {
2403 return ggml_sin_impl(ctx, a, false0);
2404}
2405
2406struct ggml_tensor * ggml_sin_inplace(
2407 struct ggml_context * ctx,
2408 struct ggml_tensor * a) {
2409 return ggml_sin_impl(ctx, a, true1);
2410}
2411
2412// ggml_cos
2413
2414static struct ggml_tensor * ggml_cos_impl(
2415 struct ggml_context * ctx,
2416 struct ggml_tensor * a,
2417 bool_Bool inplace) {
2418 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2419
2420 result->op = GGML_OP_COS;
2421 result->src[0] = a;
2422
2423 return result;
2424}
2425
2426struct ggml_tensor * ggml_cos(
2427 struct ggml_context * ctx,
2428 struct ggml_tensor * a) {
2429 return ggml_cos_impl(ctx, a, false0);
2430}
2431
2432struct ggml_tensor * ggml_cos_inplace(
2433 struct ggml_context * ctx,
2434 struct ggml_tensor * a) {
2435 return ggml_cos_impl(ctx, a, true1);
2436}
2437
2438// ggml_sum
2439
2440struct ggml_tensor * ggml_sum(
2441 struct ggml_context * ctx,
2442 struct ggml_tensor * a) {
2443 struct ggml_tensor * result = ggml_new_tensor_1d(ctx, a->type, 1);
2444
2445 result->op = GGML_OP_SUM;
2446 result->src[0] = a;
2447
2448 return result;
2449}
2450
2451// ggml_sum_rows
2452
2453struct ggml_tensor * ggml_sum_rows(
2454 struct ggml_context * ctx,
2455 struct ggml_tensor * a) {
2456 int64_t ne[GGML_MAX_DIMS4] = { 1 };
2457 for (int i = 1; i < GGML_MAX_DIMS4; ++i) {
2458 ne[i] = a->ne[i];
2459 }
2460
2461 struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS4, ne);
2462
2463 result->op = GGML_OP_SUM_ROWS;
2464 result->src[0] = a;
2465
2466 return result;
2467}
2468
2469// ggml_cumsum
2470
2471struct ggml_tensor * ggml_cumsum(
2472 struct ggml_context * ctx,
2473 struct ggml_tensor * a) {
2474 GGML_ASSERT(a->type == GGML_TYPE_F32)if (!(a->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2474, "GGML_ASSERT(%s) failed", "a->type == GGML_TYPE_F32"
);
2475
2476 struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
2477
2478 result->op = GGML_OP_CUMSUM;
2479 result->src[0] = a;
2480
2481 return result;
2482}
2483
2484// ggml_mean
2485
2486struct ggml_tensor * ggml_mean(
2487 struct ggml_context * ctx,
2488 struct ggml_tensor * a) {
2489 int64_t ne[4] = { 1, a->ne[1], a->ne[2], a->ne[3] };
2490 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
2491
2492 result->op = GGML_OP_MEAN;
2493 result->src[0] = a;
2494
2495 return result;
2496}
2497
2498// ggml_argmax
2499
2500struct ggml_tensor * ggml_argmax(
2501 struct ggml_context * ctx,
2502 struct ggml_tensor * a) {
2503 GGML_ASSERT(ggml_is_matrix(a))if (!(ggml_is_matrix(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2503, "GGML_ASSERT(%s) failed", "ggml_is_matrix(a)");
2504 GGML_ASSERT(a->ne[0] <= INT32_MAX)if (!(a->ne[0] <= (2147483647))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2504, "GGML_ASSERT(%s) failed", "a->ne[0] <= INT32_MAX"
);
2505
2506 struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, a->ne[1]);
2507
2508 result->op = GGML_OP_ARGMAX;
2509 result->src[0] = a;
2510
2511 return result;
2512}
2513
2514// ggml_count_equal
2515
2516struct ggml_tensor * ggml_count_equal(
2517 struct ggml_context * ctx,
2518 struct ggml_tensor * a,
2519 struct ggml_tensor * b) {
2520 GGML_ASSERT(ggml_are_same_shape(a, b))if (!(ggml_are_same_shape(a, b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2520, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(a, b)"
);
2521
2522 struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, 1);
2523
2524 result->op = GGML_OP_COUNT_EQUAL;
2525 result->src[0] = a;
2526 result->src[1] = b;
2527
2528 return result;
2529}
2530
2531// ggml_repeat
2532
2533struct ggml_tensor * ggml_repeat(
2534 struct ggml_context * ctx,
2535 struct ggml_tensor * a,
2536 struct ggml_tensor * b) {
2537 GGML_ASSERT(ggml_can_repeat(a, b))if (!(ggml_can_repeat(a, b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2537, "GGML_ASSERT(%s) failed", "ggml_can_repeat(a, b)");
2538
2539 struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS4, b->ne);
2540
2541 result->op = GGML_OP_REPEAT;
2542 result->src[0] = a;
2543
2544 return result;
2545}
2546
2547struct ggml_tensor * ggml_repeat_4d(
2548 struct ggml_context * ctx,
2549 struct ggml_tensor * a,
2550 int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3) {
2551 const bool_Bool can_repeat = ggml_is_empty(a) || (
2552 (ne0 % a->ne[0] == 0) &&
2553 (ne1 % a->ne[1] == 0) &&
2554 (ne2 % a->ne[2] == 0) &&
2555 (ne3 % a->ne[3] == 0)
2556 );
2557 GGML_ASSERT(can_repeat)if (!(can_repeat)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2557, "GGML_ASSERT(%s) failed", "can_repeat");
2558
2559 struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type, ne0, ne1, ne2, ne3);
2560
2561 result->op = GGML_OP_REPEAT;
2562 result->src[0] = a;
2563
2564 return result;
2565}
2566
2567// ggml_repeat_back
2568
2569struct ggml_tensor * ggml_repeat_back(
2570 struct ggml_context * ctx,
2571 struct ggml_tensor * a,
2572 struct ggml_tensor * b) {
2573 GGML_ASSERT(ggml_can_repeat(b, a))if (!(ggml_can_repeat(b, a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2573, "GGML_ASSERT(%s) failed", "ggml_can_repeat(b, a)");
2574
2575 struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS4, b->ne);
2576
2577 result->op = GGML_OP_REPEAT_BACK;
2578 result->src[0] = a;
2579
2580 return result;
2581}
2582
2583// ggml_concat
2584
2585struct ggml_tensor * ggml_concat(
2586 struct ggml_context * ctx,
2587 struct ggml_tensor * a,
2588 struct ggml_tensor * b,
2589 int dim) {
2590 GGML_ASSERT(dim >= 0 && dim < GGML_MAX_DIMS)if (!(dim >= 0 && dim < 4)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2590, "GGML_ASSERT(%s) failed", "dim >= 0 && dim < GGML_MAX_DIMS"
);
2591 GGML_ASSERT(a->type == b->type)if (!(a->type == b->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2591, "GGML_ASSERT(%s) failed", "a->type == b->type");
2592
2593 int64_t ne[GGML_MAX_DIMS4];
2594 for (int d = 0; d < GGML_MAX_DIMS4; ++d) {
2595 if (d == dim) {
2596 ne[d] = a->ne[d] + b->ne[d];
2597 continue;
2598 }
2599 GGML_ASSERT(a->ne[d] == b->ne[d])if (!(a->ne[d] == b->ne[d])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2599, "GGML_ASSERT(%s) failed", "a->ne[d] == b->ne[d]"
);
2600 ne[d] = a->ne[d];
2601 }
2602
2603 struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS4, ne);
2604
2605 ggml_set_op_params_i32(result, 0, dim);
2606
2607 result->op = GGML_OP_CONCAT;
2608 result->src[0] = a;
2609 result->src[1] = b;
2610
2611 return result;
2612}
2613
2614// ggml_abs
2615
2616struct ggml_tensor * ggml_abs(
2617 struct ggml_context * ctx,
2618 struct ggml_tensor * a) {
2619 return ggml_unary(ctx, a, GGML_UNARY_OP_ABS);
2620}
2621
2622struct ggml_tensor * ggml_abs_inplace(
2623 struct ggml_context * ctx,
2624 struct ggml_tensor * a) {
2625 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_ABS);
2626}
2627
2628// ggml_sgn
2629
2630struct ggml_tensor * ggml_sgn(
2631 struct ggml_context * ctx,
2632 struct ggml_tensor * a) {
2633 return ggml_unary(ctx, a, GGML_UNARY_OP_SGN);
2634}
2635
2636struct ggml_tensor * ggml_sgn_inplace(
2637 struct ggml_context * ctx,
2638 struct ggml_tensor * a) {
2639 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_SGN);
2640}
2641
2642// ggml_neg
2643
2644struct ggml_tensor * ggml_neg(
2645 struct ggml_context * ctx,
2646 struct ggml_tensor * a) {
2647 return ggml_unary(ctx, a, GGML_UNARY_OP_NEG);
2648}
2649
2650struct ggml_tensor * ggml_neg_inplace(
2651 struct ggml_context * ctx,
2652 struct ggml_tensor * a) {
2653 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_NEG);
2654}
2655
2656// ggml_step
2657
2658struct ggml_tensor * ggml_step(
2659 struct ggml_context * ctx,
2660 struct ggml_tensor * a) {
2661 return ggml_unary(ctx, a, GGML_UNARY_OP_STEP);
2662}
2663
2664struct ggml_tensor * ggml_step_inplace(
2665 struct ggml_context * ctx,
2666 struct ggml_tensor * a) {
2667 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_STEP);
2668}
2669
2670// ggml_tanh
2671
2672struct ggml_tensor * ggml_tanh(
2673 struct ggml_context * ctx,
2674 struct ggml_tensor * a) {
2675 return ggml_unary(ctx, a, GGML_UNARY_OP_TANH);
2676}
2677
2678struct ggml_tensor * ggml_tanh_inplace(
2679 struct ggml_context * ctx,
2680 struct ggml_tensor * a) {
2681 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_TANH);
2682}
2683
2684// ggml_elu
2685
2686struct ggml_tensor * ggml_elu(
2687 struct ggml_context * ctx,
2688 struct ggml_tensor * a) {
2689 return ggml_unary(ctx, a, GGML_UNARY_OP_ELU);
2690}
2691
2692struct ggml_tensor * ggml_elu_inplace(
2693 struct ggml_context * ctx,
2694 struct ggml_tensor * a) {
2695 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_ELU);
2696}
2697
2698// ggml_relu
2699
2700struct ggml_tensor * ggml_relu(
2701 struct ggml_context * ctx,
2702 struct ggml_tensor * a) {
2703 return ggml_unary(ctx, a, GGML_UNARY_OP_RELU);
2704}
2705
2706struct ggml_tensor * ggml_relu_inplace(
2707 struct ggml_context * ctx,
2708 struct ggml_tensor * a) {
2709 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_RELU);
2710}
2711
2712// ggml_leaky_relu
2713
2714struct ggml_tensor * ggml_leaky_relu(
2715 struct ggml_context * ctx,
2716 struct ggml_tensor * a,
2717 float negative_slope,
2718 bool_Bool inplace) {
2719 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
2720
2721 ggml_set_op_params(result, &negative_slope, sizeof(negative_slope));
2722
2723 result->op = GGML_OP_LEAKY_RELU;
2724 result->src[0] = a;
2725
2726 return result;
2727}
2728
2729// ggml_sigmoid
2730
2731struct ggml_tensor * ggml_sigmoid(
2732 struct ggml_context * ctx,
2733 struct ggml_tensor * a) {
2734 return ggml_unary(ctx, a, GGML_UNARY_OP_SIGMOID);
2735}
2736
2737struct ggml_tensor * ggml_sigmoid_inplace(
2738 struct ggml_context * ctx,
2739 struct ggml_tensor * a) {
2740 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_SIGMOID);
2741}
2742
2743// ggml_gelu
2744
2745struct ggml_tensor * ggml_gelu(
2746 struct ggml_context * ctx,
2747 struct ggml_tensor * a) {
2748 return ggml_unary(ctx, a, GGML_UNARY_OP_GELU);
2749}
2750
2751struct ggml_tensor * ggml_gelu_inplace(
2752 struct ggml_context * ctx,
2753 struct ggml_tensor * a) {
2754 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_GELU);
2755}
2756
2757// ggml_gelu_erf
2758
2759struct ggml_tensor * ggml_gelu_erf(
2760 struct ggml_context * ctx,
2761 struct ggml_tensor * a) {
2762 return ggml_unary(ctx, a, GGML_UNARY_OP_GELU_ERF);
2763}
2764
2765struct ggml_tensor * ggml_gelu_erf_inplace(
2766 struct ggml_context * ctx,
2767 struct ggml_tensor * a) {
2768 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_GELU_ERF);
2769}
2770
2771// ggml_gelu_quick
2772
2773struct ggml_tensor * ggml_gelu_quick(
2774 struct ggml_context * ctx,
2775 struct ggml_tensor * a) {
2776 return ggml_unary(ctx, a, GGML_UNARY_OP_GELU_QUICK);
2777}
2778
2779struct ggml_tensor * ggml_gelu_quick_inplace(
2780 struct ggml_context * ctx,
2781 struct ggml_tensor * a) {
2782 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_GELU_QUICK);
2783}
2784
2785// ggml_silu
2786
2787struct ggml_tensor * ggml_silu(
2788 struct ggml_context * ctx,
2789 struct ggml_tensor * a) {
2790 return ggml_unary(ctx, a, GGML_UNARY_OP_SILU);
2791}
2792
2793struct ggml_tensor * ggml_silu_inplace(
2794 struct ggml_context * ctx,
2795 struct ggml_tensor * a) {
2796 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_SILU);
2797}
2798
2799// ggml_xielu
2800
2801struct ggml_tensor * ggml_xielu(
2802 struct ggml_context * ctx,
2803 struct ggml_tensor * a,
2804 float alpha_n,
2805 float alpha_p,
2806 float beta,
2807 float eps) {
2808 struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
2809
2810 ggml_set_op_params_i32(result, 0, (int32_t) GGML_UNARY_OP_XIELU);
2811 ggml_set_op_params_f32(result, 1, beta + ggml_compute_softplus_f32(alpha_n));
2812 ggml_set_op_params_f32(result, 2, ggml_compute_softplus_f32(alpha_p));
2813 ggml_set_op_params_f32(result, 3, beta);
2814 ggml_set_op_params_f32(result, 4, eps);
2815
2816 result->op = GGML_OP_UNARY;
2817 result->src[0] = a;
2818
2819 return result;
2820}
2821
2822// ggml_silu_back
2823
2824struct ggml_tensor * ggml_silu_back(
2825 struct ggml_context * ctx,
2826 struct ggml_tensor * a,
2827 struct ggml_tensor * b) {
2828 struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
2829
2830 result->op = GGML_OP_SILU_BACK;
2831 result->src[0] = a;
2832 result->src[1] = b;
2833
2834 return result;
2835}
2836
2837// ggml hardswish
2838
2839struct ggml_tensor * ggml_hardswish(
2840 struct ggml_context * ctx,
2841 struct ggml_tensor * a) {
2842 return ggml_unary(ctx, a, GGML_UNARY_OP_HARDSWISH);
2843}
2844
2845// ggml hardsigmoid
2846
2847struct ggml_tensor * ggml_hardsigmoid(
2848 struct ggml_context * ctx,
2849 struct ggml_tensor * a) {
2850 return ggml_unary(ctx, a, GGML_UNARY_OP_HARDSIGMOID);
2851}
2852
2853// ggml exp
2854
2855struct ggml_tensor * ggml_exp(
2856 struct ggml_context * ctx,
2857 struct ggml_tensor * a) {
2858 return ggml_unary(ctx, a, GGML_UNARY_OP_EXP);
2859}
2860
2861struct ggml_tensor * ggml_exp_inplace(
2862 struct ggml_context * ctx,
2863 struct ggml_tensor * a) {
2864 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_EXP);
2865}
2866
2867// ggml_glu
2868
2869static struct ggml_tensor * ggml_glu_impl(
2870 struct ggml_context * ctx,
2871 struct ggml_tensor * a,
2872 struct ggml_tensor * b,
2873 enum ggml_glu_op op,
2874 bool_Bool swapped) {
2875 GGML_ASSERT(ggml_is_contiguous_1(a))if (!(ggml_is_contiguous_1(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2875, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(a)");
2876
2877 if (b) {
2878 GGML_ASSERT(ggml_is_contiguous_1(b))if (!(ggml_is_contiguous_1(b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2878, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_1(b)");
2879 GGML_ASSERT(ggml_are_same_shape(a, b))if (!(ggml_are_same_shape(a, b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2879, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(a, b)"
);
2880 GGML_ASSERT(a->type == b->type)if (!(a->type == b->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 2880, "GGML_ASSERT(%s) failed", "a->type == b->type");
2881 }
2882
2883 int64_t ne[GGML_MAX_DIMS4] = { a->ne[0] / 2 }; for (int i = 1; i < GGML_MAX_DIMS4; i++) ne[i] = a->ne[i];
2884 struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS4, b ? a->ne : ne, NULL((void*)0), 0);
2885
2886 ggml_set_op_params_i32(result, 0, (int32_t) op);
2887 ggml_set_op_params_i32(result, 1, (int32_t) swapped);
2888
2889 result->op = GGML_OP_GLU;
2890 result->src[0] = a;
2891 result->src[1] = b;
2892
2893 return result;
2894}
2895
2896// ggml_floor
2897
2898struct ggml_tensor * ggml_floor(
2899 struct ggml_context * ctx,
2900 struct ggml_tensor * a) {
2901 return ggml_unary(ctx, a, GGML_UNARY_OP_FLOOR);
2902}
2903
2904struct ggml_tensor * ggml_floor_inplace(
2905 struct ggml_context * ctx,
2906 struct ggml_tensor * a) {
2907 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_FLOOR);
2908}
2909
2910// ggml_ceil
2911
2912struct ggml_tensor * ggml_ceil(
2913 struct ggml_context * ctx,
2914 struct ggml_tensor * a) {
2915 return ggml_unary(ctx, a, GGML_UNARY_OP_CEIL);
2916}
2917
2918struct ggml_tensor * ggml_ceil_inplace(
2919 struct ggml_context * ctx,
2920 struct ggml_tensor * a) {
2921 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_CEIL);
2922}
2923
2924//ggml_round
2925
2926struct ggml_tensor * ggml_round(
2927 struct ggml_context * ctx,
2928 struct ggml_tensor * a) {
2929 return ggml_unary(ctx, a, GGML_UNARY_OP_ROUND);
2930}
2931
2932struct ggml_tensor * ggml_round_inplace(
2933 struct ggml_context * ctx,
2934 struct ggml_tensor * a) {
2935 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_ROUND);
2936}
2937
2938//ggml_trunc
2939
2940struct ggml_tensor * ggml_trunc(
2941 struct ggml_context * ctx,
2942 struct ggml_tensor * a) {
2943 return ggml_unary(ctx, a, GGML_UNARY_OP_TRUNC);
2944}
2945
2946struct ggml_tensor * ggml_trunc_inplace(
2947 struct ggml_context * ctx,
2948 struct ggml_tensor * a) {
2949 return ggml_unary_inplace(ctx, a, GGML_UNARY_OP_TRUNC);
2950}
2951
2952struct ggml_tensor * ggml_glu(
2953 struct ggml_context * ctx,
2954 struct ggml_tensor * a,
2955 enum ggml_glu_op op,
2956 bool_Bool swapped) {
2957 return ggml_glu_impl(ctx, a, NULL((void*)0), op, swapped);
2958}
2959
2960struct ggml_tensor * ggml_glu_split(
2961 struct ggml_context * ctx,
2962 struct ggml_tensor * a,
2963 struct ggml_tensor * b,
2964 enum ggml_glu_op op) {
2965 return ggml_glu_impl(ctx, a, b, op, false0);
2966}
2967
2968// ggml_reglu
2969
2970struct ggml_tensor * ggml_reglu(
2971 struct ggml_context * ctx,
2972 struct ggml_tensor * a) {
2973 return ggml_glu_impl(ctx, a, NULL((void*)0), GGML_GLU_OP_REGLU, false0);
2974}
2975
2976struct ggml_tensor * ggml_reglu_swapped(
2977 struct ggml_context * ctx,
2978 struct ggml_tensor * a) {
2979 return ggml_glu_impl(ctx, a, NULL((void*)0), GGML_GLU_OP_REGLU, true1);
2980}
2981
2982struct ggml_tensor * ggml_reglu_split(
2983 struct ggml_context * ctx,
2984 struct ggml_tensor * a,
2985 struct ggml_tensor * b) {
2986 return ggml_glu_impl(ctx, a, b, GGML_GLU_OP_REGLU, false0);
2987}
2988
2989// ggml_geglu
2990
2991struct ggml_tensor * ggml_geglu(
2992 struct ggml_context * ctx,
2993 struct ggml_tensor * a) {
2994 return ggml_glu_impl(ctx, a, NULL((void*)0), GGML_GLU_OP_GEGLU, false0);
2995}
2996
2997struct ggml_tensor * ggml_geglu_swapped(
2998 struct ggml_context * ctx,
2999 struct ggml_tensor * a) {
3000 return ggml_glu_impl(ctx, a, NULL((void*)0), GGML_GLU_OP_GEGLU, true1);
3001}
3002
3003struct ggml_tensor * ggml_geglu_split(
3004 struct ggml_context * ctx,
3005 struct ggml_tensor * a,
3006 struct ggml_tensor * b) {
3007 return ggml_glu_impl(ctx, a, b, GGML_GLU_OP_GEGLU, false0);
3008}
3009
3010// ggml_swiglu
3011
3012struct ggml_tensor * ggml_swiglu(
3013 struct ggml_context * ctx,
3014 struct ggml_tensor * a) {
3015 return ggml_glu_impl(ctx, a, NULL((void*)0), GGML_GLU_OP_SWIGLU, false0);
3016}
3017
3018struct ggml_tensor * ggml_swiglu_swapped(
3019 struct ggml_context * ctx,
3020 struct ggml_tensor * a) {
3021 return ggml_glu_impl(ctx, a, NULL((void*)0), GGML_GLU_OP_SWIGLU, true1);
3022}
3023
3024struct ggml_tensor * ggml_swiglu_split(
3025 struct ggml_context * ctx,
3026 struct ggml_tensor * a,
3027 struct ggml_tensor * b) {
3028 return ggml_glu_impl(ctx, a, b, GGML_GLU_OP_SWIGLU, false0);
3029}
3030
3031// ggml_geglu_erf
3032
3033struct ggml_tensor * ggml_geglu_erf(
3034 struct ggml_context * ctx,
3035 struct ggml_tensor * a) {
3036 return ggml_glu_impl(ctx, a, NULL((void*)0), GGML_GLU_OP_GEGLU_ERF, false0);
3037}
3038
3039struct ggml_tensor * ggml_geglu_erf_swapped(
3040 struct ggml_context * ctx,
3041 struct ggml_tensor * a) {
3042 return ggml_glu_impl(ctx, a, NULL((void*)0), GGML_GLU_OP_GEGLU_ERF, true1);
3043}
3044
3045struct ggml_tensor * ggml_geglu_erf_split(
3046 struct ggml_context * ctx,
3047 struct ggml_tensor * a,
3048 struct ggml_tensor * b) {
3049 return ggml_glu_impl(ctx, a, b, GGML_GLU_OP_GEGLU_ERF, false0);
3050}
3051
3052// ggml_geglu_quick
3053
3054struct ggml_tensor * ggml_geglu_quick(
3055 struct ggml_context * ctx,
3056 struct ggml_tensor * a) {
3057 return ggml_glu_impl(ctx, a, NULL((void*)0), GGML_GLU_OP_GEGLU_QUICK, false0);
3058}
3059
3060struct ggml_tensor * ggml_geglu_quick_swapped(
3061 struct ggml_context * ctx,
3062 struct ggml_tensor * a) {
3063 return ggml_glu_impl(ctx, a, NULL((void*)0), GGML_GLU_OP_GEGLU_QUICK, true1);
3064}
3065
3066struct ggml_tensor * ggml_geglu_quick_split(
3067 struct ggml_context * ctx,
3068 struct ggml_tensor * a,
3069 struct ggml_tensor * b) {
3070 return ggml_glu_impl(ctx, a, b, GGML_GLU_OP_GEGLU_QUICK, false0);
3071}
3072
3073struct ggml_tensor * ggml_swiglu_oai(
3074 struct ggml_context * ctx,
3075 struct ggml_tensor * a,
3076 struct ggml_tensor * b,
3077 float alpha,
3078 float limit) {
3079 struct ggml_tensor * result = ggml_glu_impl(ctx, a, b, GGML_GLU_OP_SWIGLU_OAI, false0);
3080 ggml_set_op_params_f32(result, 2, alpha);
3081 ggml_set_op_params_f32(result, 3, limit);
3082
3083 return result;
3084}
3085
3086// ggml_norm
3087
3088static struct ggml_tensor * ggml_norm_impl(
3089 struct ggml_context * ctx,
3090 struct ggml_tensor * a,
3091 float eps,
3092 bool_Bool inplace) {
3093 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
3094
3095 ggml_set_op_params(result, &eps, sizeof(eps));
3096
3097 result->op = GGML_OP_NORM;
3098 result->src[0] = a;
3099
3100 return result;
3101}
3102
3103struct ggml_tensor * ggml_norm(
3104 struct ggml_context * ctx,
3105 struct ggml_tensor * a,
3106 float eps) {
3107 return ggml_norm_impl(ctx, a, eps, false0);
3108}
3109
3110struct ggml_tensor * ggml_norm_inplace(
3111 struct ggml_context * ctx,
3112 struct ggml_tensor * a,
3113 float eps) {
3114 return ggml_norm_impl(ctx, a, eps, true1);
3115}
3116
3117// ggml_rms_norm
3118
3119static struct ggml_tensor * ggml_rms_norm_impl(
3120 struct ggml_context * ctx,
3121 struct ggml_tensor * a,
3122 float eps,
3123 bool_Bool inplace) {
3124 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
3125
3126 ggml_set_op_params(result, &eps, sizeof(eps));
3127
3128 result->op = GGML_OP_RMS_NORM;
3129 result->src[0] = a;
3130
3131 return result;
3132}
3133
3134struct ggml_tensor * ggml_rms_norm(
3135 struct ggml_context * ctx,
3136 struct ggml_tensor * a,
3137 float eps) {
3138 return ggml_rms_norm_impl(ctx, a, eps, false0);
3139}
3140
3141struct ggml_tensor * ggml_rms_norm_inplace(
3142 struct ggml_context * ctx,
3143 struct ggml_tensor * a,
3144 float eps) {
3145 return ggml_rms_norm_impl(ctx, a, eps, true1);
3146}
3147
3148// ggml_rms_norm_back
3149
3150struct ggml_tensor * ggml_rms_norm_back(
3151 struct ggml_context * ctx,
3152 struct ggml_tensor * a,
3153 struct ggml_tensor * b,
3154 float eps) {
3155 struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
3156
3157 ggml_set_op_params(result, &eps, sizeof(eps));
3158
3159 result->op = GGML_OP_RMS_NORM_BACK;
3160 result->src[0] = a;
3161 result->src[1] = b;
3162
3163 return result;
3164}
3165
3166// ggml_group_norm
3167
3168static struct ggml_tensor * ggml_group_norm_impl(
3169 struct ggml_context * ctx,
3170 struct ggml_tensor * a,
3171 int n_groups,
3172 float eps,
3173 bool_Bool inplace) {
3174 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
3175
3176 ggml_set_op_params_i32(result, 0, n_groups);
3177 ggml_set_op_params_f32(result, 1, eps);
3178
3179 result->op = GGML_OP_GROUP_NORM;
3180 result->src[0] = a;
3181
3182 return result;
3183}
3184
3185struct ggml_tensor * ggml_group_norm(
3186 struct ggml_context * ctx,
3187 struct ggml_tensor * a,
3188 int n_groups,
3189 float eps) {
3190 return ggml_group_norm_impl(ctx, a, n_groups, eps, false0);
3191}
3192
3193struct ggml_tensor * ggml_group_norm_inplace(
3194 struct ggml_context * ctx,
3195 struct ggml_tensor * a,
3196 int n_groups,
3197 float eps) {
3198 return ggml_group_norm_impl(ctx, a, n_groups, eps, true1);
3199}
3200
3201// ggml_l2_norm
3202
3203static struct ggml_tensor * ggml_l2_norm_impl(
3204 struct ggml_context * ctx,
3205 struct ggml_tensor * a,
3206 float eps,
3207 bool_Bool inplace) {
3208 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
3209
3210 ggml_set_op_params_f32(result, 0, eps);
3211
3212 result->op = GGML_OP_L2_NORM;
3213 result->src[0] = a;
3214
3215 return result;
3216}
3217
3218struct ggml_tensor * ggml_l2_norm(
3219 struct ggml_context * ctx,
3220 struct ggml_tensor * a,
3221 float eps) {
3222 return ggml_l2_norm_impl(ctx, a, eps, false0);
3223}
3224
3225struct ggml_tensor * ggml_l2_norm_inplace(
3226 struct ggml_context * ctx,
3227 struct ggml_tensor * a,
3228 float eps) {
3229 return ggml_l2_norm_impl(ctx, a, eps, true1);
3230}
3231
3232// ggml_mul_mat
3233
3234static inline bool_Bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
3235 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
3236
3237 return (t0->ne[0] == t1->ne[0]) &&
3238 (t1->ne[2]%t0->ne[2] == 0) && // verify t0 is broadcastable
3239 (t1->ne[3]%t0->ne[3] == 0);
3240}
3241
3242struct ggml_tensor * ggml_mul_mat(
3243 struct ggml_context * ctx,
3244 struct ggml_tensor * a,
3245 struct ggml_tensor * b) {
3246 GGML_ASSERT(ggml_can_mul_mat(a, b))if (!(ggml_can_mul_mat(a, b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3246, "GGML_ASSERT(%s) failed", "ggml_can_mul_mat(a, b)");
3247 GGML_ASSERT(!ggml_is_transposed(a))if (!(!ggml_is_transposed(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3247, "GGML_ASSERT(%s) failed", "!ggml_is_transposed(a)");
3248
3249 const int64_t ne[4] = { a->ne[1], b->ne[1], b->ne[2], b->ne[3] };
3250 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
3251
3252 result->op = GGML_OP_MUL_MAT;
3253 result->src[0] = a;
3254 result->src[1] = b;
3255
3256 return result;
3257}
3258
3259void ggml_mul_mat_set_prec(
3260 struct ggml_tensor * a,
3261 enum ggml_prec prec) {
3262 GGML_ASSERT(a->op == GGML_OP_MUL_MAT)if (!(a->op == GGML_OP_MUL_MAT)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3262, "GGML_ASSERT(%s) failed", "a->op == GGML_OP_MUL_MAT"
);
3263
3264 const int32_t prec_i32 = (int32_t) prec;
3265
3266 ggml_set_op_params_i32(a, 0, prec_i32);
3267}
3268
3269void ggml_mul_mat_set_hint(
3270 struct ggml_tensor * a,
3271 enum ggml_op_hint hint) {
3272 GGML_ASSERT(a->op == GGML_OP_MUL_MAT)if (!(a->op == GGML_OP_MUL_MAT)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3272, "GGML_ASSERT(%s) failed", "a->op == GGML_OP_MUL_MAT"
);
3273
3274 const int32_t hint_i32 = (int32_t) hint;
3275
3276 ggml_set_op_params_i32(a, 1, hint_i32);
3277}
3278
3279// ggml_mul_mat_id
3280
3281/*
3282 c = ggml_mul_mat_id(ctx, as, b, ids);
3283
3284 as -> [cols, rows, n_expert]
3285 b -> [cols, n_expert_used, n_tokens]
3286 ids -> [n_expert_used, n_tokens] (i32)
3287 c -> [rows, n_expert_used, n_tokens]
3288
3289 in b, n_expert_used can be broadcasted to match the n_expert_used of ids
3290
3291 c ~= as[:,:,i] @ b[:,i%r,t], i = ids[e,t] for all e,t in ids
3292*/
3293struct ggml_tensor * ggml_mul_mat_id(
3294 struct ggml_context * ctx,
3295 struct ggml_tensor * as,
3296 struct ggml_tensor * b,
3297 struct ggml_tensor * ids) {
3298 GGML_ASSERT(!ggml_is_transposed(as))if (!(!ggml_is_transposed(as))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3298, "GGML_ASSERT(%s) failed", "!ggml_is_transposed(as)");
3299 GGML_ASSERT(ids->type == GGML_TYPE_I32)if (!(ids->type == GGML_TYPE_I32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3299, "GGML_ASSERT(%s) failed", "ids->type == GGML_TYPE_I32"
);
3300
3301 GGML_ASSERT(as->ne[3] == 1)if (!(as->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3301, "GGML_ASSERT(%s) failed", "as->ne[3] == 1"); // as is 3d (one matrix per expert)
3302 GGML_ASSERT(b->ne[3] == 1)if (!(b->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3302, "GGML_ASSERT(%s) failed", "b->ne[3] == 1"); // b is 3d
3303 GGML_ASSERT(ids->ne[2] == 1 && ids->ne[3] == 1)if (!(ids->ne[2] == 1 && ids->ne[3] == 1)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3303, "GGML_ASSERT(%s) failed", "ids->ne[2] == 1 && ids->ne[3] == 1"
); // ids is 2d
3304 GGML_ASSERT(ids->ne[1] == b->ne[2])if (!(ids->ne[1] == b->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3304, "GGML_ASSERT(%s) failed", "ids->ne[1] == b->ne[2]"
); // must have an expert list per b row
3305 GGML_ASSERT(as->ne[0] == b->ne[0])if (!(as->ne[0] == b->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3305, "GGML_ASSERT(%s) failed", "as->ne[0] == b->ne[0]"
); // can_mul_mat
3306 GGML_ASSERT(ids->ne[0] % b->ne[1] == 0)if (!(ids->ne[0] % b->ne[1] == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3306, "GGML_ASSERT(%s) failed", "ids->ne[0] % b->ne[1] == 0"
); // can broadcast
3307
3308 const int64_t ne[4] = { as->ne[1], ids->ne[0], b->ne[2], 1 };
3309 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
3310
3311 result->op = GGML_OP_MUL_MAT_ID;
3312 result->src[0] = as;
3313 result->src[1] = b;
3314 result->src[2] = ids;
3315
3316 return result;
3317}
3318
3319// ggml_out_prod
3320
3321static inline bool_Bool ggml_can_out_prod(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
3322 static_assert_Static_assert(GGML_MAX_DIMS4 == 4, "GGML_MAX_DIMS is not 4 - update this function");
3323
3324 return (t0->ne[1] == t1->ne[1]) &&
20
←
Array access (via field 'ne') results in a null pointer dereference
3325 (t1->ne[2]%t0->ne[2] == 0) && // verify t0 is broadcastable
3326 (t1->ne[3]%t0->ne[3] == 0);
3327}
3328
3329struct ggml_tensor * ggml_out_prod(
3330 struct ggml_context * ctx,
3331 struct ggml_tensor * a,
3332 struct ggml_tensor * b) {
3333 GGML_ASSERT(ggml_can_out_prod(a, b))if (!(ggml_can_out_prod(a, b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3333, "GGML_ASSERT(%s) failed", "ggml_can_out_prod(a, b)");
18
←
Passing null pointer value via 1st parameter 't0'
→
19
←
Calling 'ggml_can_out_prod'
→
3334 GGML_ASSERT(!ggml_is_transposed(a))if (!(!ggml_is_transposed(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3334, "GGML_ASSERT(%s) failed", "!ggml_is_transposed(a)");
3335
3336 // a is broadcastable to b for ne[2] and ne[3] -> use b->ne[2] and b->ne[3]
3337 const int64_t ne[4] = { a->ne[0], b->ne[0], b->ne[2], b->ne[3] };
3338 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
3339
3340 result->op = GGML_OP_OUT_PROD;
3341 result->src[0] = a;
3342 result->src[1] = b;
3343
3344 return result;
3345}
3346
3347// ggml_scale
3348
3349static struct ggml_tensor * ggml_scale_impl(
3350 struct ggml_context * ctx,
3351 struct ggml_tensor * a,
3352 float s,
3353 float b,
3354 bool_Bool inplace) {
3355 GGML_ASSERT(ggml_is_padded_1d(a))if (!(ggml_is_padded_1d(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3355, "GGML_ASSERT(%s) failed", "ggml_is_padded_1d(a)");
3356
3357 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
3358
3359 float params[2] = { s, b };
3360 ggml_set_op_params(result, &params, sizeof(params));
3361
3362 result->op = GGML_OP_SCALE;
3363 result->src[0] = a;
3364
3365 return result;
3366}
3367
3368struct ggml_tensor * ggml_scale(
3369 struct ggml_context * ctx,
3370 struct ggml_tensor * a,
3371 float s) {
3372 return ggml_scale_impl(ctx, a, s, 0.0, false0);
3373}
3374
3375struct ggml_tensor * ggml_scale_inplace(
3376 struct ggml_context * ctx,
3377 struct ggml_tensor * a,
3378 float s) {
3379 return ggml_scale_impl(ctx, a, s, 0.0, true1);
3380}
3381
3382struct ggml_tensor * ggml_scale_bias(
3383 struct ggml_context * ctx,
3384 struct ggml_tensor * a,
3385 float s,
3386 float b) {
3387 return ggml_scale_impl(ctx, a, s, b, false0);
3388}
3389
3390struct ggml_tensor * ggml_scale_bias_inplace(
3391 struct ggml_context * ctx,
3392 struct ggml_tensor * a,
3393 float s,
3394 float b) {
3395 return ggml_scale_impl(ctx, a, s, b, true1);
3396}
3397
3398// ggml_set
3399
3400static struct ggml_tensor * ggml_set_impl(
3401 struct ggml_context * ctx,
3402 struct ggml_tensor * a,
3403 struct ggml_tensor * b,
3404 size_t nb1,
3405 size_t nb2,
3406 size_t nb3,
3407 size_t offset,
3408 bool_Bool inplace) {
3409 GGML_ASSERT(ggml_nelements(a) >= ggml_nelements(b))if (!(ggml_nelements(a) >= ggml_nelements(b))) ggml_abort(
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3409, "GGML_ASSERT(%s) failed", "ggml_nelements(a) >= ggml_nelements(b)"
);
3410
3411 // make a view of the destination
3412 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
3413
3414 GGML_ASSERT(offset < (size_t)(1 << 30))if (!(offset < (size_t)(1 << 30))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3414, "GGML_ASSERT(%s) failed", "offset < (size_t)(1 << 30)"
);
3415 int32_t params[] = { nb1, nb2, nb3, offset, inplace ? 1 : 0 };
3416 ggml_set_op_params(result, params, sizeof(params));
3417
3418 result->op = GGML_OP_SET;
3419 result->src[0] = a;
3420 result->src[1] = b;
3421
3422 return result;
3423}
3424
3425struct ggml_tensor * ggml_set(
3426 struct ggml_context * ctx,
3427 struct ggml_tensor * a,
3428 struct ggml_tensor * b,
3429 size_t nb1,
3430 size_t nb2,
3431 size_t nb3,
3432 size_t offset) {
3433 return ggml_set_impl(ctx, a, b, nb1, nb2, nb3, offset, false0);
3434}
3435
3436struct ggml_tensor * ggml_set_inplace(
3437 struct ggml_context * ctx,
3438 struct ggml_tensor * a,
3439 struct ggml_tensor * b,
3440 size_t nb1,
3441 size_t nb2,
3442 size_t nb3,
3443 size_t offset) {
3444 return ggml_set_impl(ctx, a, b, nb1, nb2, nb3, offset, true1);
3445}
3446
3447struct ggml_tensor * ggml_set_1d(
3448 struct ggml_context * ctx,
3449 struct ggml_tensor * a,
3450 struct ggml_tensor * b,
3451 size_t offset) {
3452 return ggml_set_impl(ctx, a, b, a->nb[1], a->nb[2], a->nb[3], offset, false0);
3453}
3454
3455struct ggml_tensor * ggml_set_1d_inplace(
3456 struct ggml_context * ctx,
3457 struct ggml_tensor * a,
3458 struct ggml_tensor * b,
3459 size_t offset) {
3460 return ggml_set_impl(ctx, a, b, a->nb[1], a->nb[2], a->nb[3], offset, true1);
3461}
3462
3463struct ggml_tensor * ggml_set_2d(
3464 struct ggml_context * ctx,
3465 struct ggml_tensor * a,
3466 struct ggml_tensor * b,
3467 size_t nb1,
3468 size_t offset) {
3469 return ggml_set_impl(ctx, a, b, nb1, a->nb[2], a->nb[3], offset, false0);
3470}
3471
3472struct ggml_tensor * ggml_set_2d_inplace(
3473 struct ggml_context * ctx,
3474 struct ggml_tensor * a,
3475 struct ggml_tensor * b,
3476 size_t nb1,
3477 size_t offset) {
3478 return ggml_set_impl(ctx, a, b, nb1, a->nb[2], a->nb[3], offset, true1);
3479}
3480
3481// ggml_cpy
3482
3483static struct ggml_tensor * ggml_cpy_impl(
3484 struct ggml_context * ctx,
3485 struct ggml_tensor * a,
3486 struct ggml_tensor * b) {
3487 GGML_ASSERT(ggml_nelements(a) == ggml_nelements(b))if (!(ggml_nelements(a) == ggml_nelements(b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3487, "GGML_ASSERT(%s) failed", "ggml_nelements(a) == ggml_nelements(b)"
);
3488
3489 // make a view of the destination
3490 struct ggml_tensor * result = ggml_view_tensor(ctx, b);
3491 if (strlen(b->name) > 0) {
3492 ggml_format_name(result, "%s (copy of %s)", b->name, a->name);
3493 } else {
3494 ggml_format_name(result, "%s (copy)", a->name);
3495 }
3496
3497 result->op = GGML_OP_CPY;
3498 result->src[0] = a;
3499 result->src[1] = b;
3500
3501 return result;
3502}
3503
3504struct ggml_tensor * ggml_cpy(
3505 struct ggml_context * ctx,
3506 struct ggml_tensor * a,
3507 struct ggml_tensor * b) {
3508 return ggml_cpy_impl(ctx, a, b);
3509}
3510
3511struct ggml_tensor * ggml_cast(
3512 struct ggml_context * ctx,
3513 struct ggml_tensor * a,
3514 enum ggml_type type) {
3515 struct ggml_tensor * result = ggml_new_tensor(ctx, type, GGML_MAX_DIMS4, a->ne);
3516 ggml_format_name(result, "%s (copy)", a->name);
3517
3518 result->op = GGML_OP_CPY;
3519 result->src[0] = a;
3520 result->src[1] = result; // note: this self-reference might seem redundant, but it's actually needed by some
3521 // backends for consistency with ggml_cpy_impl() above
3522
3523 return result;
3524}
3525
3526// ggml_cont
3527
3528static struct ggml_tensor * ggml_cont_impl(
3529 struct ggml_context * ctx,
3530 struct ggml_tensor * a) {
3531 struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
3532 ggml_format_name(result, "%s (cont)", a->name);
3533
3534 result->op = GGML_OP_CONT;
3535 result->src[0] = a;
3536
3537 return result;
3538}
3539
3540struct ggml_tensor * ggml_cont(
3541 struct ggml_context * ctx,
3542 struct ggml_tensor * a) {
3543 return ggml_cont_impl(ctx, a);
3544}
3545
3546// make contiguous, with new shape
3547GGML_API__attribute__ ((visibility ("default"))) extern struct ggml_tensor * ggml_cont_1d(
3548 struct ggml_context * ctx,
3549 struct ggml_tensor * a,
3550 int64_t ne0) {
3551 return ggml_cont_4d(ctx, a, ne0, 1, 1, 1);
3552}
3553
3554GGML_API__attribute__ ((visibility ("default"))) extern struct ggml_tensor * ggml_cont_2d(
3555 struct ggml_context * ctx,
3556 struct ggml_tensor * a,
3557 int64_t ne0,
3558 int64_t ne1) {
3559 return ggml_cont_4d(ctx, a, ne0, ne1, 1, 1);
3560}
3561
3562GGML_API__attribute__ ((visibility ("default"))) extern struct ggml_tensor * ggml_cont_3d(
3563 struct ggml_context * ctx,
3564 struct ggml_tensor * a,
3565 int64_t ne0,
3566 int64_t ne1,
3567 int64_t ne2) {
3568 return ggml_cont_4d(ctx, a, ne0, ne1, ne2, 1);
3569}
3570
3571struct ggml_tensor * ggml_cont_4d(
3572 struct ggml_context * ctx,
3573 struct ggml_tensor * a,
3574 int64_t ne0,
3575 int64_t ne1,
3576 int64_t ne2,
3577 int64_t ne3) {
3578 GGML_ASSERT(ggml_nelements(a) == (ne0*ne1*ne2*ne3))if (!(ggml_nelements(a) == (ne0*ne1*ne2*ne3))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3578, "GGML_ASSERT(%s) failed", "ggml_nelements(a) == (ne0*ne1*ne2*ne3)"
);
3579
3580 struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type, ne0, ne1, ne2, ne3);
3581 ggml_format_name(result, "%s (cont)", a->name);
3582
3583 result->op = GGML_OP_CONT;
3584 result->src[0] = a;
3585
3586 return result;
3587}
3588
3589// ggml_reshape
3590
3591struct ggml_tensor * ggml_reshape(
3592 struct ggml_context * ctx,
3593 struct ggml_tensor * a,
3594 struct ggml_tensor * b) {
3595 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3595, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
3596 // as only the shape of b is relevant, and not its memory layout, b is allowed to be non contiguous.
3597 GGML_ASSERT(ggml_nelements(a) == ggml_nelements(b))if (!(ggml_nelements(a) == ggml_nelements(b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3597, "GGML_ASSERT(%s) failed", "ggml_nelements(a) == ggml_nelements(b)"
);
3598
3599 struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS4, b->ne, a, 0);
3600 ggml_format_name(result, "%s (reshaped)", a->name);
3601
3602 result->op = GGML_OP_RESHAPE;
3603 result->src[0] = a;
3604
3605 return result;
3606}
3607
3608struct ggml_tensor * ggml_reshape_1d(
3609 struct ggml_context * ctx,
3610 struct ggml_tensor * a,
3611 int64_t ne0) {
3612 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3612, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
3613 GGML_ASSERT(ggml_nelements(a) == ne0)if (!(ggml_nelements(a) == ne0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3613, "GGML_ASSERT(%s) failed", "ggml_nelements(a) == ne0");
3614
3615 const int64_t ne[1] = { ne0 };
3616 struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 1, ne, a, 0);
3617 ggml_format_name(result, "%s (reshaped)", a->name);
3618
3619 result->op = GGML_OP_RESHAPE;
3620 result->src[0] = a;
3621
3622 return result;
3623}
3624
3625struct ggml_tensor * ggml_reshape_2d(
3626 struct ggml_context * ctx,
3627 struct ggml_tensor * a,
3628 int64_t ne0,
3629 int64_t ne1) {
3630 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3630, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
3631 GGML_ASSERT(ggml_nelements(a) == ne0*ne1)if (!(ggml_nelements(a) == ne0*ne1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3631, "GGML_ASSERT(%s) failed", "ggml_nelements(a) == ne0*ne1"
);
3632
3633 const int64_t ne[2] = { ne0, ne1 };
3634 struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 2, ne, a, 0);
3635 ggml_format_name(result, "%s (reshaped)", a->name);
3636
3637 result->op = GGML_OP_RESHAPE;
3638 result->src[0] = a;
3639
3640 return result;
3641}
3642
3643struct ggml_tensor * ggml_reshape_3d(
3644 struct ggml_context * ctx,
3645 struct ggml_tensor * a,
3646 int64_t ne0,
3647 int64_t ne1,
3648 int64_t ne2) {
3649 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3649, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
3650 GGML_ASSERT(ggml_nelements(a) == ne0*ne1*ne2)if (!(ggml_nelements(a) == ne0*ne1*ne2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3650, "GGML_ASSERT(%s) failed", "ggml_nelements(a) == ne0*ne1*ne2"
);
3651
3652 const int64_t ne[3] = { ne0, ne1, ne2 };
3653 struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 3, ne, a, 0);
3654 ggml_format_name(result, "%s (reshaped)", a->name);
3655
3656 result->op = GGML_OP_RESHAPE;
3657 result->src[0] = a;
3658
3659 return result;
3660}
3661
3662struct ggml_tensor * ggml_reshape_4d(
3663 struct ggml_context * ctx,
3664 struct ggml_tensor * a,
3665 int64_t ne0,
3666 int64_t ne1,
3667 int64_t ne2,
3668 int64_t ne3) {
3669 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3669, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
3670 GGML_ASSERT(ggml_nelements(a) == ne0*ne1*ne2*ne3)if (!(ggml_nelements(a) == ne0*ne1*ne2*ne3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3670, "GGML_ASSERT(%s) failed", "ggml_nelements(a) == ne0*ne1*ne2*ne3"
);
3671
3672 const int64_t ne[4] = { ne0, ne1, ne2, ne3 };
3673 struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 4, ne, a, 0);
3674 ggml_format_name(result, "%s (reshaped)", a->name);
3675
3676 result->op = GGML_OP_RESHAPE;
3677 result->src[0] = a;
3678
3679 return result;
3680}
3681
3682static struct ggml_tensor * ggml_view_impl(
3683 struct ggml_context * ctx,
3684 struct ggml_tensor * a,
3685 int n_dims,
3686 const int64_t * ne,
3687 size_t offset) {
3688 struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, n_dims, ne, a, offset);
3689 ggml_format_name(result, "%s (view)", a->name);
3690
3691 ggml_set_op_params(result, &offset, sizeof(offset));
3692
3693 result->op = GGML_OP_VIEW;
3694 result->src[0] = a;
3695
3696 return result;
3697}
3698
3699// ggml_view_1d
3700
3701struct ggml_tensor * ggml_view_1d(
3702 struct ggml_context * ctx,
3703 struct ggml_tensor * a,
3704 int64_t ne0,
3705 size_t offset) {
3706 struct ggml_tensor * result = ggml_view_impl(ctx, a, 1, &ne0, offset);
3707
3708 return result;
3709}
3710
3711// ggml_view_2d
3712
3713struct ggml_tensor * ggml_view_2d(
3714 struct ggml_context * ctx,
3715 struct ggml_tensor * a,
3716 int64_t ne0,
3717 int64_t ne1,
3718 size_t nb1,
3719 size_t offset) {
3720 const int64_t ne[2] = { ne0, ne1 };
3721
3722 struct ggml_tensor * result = ggml_view_impl(ctx, a, 2, ne, offset);
3723
3724 result->nb[1] = nb1;
3725 result->nb[2] = result->nb[1]*ne1;
3726 result->nb[3] = result->nb[2];
3727
3728 return result;
3729}
3730
3731// ggml_view_3d
3732
3733struct ggml_tensor * ggml_view_3d(
3734 struct ggml_context * ctx,
3735 struct ggml_tensor * a,
3736 int64_t ne0,
3737 int64_t ne1,
3738 int64_t ne2,
3739 size_t nb1,
3740 size_t nb2,
3741 size_t offset) {
3742 const int64_t ne[3] = { ne0, ne1, ne2 };
3743
3744 struct ggml_tensor * result = ggml_view_impl(ctx, a, 3, ne, offset);
3745
3746 result->nb[1] = nb1;
3747 result->nb[2] = nb2;
3748 result->nb[3] = result->nb[2]*ne2;
3749
3750 return result;
3751}
3752
3753// ggml_view_4d
3754
3755struct ggml_tensor * ggml_view_4d(
3756 struct ggml_context * ctx,
3757 struct ggml_tensor * a,
3758 int64_t ne0,
3759 int64_t ne1,
3760 int64_t ne2,
3761 int64_t ne3,
3762 size_t nb1,
3763 size_t nb2,
3764 size_t nb3,
3765 size_t offset) {
3766 const int64_t ne[4] = { ne0, ne1, ne2, ne3 };
3767
3768 struct ggml_tensor * result = ggml_view_impl(ctx, a, 4, ne, offset);
3769
3770 result->nb[1] = nb1;
3771 result->nb[2] = nb2;
3772 result->nb[3] = nb3;
3773
3774 return result;
3775}
3776
3777// ggml_permute
3778
3779struct ggml_tensor * ggml_permute(
3780 struct ggml_context * ctx,
3781 struct ggml_tensor * a,
3782 int axis0,
3783 int axis1,
3784 int axis2,
3785 int axis3) {
3786 GGML_ASSERT(axis0 >= 0 && axis0 < GGML_MAX_DIMS)if (!(axis0 >= 0 && axis0 < 4)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3786, "GGML_ASSERT(%s) failed", "axis0 >= 0 && axis0 < GGML_MAX_DIMS"
);
3787 GGML_ASSERT(axis1 >= 0 && axis1 < GGML_MAX_DIMS)if (!(axis1 >= 0 && axis1 < 4)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3787, "GGML_ASSERT(%s) failed", "axis1 >= 0 && axis1 < GGML_MAX_DIMS"
);
3788 GGML_ASSERT(axis2 >= 0 && axis2 < GGML_MAX_DIMS)if (!(axis2 >= 0 && axis2 < 4)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3788, "GGML_ASSERT(%s) failed", "axis2 >= 0 && axis2 < GGML_MAX_DIMS"
);
3789 GGML_ASSERT(axis3 >= 0 && axis3 < GGML_MAX_DIMS)if (!(axis3 >= 0 && axis3 < 4)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3789, "GGML_ASSERT(%s) failed", "axis3 >= 0 && axis3 < GGML_MAX_DIMS"
);
3790
3791 GGML_ASSERT(axis0 != axis1)if (!(axis0 != axis1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3791, "GGML_ASSERT(%s) failed", "axis0 != axis1");
3792 GGML_ASSERT(axis0 != axis2)if (!(axis0 != axis2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3792, "GGML_ASSERT(%s) failed", "axis0 != axis2");
3793 GGML_ASSERT(axis0 != axis3)if (!(axis0 != axis3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3793, "GGML_ASSERT(%s) failed", "axis0 != axis3");
3794 GGML_ASSERT(axis1 != axis2)if (!(axis1 != axis2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3794, "GGML_ASSERT(%s) failed", "axis1 != axis2");
3795 GGML_ASSERT(axis1 != axis3)if (!(axis1 != axis3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3795, "GGML_ASSERT(%s) failed", "axis1 != axis3");
3796 GGML_ASSERT(axis2 != axis3)if (!(axis2 != axis3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3796, "GGML_ASSERT(%s) failed", "axis2 != axis3");
3797
3798 struct ggml_tensor * result = ggml_view_tensor(ctx, a);
3799 ggml_format_name(result, "%s (permuted)", a->name);
3800
3801 int ne[GGML_MAX_DIMS4];
3802 int nb[GGML_MAX_DIMS4];
3803
3804 ne[axis0] = a->ne[0];
3805 ne[axis1] = a->ne[1];
3806 ne[axis2] = a->ne[2];
3807 ne[axis3] = a->ne[3];
3808
3809 nb[axis0] = a->nb[0];
3810 nb[axis1] = a->nb[1];
3811 nb[axis2] = a->nb[2];
3812 nb[axis3] = a->nb[3];
3813
3814 result->ne[0] = ne[0];
3815 result->ne[1] = ne[1];
3816 result->ne[2] = ne[2];
3817 result->ne[3] = ne[3];
3818
3819 result->nb[0] = nb[0];
3820 result->nb[1] = nb[1];
3821 result->nb[2] = nb[2];
3822 result->nb[3] = nb[3];
3823
3824 result->op = GGML_OP_PERMUTE;
3825 result->src[0] = a;
3826
3827 int32_t params[] = { axis0, axis1, axis2, axis3 };
3828 ggml_set_op_params(result, params, sizeof(params));
3829
3830 return result;
3831}
3832
3833// ggml_transpose
3834
3835struct ggml_tensor * ggml_transpose(
3836 struct ggml_context * ctx,
3837 struct ggml_tensor * a) {
3838 struct ggml_tensor * result = ggml_view_tensor(ctx, a);
3839 ggml_format_name(result, "%s (transposed)", a->name);
3840
3841 result->ne[0] = a->ne[1];
3842 result->ne[1] = a->ne[0];
3843
3844 result->nb[0] = a->nb[1];
3845 result->nb[1] = a->nb[0];
3846
3847 result->op = GGML_OP_TRANSPOSE;
3848 result->src[0] = a;
3849
3850 return result;
3851}
3852
3853// ggml_get_rows
3854
3855struct ggml_tensor * ggml_get_rows(
3856 struct ggml_context * ctx,
3857 struct ggml_tensor * a,
3858 struct ggml_tensor * b) {
3859 GGML_ASSERT(a->ne[2] == b->ne[1])if (!(a->ne[2] == b->ne[1])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3859, "GGML_ASSERT(%s) failed", "a->ne[2] == b->ne[1]"
);
3860 GGML_ASSERT(a->ne[3] == b->ne[2])if (!(a->ne[3] == b->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3860, "GGML_ASSERT(%s) failed", "a->ne[3] == b->ne[2]"
);
3861 GGML_ASSERT(b->ne[3] == 1)if (!(b->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3861, "GGML_ASSERT(%s) failed", "b->ne[3] == 1");
3862 GGML_ASSERT(b->type == GGML_TYPE_I32)if (!(b->type == GGML_TYPE_I32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3862, "GGML_ASSERT(%s) failed", "b->type == GGML_TYPE_I32"
);
3863
3864 // TODO: implement non F32 return
3865 enum ggml_type type = GGML_TYPE_F32;
3866 if (a->type == GGML_TYPE_I32) {
3867 type = a->type;
3868 }
3869 struct ggml_tensor * result = ggml_new_tensor_4d(ctx, type, a->ne[0], b->ne[0], b->ne[1], b->ne[2]);
3870
3871 result->op = GGML_OP_GET_ROWS;
3872 result->src[0] = a;
3873 result->src[1] = b;
3874
3875 return result;
3876}
3877
3878// ggml_get_rows_back
3879
3880struct ggml_tensor * ggml_get_rows_back(
3881 struct ggml_context * ctx,
3882 struct ggml_tensor * a,
3883 struct ggml_tensor * b,
3884 struct ggml_tensor * c) {
3885 GGML_ASSERT(ggml_is_matrix(a) && ggml_is_vector(b) && b->type == GGML_TYPE_I32)if (!(ggml_is_matrix(a) && ggml_is_vector(b) &&
b->type == GGML_TYPE_I32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3885, "GGML_ASSERT(%s) failed", "ggml_is_matrix(a) && ggml_is_vector(b) && b->type == GGML_TYPE_I32"
);
3886 GGML_ASSERT(ggml_is_matrix(c) && (a->ne[0] == c->ne[0]))if (!(ggml_is_matrix(c) && (a->ne[0] == c->ne[0
]))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3886, "GGML_ASSERT(%s) failed", "ggml_is_matrix(c) && (a->ne[0] == c->ne[0])"
);
3887
3888 // TODO: implement non F32 return
3889 //struct ggml_tensor * result = ggml_new_tensor_2d(ctx, a->type, a->ne[0], b->ne[0]);
3890 struct ggml_tensor * result = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, c->ne[0], c->ne[1]);
3891
3892 result->op = GGML_OP_GET_ROWS_BACK;
3893 result->src[0] = a;
3894 result->src[1] = b;
3895
3896 return result;
3897}
3898
3899// ggml_set_rows
3900
3901struct ggml_tensor * ggml_set_rows(
3902 struct ggml_context * ctx,
3903 struct ggml_tensor * a,
3904 struct ggml_tensor * b,
3905 struct ggml_tensor * c) {
3906 GGML_ASSERT(a->ne[0] == b->ne[0])if (!(a->ne[0] == b->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3906, "GGML_ASSERT(%s) failed", "a->ne[0] == b->ne[0]"
);
3907 GGML_ASSERT(a->ne[2] == b->ne[2])if (!(a->ne[2] == b->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3907, "GGML_ASSERT(%s) failed", "a->ne[2] == b->ne[2]"
);
3908 GGML_ASSERT(a->ne[3] == b->ne[3])if (!(a->ne[3] == b->ne[3])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3908, "GGML_ASSERT(%s) failed", "a->ne[3] == b->ne[3]"
);
3909 GGML_ASSERT(b->ne[1] == c->ne[0])if (!(b->ne[1] == c->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3909, "GGML_ASSERT(%s) failed", "b->ne[1] == c->ne[0]"
);
3910 GGML_ASSERT(b->ne[2] % c->ne[1] == 0)if (!(b->ne[2] % c->ne[1] == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3910, "GGML_ASSERT(%s) failed", "b->ne[2] % c->ne[1] == 0"
);
3911 GGML_ASSERT(b->ne[3] % c->ne[2] == 0)if (!(b->ne[3] % c->ne[2] == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3911, "GGML_ASSERT(%s) failed", "b->ne[3] % c->ne[2] == 0"
);
3912 GGML_ASSERT(c->ne[3] == 1)if (!(c->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3912, "GGML_ASSERT(%s) failed", "c->ne[3] == 1");
3913 GGML_ASSERT(b->type == GGML_TYPE_F32)if (!(b->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3913, "GGML_ASSERT(%s) failed", "b->type == GGML_TYPE_F32"
);
3914 GGML_ASSERT(c->type == GGML_TYPE_I64 || c->type == GGML_TYPE_I32)if (!(c->type == GGML_TYPE_I64 || c->type == GGML_TYPE_I32
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3914, "GGML_ASSERT(%s) failed", "c->type == GGML_TYPE_I64 || c->type == GGML_TYPE_I32"
);
3915
3916 GGML_ASSERT(ggml_is_contiguous_rows(a))if (!(ggml_is_contiguous_rows(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3916, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_rows(a)"
);
3917 GGML_ASSERT(ggml_is_contiguous_rows(b))if (!(ggml_is_contiguous_rows(b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3917, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_rows(b)"
);
3918
3919 struct ggml_tensor * result = ggml_view_tensor(ctx, a);
3920
3921 result->op = GGML_OP_SET_ROWS;
3922 result->src[0] = b;
3923 result->src[1] = c;
3924 result->src[2] = a; // note: order is weird due to legacy reasons (https://github.com/ggml-org/llama.cpp/pull/16063#discussion_r2385795931)
3925
3926 return result;
3927}
3928
3929// ggml_diag
3930
3931struct ggml_tensor * ggml_diag(
3932 struct ggml_context * ctx,
3933 struct ggml_tensor * a) {
3934 GGML_ASSERT(a->ne[1] == 1)if (!(a->ne[1] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 3934, "GGML_ASSERT(%s) failed", "a->ne[1] == 1");
3935
3936 const int64_t ne[4] = { a->ne[0], a->ne[0], a->ne[2], a->ne[3] };
3937 struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, 4, ne);
3938
3939 result->op = GGML_OP_DIAG;
3940 result->src[0] = a;
3941
3942 return result;
3943}
3944
3945// ggml_diag_mask_inf
3946
3947static struct ggml_tensor * ggml_diag_mask_inf_impl(
3948 struct ggml_context * ctx,
3949 struct ggml_tensor * a,
3950 int n_past,
3951 bool_Bool inplace) {
3952 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
3953
3954 int32_t params[] = { n_past };
3955 ggml_set_op_params(result, params, sizeof(params));
3956
3957 result->op = GGML_OP_DIAG_MASK_INF;
3958 result->src[0] = a;
3959
3960 return result;
3961}
3962
3963struct ggml_tensor * ggml_diag_mask_inf(
3964 struct ggml_context * ctx,
3965 struct ggml_tensor * a,
3966 int n_past) {
3967 return ggml_diag_mask_inf_impl(ctx, a, n_past, false0);
3968}
3969
3970struct ggml_tensor * ggml_diag_mask_inf_inplace(
3971 struct ggml_context * ctx,
3972 struct ggml_tensor * a,
3973 int n_past) {
3974 return ggml_diag_mask_inf_impl(ctx, a, n_past, true1);
3975}
3976
3977// ggml_diag_mask_zero
3978
3979static struct ggml_tensor * ggml_diag_mask_zero_impl(
3980 struct ggml_context * ctx,
3981 struct ggml_tensor * a,
3982 int n_past,
3983 bool_Bool inplace) {
3984 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
3985
3986 int32_t params[] = { n_past };
3987 ggml_set_op_params(result, params, sizeof(params));
3988
3989 result->op = GGML_OP_DIAG_MASK_ZERO;
3990 result->src[0] = a;
3991
3992 return result;
3993}
3994
3995struct ggml_tensor * ggml_diag_mask_zero(
3996 struct ggml_context * ctx,
3997 struct ggml_tensor * a,
3998 int n_past) {
3999 return ggml_diag_mask_zero_impl(ctx, a, n_past, false0);
4000}
4001
4002struct ggml_tensor * ggml_diag_mask_zero_inplace(
4003 struct ggml_context * ctx,
4004 struct ggml_tensor * a,
4005 int n_past) {
4006 return ggml_diag_mask_zero_impl(ctx, a, n_past, true1);
4007}
4008
4009// ggml_soft_max
4010
4011static struct ggml_tensor * ggml_soft_max_impl(
4012 struct ggml_context * ctx,
4013 struct ggml_tensor * a,
4014 struct ggml_tensor * mask,
4015 float scale,
4016 float max_bias,
4017 bool_Bool inplace) {
4018 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4018, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
4019
4020 if (mask) {
4021 GGML_ASSERT(mask->type == GGML_TYPE_F16 || mask->type == GGML_TYPE_F32)if (!(mask->type == GGML_TYPE_F16 || mask->type == GGML_TYPE_F32
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4021, "GGML_ASSERT(%s) failed", "mask->type == GGML_TYPE_F16 || mask->type == GGML_TYPE_F32"
);
4022 GGML_ASSERT(ggml_is_contiguous(mask))if (!(ggml_is_contiguous(mask))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4022, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(mask)");
4023 GGML_ASSERT(mask->ne[0] == a->ne[0])if (!(mask->ne[0] == a->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4023, "GGML_ASSERT(%s) failed", "mask->ne[0] == a->ne[0]"
);
4024 GGML_ASSERT(mask->ne[1] >= a->ne[1])if (!(mask->ne[1] >= a->ne[1])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4024, "GGML_ASSERT(%s) failed", "mask->ne[1] >= a->ne[1]"
);
4025 GGML_ASSERT(a->ne[2]%mask->ne[2] == 0)if (!(a->ne[2]%mask->ne[2] == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4025, "GGML_ASSERT(%s) failed", "a->ne[2]%mask->ne[2] == 0"
);
4026 GGML_ASSERT(a->ne[3]%mask->ne[3] == 0)if (!(a->ne[3]%mask->ne[3] == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4026, "GGML_ASSERT(%s) failed", "a->ne[3]%mask->ne[3] == 0"
);
4027 }
4028
4029 if (max_bias > 0.0f) {
4030 GGML_ASSERT(mask)if (!(mask)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4030, "GGML_ASSERT(%s) failed", "mask");
4031 }
4032
4033 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
4034
4035 float params[] = { scale, max_bias };
4036 ggml_set_op_params(result, params, sizeof(params));
4037
4038 result->op = GGML_OP_SOFT_MAX;
4039 result->src[0] = a;
4040 result->src[1] = mask;
4041
4042 return result;
4043}
4044
4045struct ggml_tensor * ggml_soft_max(
4046 struct ggml_context * ctx,
4047 struct ggml_tensor * a) {
4048 return ggml_soft_max_impl(ctx, a, NULL((void*)0), 1.0f, 0.0f, false0);
4049}
4050
4051struct ggml_tensor * ggml_soft_max_inplace(
4052 struct ggml_context * ctx,
4053 struct ggml_tensor * a) {
4054 return ggml_soft_max_impl(ctx, a, NULL((void*)0), 1.0f, 0.0f, true1);
4055}
4056
4057struct ggml_tensor * ggml_soft_max_ext(
4058 struct ggml_context * ctx,
4059 struct ggml_tensor * a,
4060 struct ggml_tensor * mask,
4061 float scale,
4062 float max_bias) {
4063 return ggml_soft_max_impl(ctx, a, mask, scale, max_bias, false0);
4064}
4065
4066struct ggml_tensor * ggml_soft_max_ext_inplace(
4067 struct ggml_context * ctx,
4068 struct ggml_tensor * a,
4069 struct ggml_tensor * mask,
4070 float scale,
4071 float max_bias) {
4072 return ggml_soft_max_impl(ctx, a, mask, scale, max_bias, true1);
4073}
4074
4075void ggml_soft_max_add_sinks(
4076 struct ggml_tensor * a,
4077 struct ggml_tensor * sinks) {
4078 if (!sinks) {
4079 a->src[2] = NULL((void*)0);
4080 return;
4081 }
4082
4083 GGML_ASSERT(a->op == GGML_OP_SOFT_MAX)if (!(a->op == GGML_OP_SOFT_MAX)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4083, "GGML_ASSERT(%s) failed", "a->op == GGML_OP_SOFT_MAX"
);
4084 GGML_ASSERT(a->src[2] == NULL)if (!(a->src[2] == ((void*)0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4084, "GGML_ASSERT(%s) failed", "a->src[2] == NULL");
4085 GGML_ASSERT(a->src[0]->ne[2] == sinks->ne[0])if (!(a->src[0]->ne[2] == sinks->ne[0])) ggml_abort(
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4085, "GGML_ASSERT(%s) failed", "a->src[0]->ne[2] == sinks->ne[0]"
);
4086 GGML_ASSERT(sinks->type == GGML_TYPE_F32)if (!(sinks->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4086, "GGML_ASSERT(%s) failed", "sinks->type == GGML_TYPE_F32"
);
4087
4088 a->src[2] = sinks;
4089}
4090
4091// ggml_soft_max_ext_back
4092
4093static struct ggml_tensor * ggml_soft_max_ext_back_impl(
4094 struct ggml_context * ctx,
4095 struct ggml_tensor * a,
4096 struct ggml_tensor * b,
4097 float scale,
4098 float max_bias,
4099 bool_Bool inplace) {
4100 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
4101
4102 result->op = GGML_OP_SOFT_MAX_BACK;
4103 result->src[0] = a;
4104 result->src[1] = b;
4105
4106 memcpy((float *) result->op_params + 0, &scale, sizeof(float));
4107 memcpy((float *) result->op_params + 1, &max_bias, sizeof(float));
4108
4109 return result;
4110}
4111
4112struct ggml_tensor * ggml_soft_max_ext_back(
4113 struct ggml_context * ctx,
4114 struct ggml_tensor * a,
4115 struct ggml_tensor * b,
4116 float scale,
4117 float max_bias) {
4118 return ggml_soft_max_ext_back_impl(ctx, a, b, scale, max_bias, false0);
4119}
4120
4121struct ggml_tensor * ggml_soft_max_ext_back_inplace(
4122 struct ggml_context * ctx,
4123 struct ggml_tensor * a,
4124 struct ggml_tensor * b,
4125 float scale,
4126 float max_bias) {
4127 return ggml_soft_max_ext_back_impl(ctx, a, b, scale, max_bias, true1);
4128}
4129
4130// ggml_rope
4131
4132static struct ggml_tensor * ggml_rope_impl(
4133 struct ggml_context * ctx,
4134 struct ggml_tensor * a,
4135 struct ggml_tensor * b,
4136 struct ggml_tensor * c,
4137 int n_dims,
4138 int sections[GGML_MROPE_SECTIONS4],
4139 int mode,
4140 int n_ctx_orig,
4141 float freq_base,
4142 float freq_scale,
4143 float ext_factor,
4144 float attn_factor,
4145 float beta_fast,
4146 float beta_slow,
4147 bool_Bool inplace) {
4148 GGML_ASSERT((mode & 1) == 0 && "mode & 1 == 1 is no longer supported")if (!((mode & 1) == 0 && "mode & 1 == 1 is no longer supported"
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4148, "GGML_ASSERT(%s) failed", "(mode & 1) == 0 && \"mode & 1 == 1 is no longer supported\""
);
4149
4150 GGML_ASSERT(ggml_is_vector(b))if (!(ggml_is_vector(b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4150, "GGML_ASSERT(%s) failed", "ggml_is_vector(b)");
4151 GGML_ASSERT(b->type == GGML_TYPE_I32)if (!(b->type == GGML_TYPE_I32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4151, "GGML_ASSERT(%s) failed", "b->type == GGML_TYPE_I32"
);
4152
4153 bool_Bool mrope_used = mode & GGML_ROPE_TYPE_MROPE8;
4154 if (mrope_used) {
4155 GGML_ASSERT(a->ne[2] * 4 == b->ne[0])if (!(a->ne[2] * 4 == b->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4155, "GGML_ASSERT(%s) failed", "a->ne[2] * 4 == b->ne[0]"
); // mrope expecting 4 position ids per token
4156 } else {
4157 GGML_ASSERT(a->ne[2] == b->ne[0])if (!(a->ne[2] == b->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4157, "GGML_ASSERT(%s) failed", "a->ne[2] == b->ne[0]"
);
4158 }
4159
4160 if (c) {
4161 GGML_ASSERT(c->type == GGML_TYPE_F32)if (!(c->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4161, "GGML_ASSERT(%s) failed", "c->type == GGML_TYPE_F32"
);
4162 GGML_ASSERT(c->ne[0] >= n_dims / 2)if (!(c->ne[0] >= n_dims / 2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4162, "GGML_ASSERT(%s) failed", "c->ne[0] >= n_dims / 2"
);
4163 }
4164
4165 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
4166
4167 int32_t params[15] = { /*n_past*/ 0, n_dims, mode, /*n_ctx*/ 0, n_ctx_orig };
4168 memcpy(params + 5, &freq_base, sizeof(float));
4169 memcpy(params + 6, &freq_scale, sizeof(float));
4170 memcpy(params + 7, &ext_factor, sizeof(float));
4171 memcpy(params + 8, &attn_factor, sizeof(float));
4172 memcpy(params + 9, &beta_fast, sizeof(float));
4173 memcpy(params + 10, &beta_slow, sizeof(float));
4174 if (mrope_used && sections) {
4175 memcpy(params + 11, sections, sizeof(int32_t) * GGML_MROPE_SECTIONS4);
4176 } else {
4177 memset(params + 11, 0, sizeof(int32_t) * GGML_MROPE_SECTIONS4);
4178 }
4179 ggml_set_op_params(result, params, sizeof(params));
4180
4181 result->op = GGML_OP_ROPE;
4182 result->src[0] = a;
4183 result->src[1] = b;
4184 result->src[2] = c;
4185
4186 return result;
4187}
4188
4189struct ggml_tensor * ggml_rope(
4190 struct ggml_context * ctx,
4191 struct ggml_tensor * a,
4192 struct ggml_tensor * b,
4193 int n_dims,
4194 int mode) {
4195 return ggml_rope_impl(
4196 ctx, a, b, NULL((void*)0), n_dims, NULL((void*)0), mode, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, false0
4197 );
4198}
4199
4200struct ggml_tensor * ggml_rope_multi(
4201 struct ggml_context * ctx,
4202 struct ggml_tensor * a,
4203 struct ggml_tensor * b,
4204 struct ggml_tensor * c,
4205 int n_dims,
4206 int sections[GGML_MROPE_SECTIONS4],
4207 int mode,
4208 int n_ctx_orig,
4209 float freq_base,
4210 float freq_scale,
4211 float ext_factor,
4212 float attn_factor,
4213 float beta_fast,
4214 float beta_slow) {
4215 return ggml_rope_impl(
4216 ctx, a, b, c, n_dims, sections, mode, n_ctx_orig, freq_base, freq_scale,
4217 ext_factor, attn_factor, beta_fast, beta_slow, false0
4218 );
4219}
4220
4221struct ggml_tensor * ggml_rope_multi_inplace(
4222 struct ggml_context * ctx,
4223 struct ggml_tensor * a,
4224 struct ggml_tensor * b,
4225 struct ggml_tensor * c,
4226 int n_dims,
4227 int sections[GGML_MROPE_SECTIONS4],
4228 int mode,
4229 int n_ctx_orig,
4230 float freq_base,
4231 float freq_scale,
4232 float ext_factor,
4233 float attn_factor,
4234 float beta_fast,
4235 float beta_slow) {
4236 return ggml_rope_impl(
4237 ctx, a, b, c, n_dims, sections, mode, n_ctx_orig, freq_base, freq_scale,
4238 ext_factor, attn_factor, beta_fast, beta_slow, true1
4239 );
4240}
4241
4242struct ggml_tensor * ggml_rope_inplace(
4243 struct ggml_context * ctx,
4244 struct ggml_tensor * a,
4245 struct ggml_tensor * b,
4246 int n_dims,
4247 int mode) {
4248 return ggml_rope_impl(
4249 ctx, a, b, NULL((void*)0), n_dims, NULL((void*)0), mode, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, true1
4250 );
4251}
4252
4253struct ggml_tensor * ggml_rope_ext(
4254 struct ggml_context * ctx,
4255 struct ggml_tensor * a,
4256 struct ggml_tensor * b,
4257 struct ggml_tensor * c,
4258 int n_dims,
4259 int mode,
4260 int n_ctx_orig,
4261 float freq_base,
4262 float freq_scale,
4263 float ext_factor,
4264 float attn_factor,
4265 float beta_fast,
4266 float beta_slow) {
4267 return ggml_rope_impl(
4268 ctx, a, b, c, n_dims, NULL((void*)0), mode, n_ctx_orig, freq_base, freq_scale,
4269 ext_factor, attn_factor, beta_fast, beta_slow, false0
4270 );
4271}
4272
4273struct ggml_tensor * ggml_rope_ext_inplace(
4274 struct ggml_context * ctx,
4275 struct ggml_tensor * a,
4276 struct ggml_tensor * b,
4277 struct ggml_tensor * c,
4278 int n_dims,
4279 int mode,
4280 int n_ctx_orig,
4281 float freq_base,
4282 float freq_scale,
4283 float ext_factor,
4284 float attn_factor,
4285 float beta_fast,
4286 float beta_slow) {
4287 return ggml_rope_impl(
4288 ctx, a, b, c, n_dims, NULL((void*)0), mode, n_ctx_orig, freq_base, freq_scale,
4289 ext_factor, attn_factor, beta_fast, beta_slow, true1
4290 );
4291}
4292
4293struct ggml_tensor * ggml_rope_custom(
4294 struct ggml_context * ctx,
4295 struct ggml_tensor * a,
4296 struct ggml_tensor * b,
4297 int n_dims,
4298 int mode,
4299 int n_ctx_orig,
4300 float freq_base,
4301 float freq_scale,
4302 float ext_factor,
4303 float attn_factor,
4304 float beta_fast,
4305 float beta_slow) {
4306 return ggml_rope_impl(
4307 ctx, a, b, NULL((void*)0), n_dims, NULL((void*)0), mode, n_ctx_orig, freq_base, freq_scale,
4308 ext_factor, attn_factor, beta_fast, beta_slow, false0
4309 );
4310}
4311
4312struct ggml_tensor * ggml_rope_custom_inplace(
4313 struct ggml_context * ctx,
4314 struct ggml_tensor * a,
4315 struct ggml_tensor * b,
4316 int n_dims,
4317 int mode,
4318 int n_ctx_orig,
4319 float freq_base,
4320 float freq_scale,
4321 float ext_factor,
4322 float attn_factor,
4323 float beta_fast,
4324 float beta_slow) {
4325 return ggml_rope_impl(
4326 ctx, a, b, NULL((void*)0), n_dims, NULL((void*)0), mode, n_ctx_orig, freq_base, freq_scale,
4327 ext_factor, attn_factor, beta_fast, beta_slow, true1
4328 );
4329}
4330
4331// Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get
4332// `corr_dim(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`
4333static float ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, float base) {
4334 return n_dims * logf(n_ctx_orig / (n_rot * 2 * (float)M_PI3.14159265358979323846)) / (2 * logf(base));
4335}
4336
4337void ggml_rope_yarn_corr_dims(
4338 int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]
4339) {
4340 // start and end correction dims
4341 float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base));
4342 float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base));
4343 dims[0] = MAX(0, start)((0) > (start) ? (0) : (start));
4344 dims[1] = MIN(n_dims - 1, end)((n_dims - 1) < (end) ? (n_dims - 1) : (end));
4345}
4346
4347// ggml_rope_back
4348
4349struct ggml_tensor * ggml_rope_ext_back(
4350 struct ggml_context * ctx,
4351 struct ggml_tensor * a,
4352 struct ggml_tensor * b,
4353 struct ggml_tensor * c,
4354 int n_dims,
4355 int mode,
4356 int n_ctx_orig,
4357 float freq_base,
4358 float freq_scale,
4359 float ext_factor,
4360 float attn_factor,
4361 float beta_fast,
4362 float beta_slow) {
4363 struct ggml_tensor * result = ggml_rope_ext(
4364 ctx, a, b, c, n_dims, mode, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
4365 result->op = GGML_OP_ROPE_BACK;
4366 return result;
4367}
4368
4369struct ggml_tensor * ggml_rope_multi_back(
4370 struct ggml_context * ctx,
4371 struct ggml_tensor * a,
4372 struct ggml_tensor * b,
4373 struct ggml_tensor * c,
4374 int n_dims,
4375 int sections[4],
4376 int mode,
4377 int n_ctx_orig,
4378 float freq_base,
4379 float freq_scale,
4380 float ext_factor,
4381 float attn_factor,
4382 float beta_fast,
4383 float beta_slow) {
4384 struct ggml_tensor * result = ggml_rope_multi(
4385 ctx, a, b, c, n_dims, sections, mode, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
4386 result->op = GGML_OP_ROPE_BACK;
4387 return result;
4388}
4389// ggml_clamp
4390
4391struct ggml_tensor * ggml_clamp(
4392 struct ggml_context * ctx,
4393 struct ggml_tensor * a,
4394 float min,
4395 float max) {
4396 // TODO: when implement backward, fix this:
4397 struct ggml_tensor * result = ggml_view_tensor(ctx, a);
4398
4399 float params[] = { min, max };
4400 ggml_set_op_params(result, params, sizeof(params));
4401
4402 result->op = GGML_OP_CLAMP;
4403 result->src[0] = a;
4404
4405 return result;
4406}
4407
4408static int64_t ggml_calc_conv_output_size(int64_t ins, int64_t ks, int s, int p, int d) {
4409 return (ins + 2 * p - d * (ks - 1) - 1) / s + 1;
4410}
4411
4412// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
4413// a: [OC，IC, KH, KW]
4414// b: [N, IC, IH, IW]
4415// result: [N, OH, OW, IC*KH*KW]
4416struct ggml_tensor * ggml_im2col(
4417 struct ggml_context * ctx,
4418 struct ggml_tensor * a,
4419 struct ggml_tensor * b,
4420 int s0,
4421 int s1,
4422 int p0,
4423 int p1,
4424 int d0,
4425 int d1,
4426 bool_Bool is_2D,
4427 enum ggml_type dst_type) {
4428 if (is_2D) {
4429 GGML_ASSERT(a->ne[2] == b->ne[2])if (!(a->ne[2] == b->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4429, "GGML_ASSERT(%s) failed", "a->ne[2] == b->ne[2]"
);
4430 } else {
4431 //GGML_ASSERT(b->ne[1] % a->ne[1] == 0);
4432 GGML_ASSERT(b->ne[1] == a->ne[1])if (!(b->ne[1] == a->ne[1])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4432, "GGML_ASSERT(%s) failed", "b->ne[1] == a->ne[1]"
);
4433 GGML_ASSERT(b->ne[3] == 1)if (!(b->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4433, "GGML_ASSERT(%s) failed", "b->ne[3] == 1");
4434 }
4435
4436 const int64_t OH = is_2D ? ggml_calc_conv_output_size(b->ne[1], a->ne[1], s1, p1, d1) : 0;
4437 const int64_t OW = ggml_calc_conv_output_size(b->ne[0], a->ne[0], s0, p0, d0);
4438
4439 GGML_ASSERT((!is_2D || OH > 0) && "b too small compared to a")if (!((!is_2D || OH > 0) && "b too small compared to a"
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4439, "GGML_ASSERT(%s) failed", "(!is_2D || OH > 0) && \"b too small compared to a\""
);
4440 GGML_ASSERT((OW > 0) && "b too small compared to a")if (!((OW > 0) && "b too small compared to a")) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4440, "GGML_ASSERT(%s) failed", "(OW > 0) && \"b too small compared to a\""
);
4441
4442 const int64_t ne[4] = {
4443 is_2D ? (a->ne[2] * a->ne[1] * a->ne[0]) : a->ne[1] * a->ne[0],
4444 OW,
4445 is_2D ? OH : b->ne[2],
4446 is_2D ? b->ne[3] : 1,
4447 };
4448
4449 struct ggml_tensor * result = ggml_new_tensor(ctx, dst_type, 4, ne);
4450 int32_t params[] = { s0, s1, p0, p1, d0, d1, (is_2D ? 1 : 0) };
4451 ggml_set_op_params(result, params, sizeof(params));
4452
4453 result->op = GGML_OP_IM2COL;
4454 result->src[0] = a;
4455 result->src[1] = b;
4456
4457 return result;
4458}
4459
4460struct ggml_tensor * ggml_im2col_back(
4461 struct ggml_context * ctx,
4462 struct ggml_tensor * a,
4463 struct ggml_tensor * b,
4464 int64_t * ne,
4465 int s0,
4466 int s1,
4467 int p0,
4468 int p1,
4469 int d0,
4470 int d1,
4471 bool_Bool is_2D) {
4472 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
4473 int32_t params[] = { s0, s1, p0, p1, d0, d1, (is_2D ? 1 : 0) };
4474 ggml_set_op_params(result, params, sizeof(params));
4475
4476 result->op = GGML_OP_IM2COL_BACK;
4477 result->src[0] = a;
4478 result->src[1] = b;
4479
4480 return result;
4481}
4482
4483// ggml_conv_1d
4484
4485struct ggml_tensor * ggml_conv_1d(
4486 struct ggml_context * ctx,
4487 struct ggml_tensor * a,
4488 struct ggml_tensor * b,
4489 int s0,
4490 int p0,
4491 int d0) {
4492 struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, 0, p0, 0, d0, 0, false0, GGML_TYPE_F16); // [N, OL, IC * K]
4493
4494 struct ggml_tensor * result =
4495 ggml_mul_mat(ctx,
4496 ggml_reshape_2d(ctx, im2col, im2col->ne[0], (im2col->ne[2] * im2col->ne[1])), // [N, OL, IC * K] => [N*OL, IC * K]
4497 ggml_reshape_2d(ctx, a, (a->ne[0] * a->ne[1]), a->ne[2])); // [OC，IC, K] => [OC, IC * K]
4498
4499 result = ggml_reshape_3d(ctx, result, im2col->ne[1], a->ne[2], im2col->ne[2]); // [N, OC, OL]
4500
4501 return result;
4502}
4503
4504// ggml_conv_1d_ph
4505
4506struct ggml_tensor* ggml_conv_1d_ph(
4507 struct ggml_context * ctx,
4508 struct ggml_tensor * a,
4509 struct ggml_tensor * b,
4510 int s,
4511 int d) {
4512 return ggml_conv_1d(ctx, a, b, s, a->ne[0] / 2, d);
4513}
4514
4515// ggml_conv_1d_dw
4516
4517struct ggml_tensor * ggml_conv_1d_dw(
4518 struct ggml_context * ctx,
4519 struct ggml_tensor * a,
4520 struct ggml_tensor * b,
4521 int s0,
4522 int p0,
4523 int d0) {
4524 struct ggml_tensor * new_b = ggml_reshape_4d(ctx, b, b->ne[0], 1, b->ne[1], b->ne[2]);
4525
4526 struct ggml_tensor * im2col = ggml_im2col(ctx, a, new_b, s0, 0, p0, 0, d0, 0, false0, GGML_TYPE_F16);
4527
4528 struct ggml_tensor * result = ggml_mul_mat(ctx, im2col, a);
4529
4530 result = ggml_reshape_3d(ctx, result, result->ne[0], result->ne[2], 1);
4531
4532 return result;
4533}
4534
4535// ggml_conv_1d_dw_ph
4536
4537struct ggml_tensor * ggml_conv_1d_dw_ph(
4538 struct ggml_context * ctx,
4539 struct ggml_tensor * a,
4540 struct ggml_tensor * b,
4541 int s0,
4542 int d0) {
4543 return ggml_conv_1d_dw(ctx, a, b, s0, a->ne[0] / 2, d0);
4544}
4545
4546// ggml_col2im_1d
4547
4548struct ggml_tensor * ggml_col2im_1d(
4549 struct ggml_context * ctx,
4550 struct ggml_tensor * a,
4551 int s0,
4552 int oc,
4553 int p0) {
4554 GGML_ASSERT(ggml_is_matrix(a))if (!(ggml_is_matrix(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4554, "GGML_ASSERT(%s) failed", "ggml_is_matrix(a)");
4555 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4555, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
4556 GGML_ASSERT(a->type == GGML_TYPE_F32 || a->type == GGML_TYPE_F16 || a->type == GGML_TYPE_BF16)if (!(a->type == GGML_TYPE_F32 || a->type == GGML_TYPE_F16
|| a->type == GGML_TYPE_BF16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4556, "GGML_ASSERT(%s) failed", "a->type == GGML_TYPE_F32 || a->type == GGML_TYPE_F16 || a->type == GGML_TYPE_BF16"
);
4557 GGML_ASSERT(s0 > 0)if (!(s0 > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4557, "GGML_ASSERT(%s) failed", "s0 > 0");
4558 GGML_ASSERT(oc > 0)if (!(oc > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4558, "GGML_ASSERT(%s) failed", "oc > 0");
4559 GGML_ASSERT(p0 >= 0)if (!(p0 >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4559, "GGML_ASSERT(%s) failed", "p0 >= 0");
4560
4561 const int64_t K_OC = a->ne[0];
4562 const int64_t T_in = a->ne[1];
4563 const int64_t K = K_OC / oc;
4564 const int64_t T_out = (T_in - 1) * s0 + K - 2 * p0;
4565
4566 GGML_ASSERT(K_OC == K * oc)if (!(K_OC == K * oc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4566, "GGML_ASSERT(%s) failed", "K_OC == K * oc"); // a->ne[0] must be a whole number of oc blocks
4567 GGML_ASSERT(K > 0 && T_out > 0)if (!(K > 0 && T_out > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4567, "GGML_ASSERT(%s) failed", "K > 0 && T_out > 0"
);
4568
4569 const int64_t ne[4] = { T_out, oc, 1, 1 };
4570 struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, 2, ne);
4571
4572 int32_t params[] = { s0, (int32_t)oc, (int32_t)p0 };
4573 ggml_set_op_params(result, params, sizeof(params));
4574
4575 result->op = GGML_OP_COL2IM_1D;
4576 result->src[0] = a;
4577
4578 return result;
4579}
4580
4581// ggml_conv_transpose_1d
4582
4583static int64_t ggml_calc_conv_transpose_1d_output_size(int64_t ins, int64_t ks, int s, int p, int d) {
4584 return (ins - 1) * s - 2 * p + d * (ks - 1) + 1;
4585}
4586
4587GGML_API__attribute__ ((visibility ("default"))) extern struct ggml_tensor * ggml_conv_transpose_1d(
4588 struct ggml_context * ctx,
4589 struct ggml_tensor * a,
4590 struct ggml_tensor * b,
4591 int s0,
4592 int p0,
4593 int d0) {
4594 GGML_ASSERT(ggml_is_matrix(b))if (!(ggml_is_matrix(b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4594, "GGML_ASSERT(%s) failed", "ggml_is_matrix(b)");
4595 GGML_ASSERT(a->ne[2] == b->ne[1])if (!(a->ne[2] == b->ne[1])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4595, "GGML_ASSERT(%s) failed", "a->ne[2] == b->ne[1]"
);
4596 GGML_ASSERT(a->ne[3] == 1)if (!(a->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4596, "GGML_ASSERT(%s) failed", "a->ne[3] == 1");
4597
4598 GGML_ASSERT(p0 == 0)if (!(p0 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4598, "GGML_ASSERT(%s) failed", "p0 == 0");
4599 GGML_ASSERT(d0 == 1)if (!(d0 == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4599, "GGML_ASSERT(%s) failed", "d0 == 1");
4600
4601 const int64_t ne[4] = {
4602 ggml_calc_conv_transpose_1d_output_size(b->ne[0], a->ne[0], s0, 0 /*p0*/, 1 /*d0*/),
4603 a->ne[1], b->ne[2], 1,
4604 };
4605 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
4606
4607 int32_t params[] = { s0, p0, d0 };
4608 ggml_set_op_params(result, params, sizeof(params));
4609
4610 result->op = GGML_OP_CONV_TRANSPOSE_1D;
4611 result->src[0] = a;
4612 result->src[1] = b;
4613
4614 return result;
4615}
4616
4617// ggml_conv_2d
4618
4619// a: [OC，IC, KH, KW]
4620// b: [N, IC, IH, IW]
4621// result: [N, OC, OH, OW]
4622struct ggml_tensor * ggml_conv_2d(
4623 struct ggml_context * ctx,
4624 struct ggml_tensor * a,
4625 struct ggml_tensor * b,
4626 int s0,
4627 int s1,
4628 int p0,
4629 int p1,
4630 int d0,
4631 int d1) {
4632 struct ggml_tensor * im2col = ggml_im2col(ctx, a, b, s0, s1, p0, p1, d0, d1, true1, a->type); // [N, OH, OW, IC * KH * KW]
4633
4634 struct ggml_tensor * result =
4635 ggml_mul_mat(ctx,
4636 ggml_reshape_2d(ctx, im2col, im2col->ne[0], im2col->ne[3] * im2col->ne[2] * im2col->ne[1]), // [N, OH, OW, IC * KH * KW] => [N*OH*OW, IC * KH * KW]
4637 ggml_reshape_2d(ctx, a, (a->ne[0] * a->ne[1] * a->ne[2]), a->ne[3])); // [OC，IC, KH, KW] => [OC, IC * KH * KW]
4638
4639 result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], im2col->ne[3], a->ne[3]); // [OC, N, OH, OW]
4640 result = ggml_cont(ctx, ggml_permute(ctx, result, 0, 1, 3, 2)); // [N, OC, OH, OW]
4641
4642
4643 return result;
4644}
4645
4646// a: [OC*IC, KD, KH, KW]
4647// b: [N*IC, ID, IH, IW]
4648// result: [N*OD, OH, OW, IC * KD * KH * KW]
4649struct ggml_tensor * ggml_im2col_3d(
4650 struct ggml_context * ctx,
4651 struct ggml_tensor * a,
4652 struct ggml_tensor * b,
4653 int64_t IC,
4654 int s0, // stride width
4655 int s1, // stride height
4656 int s2, // stride depth
4657 int p0, // padding width
4658 int p1, // padding height
4659 int p2, // padding depth
4660 int d0, // dilation width
4661 int d1, // dilation height
4662 int d2, // dilation depth
4663 enum ggml_type dst_type) {
4664 const int64_t N = b->ne[3] / IC;
4665 const int64_t ID = b->ne[2];
4666 const int64_t IH = b->ne[1];
4667 const int64_t IW = b->ne[0];
4668
4669 const int64_t OC = a->ne[3] / IC;
4670 UNUSED(OC)(void)(OC);
4671 const int64_t KD = a->ne[2];
4672 const int64_t KH = a->ne[1];
4673 const int64_t KW = a->ne[0];
4674 const int64_t OD = ggml_calc_conv_output_size(ID, KD, s2, p2, d2);
4675 const int64_t OH = ggml_calc_conv_output_size(IH, KH, s1, p1, d1);
4676 const int64_t OW = ggml_calc_conv_output_size(IW, KW, s0, p0, d0);
4677
4678 GGML_ASSERT((OD > 0) && "b too small compared to a")if (!((OD > 0) && "b too small compared to a")) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4678, "GGML_ASSERT(%s) failed", "(OD > 0) && \"b too small compared to a\""
);
4679 GGML_ASSERT((OH > 0) && "b too small compared to a")if (!((OH > 0) && "b too small compared to a")) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4679, "GGML_ASSERT(%s) failed", "(OH > 0) && \"b too small compared to a\""
);
4680 GGML_ASSERT((OW > 0) && "b too small compared to a")if (!((OW > 0) && "b too small compared to a")) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4680, "GGML_ASSERT(%s) failed", "(OW > 0) && \"b too small compared to a\""
);
4681
4682
4683 const int64_t ne[4] = {KW*KH*KD*IC, OW, OH, OD*N};
4684
4685 struct ggml_tensor * result = ggml_new_tensor(ctx, dst_type, 4, ne);
4686 int32_t params[] = { s0, s1, s2, p0, p1, p2, d0, d1, d2, (int32_t)IC};
4687 ggml_set_op_params(result, params, sizeof(params));
4688
4689 result->op = GGML_OP_IM2COL_3D;
4690 result->src[0] = a;
4691 result->src[1] = b;
4692
4693 return result;
4694}
4695
4696// a: [OC*IC, KD, KH, KW]
4697// b: [N*IC, ID, IH, IW]
4698// result: [N*OC, OD, OH, OW]
4699struct ggml_tensor * ggml_conv_3d(
4700 struct ggml_context * ctx,
4701 struct ggml_tensor * a,
4702 struct ggml_tensor * b,
4703 int64_t IC,
4704 int s0, // stride width
4705 int s1, // stride height
4706 int s2, // stride depth
4707 int p0, // padding width
4708 int p1, // padding height
4709 int p2, // padding depth
4710 int d0, // dilation width
4711 int d1, // dilation height
4712 int d2 // dilation depth
4713 ) {
4714 struct ggml_tensor * im2col = ggml_im2col_3d(ctx, a, b, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, a->type); // [N*OD, OH, OW, IC * KD * KH * KW]
4715
4716 int64_t OC = a->ne[3] / IC;
4717 int64_t N = b->ne[3] / IC;
4718 struct ggml_tensor * result =
4719 ggml_mul_mat(ctx,
4720 ggml_reshape_2d(ctx, im2col, im2col->ne[0], im2col->ne[3] * im2col->ne[2] * im2col->ne[1]), // [N*OD, OH, OW, IC * KD * KH * KW] => [N*OD*OH*OW, IC * KD * KH * KW]
4721 ggml_reshape_2d(ctx, a, (a->ne[0] * a->ne[1] * a->ne[2] * IC), OC)); // [OC*IC, KD, KH, KW] => [OC, IC * KD * KH * KW]
4722
4723 int64_t OD = im2col->ne[3] / N;
4724 result = ggml_reshape_4d(ctx, result, im2col->ne[1]*im2col->ne[2], OD, N, OC); // [OC, N*OD*OH*OW] => [OC, N, OD, OH*OW]
4725 result = ggml_cont(ctx, ggml_permute(ctx, result, 0, 1, 3, 2)); // [N, OC, OD, OH*OW]
4726 result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], OD, OC * N); // [N*OC, OD, OH, OW]
4727
4728 return result;
4729}
4730
4731// ggml_conv_2d_sk_p0
4732
4733struct ggml_tensor * ggml_conv_2d_sk_p0(
4734 struct ggml_context * ctx,
4735 struct ggml_tensor * a,
4736 struct ggml_tensor * b) {
4737 return ggml_conv_2d(ctx, a, b, a->ne[0], a->ne[1], 0, 0, 1, 1);
4738}
4739
4740// ggml_conv_2d_s1_ph
4741
4742struct ggml_tensor * ggml_conv_2d_s1_ph(
4743 struct ggml_context * ctx,
4744 struct ggml_tensor * a,
4745 struct ggml_tensor * b) {
4746 return ggml_conv_2d(ctx, a, b, 1, 1, a->ne[0] / 2, a->ne[1] / 2, 1, 1);
4747}
4748
4749// ggml_conv_2d_dw
4750
4751struct ggml_tensor * ggml_conv_2d_dw(
4752 struct ggml_context * ctx,
4753 struct ggml_tensor * a,
4754 struct ggml_tensor * b,
4755 int s0,
4756 int s1,
4757 int p0,
4758 int p1,
4759 int d0,
4760 int d1) {
4761 struct ggml_tensor * new_a = ggml_reshape_4d(ctx, a, a->ne[0], a->ne[1], 1, a->ne[2] * a->ne[3]);
4762 struct ggml_tensor * im2col = ggml_im2col(ctx, new_a,
4763 ggml_reshape_4d(ctx, b, b->ne[0], b->ne[1], 1, b->ne[2] * b->ne[3]),
4764 s0, s1, p0, p1, d0, d1, true1, GGML_TYPE_F16); // [N * IC, OH, OW, KH * KW]
4765 struct ggml_tensor * new_b = ggml_reshape_4d(ctx, im2col, im2col->ne[0], im2col->ne[2] * im2col->ne[1], b->ne[2], b->ne[3]); // [N * IC, OH, OW, KH * KW] => [N, IC, OH * OW, KH * KW]
4766
4767 new_a = ggml_reshape_4d(ctx, new_a, (new_a->ne[0] * new_a->ne[1]), new_a->ne[2], new_a->ne[3], 1); // [OC，1, KH, KW] => [1, OC, 1, KH * KW]
4768 struct ggml_tensor * result = ggml_mul_mat(ctx, new_a, new_b);
4769 result = ggml_reshape_4d(ctx, result, im2col->ne[1], im2col->ne[2], b->ne[2], b->ne[3]); // [N, OC, OH, OW]
4770
4771 return result;
4772}
4773
4774// ggml_conv_2d_dw_direct
4775
4776struct ggml_tensor * ggml_conv_2d_dw_direct(
4777 struct ggml_context * ctx,
4778 struct ggml_tensor * a,
4779 struct ggml_tensor * b,
4780 int stride0,
4781 int stride1,
4782 int pad0,
4783 int pad1,
4784 int dilation0,
4785 int dilation1) {
4786 GGML_ASSERT(a->ne[2] == 1)if (!(a->ne[2] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4786, "GGML_ASSERT(%s) failed", "a->ne[2] == 1");
4787 GGML_ASSERT(a->ne[3] == b->ne[2])if (!(a->ne[3] == b->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4787, "GGML_ASSERT(%s) failed", "a->ne[3] == b->ne[2]"
);
4788 int64_t ne[4];
4789 ne[0] = ggml_calc_conv_output_size(b->ne[0], a->ne[0], stride0, pad0, dilation0);
4790 ne[1] = ggml_calc_conv_output_size(b->ne[1], a->ne[1], stride1, pad1, dilation1);
4791 ne[2] = b->ne[2];
4792 ne[3] = b->ne[3];
4793
4794 struct ggml_tensor * result = ggml_new_tensor(ctx, b->type, 4, ne);
4795
4796 if (ggml_is_contiguous_channels(b)) {
4797 // Result will be permuted the same way as input (CWHN order)
4798 const int64_t type_size = ggml_type_size(result->type);
4799 GGML_ASSERT(ggml_blck_size(result->type) == 1)if (!(ggml_blck_size(result->type) == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4799, "GGML_ASSERT(%s) failed", "ggml_blck_size(result->type) == 1"
);
4800 result->nb[0] = result->ne[2] * type_size;
4801 result->nb[1] = result->ne[0] * result->nb[0];
4802 result->nb[2] = type_size;
4803 }
4804
4805 int32_t params[] = { stride0, stride1, pad0, pad1, dilation0, dilation1 };
4806 ggml_set_op_params(result, params, sizeof(params));
4807
4808 result->op = GGML_OP_CONV_2D_DW;
4809 result->src[0] = a;
4810 result->src[1] = b;
4811 return result;
4812}
4813
4814// ggml_conv_2d_direct
4815
4816struct ggml_tensor * ggml_conv_2d_direct(
4817 struct ggml_context * ctx,
4818 struct ggml_tensor * a, // convolution kernel [KW, KH, IC, OC]
4819 struct ggml_tensor * b, // input data [W, H, C, N]
4820 int s0, // stride dimension 0
4821 int s1, // stride dimension 1
4822 int p0, // padding dimension 0
4823 int p1, // padding dimension 1
4824 int d0, // dilation dimension 0
4825 int d1) {// dilation dimension 1
4826
4827 GGML_ASSERT(a->ne[2] == b->ne[2])if (!(a->ne[2] == b->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4827, "GGML_ASSERT(%s) failed", "a->ne[2] == b->ne[2]"
);
4828 //GGML_ASSERT(a->type == b->type);
4829
4830 int64_t ne[4];
4831 ne[0] = ggml_calc_conv_output_size(b->ne[0], a->ne[0], s0, p0, d0);
4832 ne[1] = ggml_calc_conv_output_size(b->ne[1], a->ne[1], s1, p1, d1);
4833 ne[2] = a->ne[3];
4834 ne[3] = b->ne[3];
4835
4836 struct ggml_tensor * result = ggml_new_tensor(ctx, b->type, 4, ne);
4837
4838 ggml_set_op_params_i32(result, 0, s0);
4839 ggml_set_op_params_i32(result, 1, s1);
4840 ggml_set_op_params_i32(result, 2, p0);
4841 ggml_set_op_params_i32(result, 3, p1);
4842 ggml_set_op_params_i32(result, 4, d0);
4843 ggml_set_op_params_i32(result, 5, d1);
4844
4845 result->op = GGML_OP_CONV_2D;
4846 result->src[0] = a;
4847 result->src[1] = b;
4848
4849 return result;
4850}
4851
4852// ggml_conv_3d_direct
4853
4854struct ggml_tensor * ggml_conv_3d_direct(
4855 struct ggml_context * ctx,
4856 struct ggml_tensor * a,
4857 struct ggml_tensor * b,
4858 int s0,
4859 int s1,
4860 int s2,
4861 int p0,
4862 int p1,
4863 int p2,
4864 int d0,
4865 int d1,
4866 int d2,
4867 int c,
4868 int n,
4869 int oc) {
4870
4871 GGML_ASSERT(a->ne[3] == (int64_t) c * oc)if (!(a->ne[3] == (int64_t) c * oc)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4871, "GGML_ASSERT(%s) failed", "a->ne[3] == (int64_t) c * oc"
);
4872 GGML_ASSERT(b->ne[3] == (int64_t) c * n)if (!(b->ne[3] == (int64_t) c * n)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4872, "GGML_ASSERT(%s) failed", "b->ne[3] == (int64_t) c * n"
);
4873
4874 int64_t ne[4];
4875 ne[0] = ggml_calc_conv_output_size(b->ne[0], a->ne[0], s0, p0, d0);
4876 ne[1] = ggml_calc_conv_output_size(b->ne[1], a->ne[1], s1, p1, d1);
4877 ne[2] = ggml_calc_conv_output_size(b->ne[2], a->ne[2], s2, p2, d2);
4878 ne[3] = (int64_t) oc * n;
4879
4880 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
4881
4882 ggml_set_op_params_i32(result, 0, s0);
4883 ggml_set_op_params_i32(result, 1, s1);
4884 ggml_set_op_params_i32(result, 2, s2);
4885 ggml_set_op_params_i32(result, 3, p0);
4886 ggml_set_op_params_i32(result, 4, p1);
4887 ggml_set_op_params_i32(result, 5, p2);
4888 ggml_set_op_params_i32(result, 6, d0);
4889 ggml_set_op_params_i32(result, 7, d1);
4890 ggml_set_op_params_i32(result, 8, d2);
4891 ggml_set_op_params_i32(result, 9, c);
4892 ggml_set_op_params_i32(result, 10, n);
4893 ggml_set_op_params_i32(result, 11, oc);
4894
4895 result->op = GGML_OP_CONV_3D;
4896 result->src[0] = a;
4897 result->src[1] = b;
4898
4899 return result;
4900}
4901
4902// ggml_conv_transpose_2d_p0
4903
4904static int64_t ggml_calc_conv_transpose_output_size(int64_t ins, int64_t ks, int s, int p) {
4905 return (ins - 1) * s - 2 * p + ks;
4906}
4907
4908struct ggml_tensor * ggml_conv_transpose_2d_p0(
4909 struct ggml_context * ctx,
4910 struct ggml_tensor * a,
4911 struct ggml_tensor * b,
4912 int stride) {
4913 GGML_ASSERT(a->ne[3] == b->ne[2])if (!(a->ne[3] == b->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4913, "GGML_ASSERT(%s) failed", "a->ne[3] == b->ne[2]"
);
4914
4915 const int64_t ne[4] = {
4916 ggml_calc_conv_transpose_output_size(b->ne[0], a->ne[0], stride, 0 /*p0*/),
4917 ggml_calc_conv_transpose_output_size(b->ne[1], a->ne[1], stride, 0 /*p1*/),
4918 a->ne[2], b->ne[3],
4919 };
4920
4921 struct ggml_tensor* result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
4922
4923 ggml_set_op_params_i32(result, 0, stride);
4924
4925 result->op = GGML_OP_CONV_TRANSPOSE_2D;
4926 result->src[0] = a;
4927 result->src[1] = b;
4928
4929 return result;
4930}
4931
4932// ggml_pool_*
4933
4934static int64_t ggml_calc_pool_output_size(int64_t ins, int ks, int s, float p) {
4935 return (ins + 2 * p - ks) / s + 1;
4936}
4937
4938// ggml_pool_1d
4939
4940struct ggml_tensor * ggml_pool_1d(
4941 struct ggml_context * ctx,
4942 struct ggml_tensor * a,
4943 enum ggml_op_pool op,
4944 int k0,
4945 int s0,
4946 int p0) {
4947 const int64_t ne[4] = {
4948 ggml_calc_pool_output_size(a->ne[0], k0, s0, p0),
4949 a->ne[1],
4950 a->ne[2],
4951 a->ne[3],
4952 };
4953 GGML_ASSERT(ne[0] > 0)if (!(ne[0] > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4953, "GGML_ASSERT(%s) failed", "ne[0] > 0");
4954
4955 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
4956
4957 int32_t params[] = { op, k0, s0, p0 };
4958 ggml_set_op_params(result, params, sizeof(params));
4959
4960 result->op = GGML_OP_POOL_1D;
4961 result->src[0] = a;
4962
4963 return result;
4964}
4965
4966// ggml_pool_2d
4967
4968struct ggml_tensor * ggml_pool_2d(
4969 struct ggml_context * ctx,
4970 struct ggml_tensor * a,
4971 enum ggml_op_pool op,
4972 int k0,
4973 int k1,
4974 int s0,
4975 int s1,
4976 float p0,
4977 float p1) {
4978 struct ggml_tensor * result;
4979 const int64_t ne[4] = {
4980 ggml_calc_pool_output_size(a->ne[0], k0, s0, p0),
4981 ggml_calc_pool_output_size(a->ne[1], k1, s1, p1),
4982 a->ne[2],
4983 a->ne[3],
4984 };
4985 GGML_ASSERT(ne[0] > 0)if (!(ne[0] > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4985, "GGML_ASSERT(%s) failed", "ne[0] > 0");
4986 GGML_ASSERT(ne[1] > 0)if (!(ne[1] > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 4986, "GGML_ASSERT(%s) failed", "ne[1] > 0");
4987
4988 result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
4989
4990 int32_t params[] = { op, k0, k1, s0, s1, p0, p1 };
4991 ggml_set_op_params(result, params, sizeof(params));
4992
4993 result->op = GGML_OP_POOL_2D;
4994 result->src[0] = a;
4995
4996 return result;
4997}
4998
4999struct ggml_tensor * ggml_pool_2d_back(
5000 struct ggml_context * ctx,
5001 struct ggml_tensor * a,
5002 struct ggml_tensor * af,
5003 enum ggml_op_pool op,
5004 int k0,
5005 int k1,
5006 int s0,
5007 int s1,
5008 float p0,
5009 float p1) {
5010 struct ggml_tensor * result;
5011 result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, af->ne);
5012
5013 int32_t params[] = { op, k0, k1, s0, s1, p0, p1 };
5014 ggml_set_op_params(result, params, sizeof(params));
5015
5016 result->op = GGML_OP_POOL_2D_BACK;
5017 result->src[0] = a;
5018 result->src[1] = af;
5019
5020 return result;
5021}
5022
5023// ggml_upscale / ggml_interpolate
5024
5025static struct ggml_tensor * ggml_interpolate_impl(
5026 struct ggml_context * ctx,
5027 struct ggml_tensor * a,
5028 int64_t ne0,
5029 int64_t ne1,
5030 int64_t ne2,
5031 int64_t ne3,
5032 uint32_t mode) {
5033 GGML_ASSERT((mode & 0xFF) < GGML_SCALE_MODE_COUNT)if (!((mode & 0xFF) < GGML_SCALE_MODE_COUNT)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5033, "GGML_ASSERT(%s) failed", "(mode & 0xFF) < GGML_SCALE_MODE_COUNT"
);
5034 // TODO: implement antialias for modes other than bilinear
5035 GGML_ASSERT(!(mode & GGML_SCALE_FLAG_ANTIALIAS) || (mode & 0xFF) == GGML_SCALE_MODE_BILINEAR)if (!(!(mode & GGML_SCALE_FLAG_ANTIALIAS) || (mode & 0xFF
) == GGML_SCALE_MODE_BILINEAR)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5035, "GGML_ASSERT(%s) failed", "!(mode & GGML_SCALE_FLAG_ANTIALIAS) || (mode & 0xFF) == GGML_SCALE_MODE_BILINEAR"
);
5036 GGML_ASSERT(a->type == GGML_TYPE_F32)if (!(a->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5036, "GGML_ASSERT(%s) failed", "a->type == GGML_TYPE_F32"
);
5037
5038 struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type, ne0, ne1, ne2, ne3);
5039
5040 ggml_set_op_params_i32(result, 0, (int32_t)mode);
5041
5042 result->op = GGML_OP_UPSCALE;
5043 result->src[0] = a;
5044
5045 return result;
5046}
5047
5048struct ggml_tensor * ggml_upscale(
5049 struct ggml_context * ctx,
5050 struct ggml_tensor * a,
5051 int scale_factor,
5052 enum ggml_scale_mode mode) {
5053 GGML_ASSERT(scale_factor > 1)if (!(scale_factor > 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5053, "GGML_ASSERT(%s) failed", "scale_factor > 1");
5054 return ggml_interpolate_impl(ctx, a, a->ne[0] * scale_factor, a->ne[1] * scale_factor, a->ne[2], a->ne[3], mode);
5055}
5056
5057struct ggml_tensor * ggml_upscale_ext(
5058 struct ggml_context * ctx,
5059 struct ggml_tensor * a,
5060 int ne0,
5061 int ne1,
5062 int ne2,
5063 int ne3,
5064 enum ggml_scale_mode mode) {
5065 return ggml_interpolate_impl(ctx, a, ne0, ne1, ne2, ne3, mode);
5066}
5067
5068struct ggml_tensor * ggml_interpolate(
5069 struct ggml_context * ctx,
5070 struct ggml_tensor * a,
5071 int64_t ne0,
5072 int64_t ne1,
5073 int64_t ne2,
5074 int64_t ne3,
5075 uint32_t mode) {
5076 return ggml_interpolate_impl(ctx, a, ne0, ne1, ne2, ne3, mode);
5077}
5078
5079// ggml_pad
5080
5081struct ggml_tensor * ggml_pad(
5082 struct ggml_context * ctx,
5083 struct ggml_tensor * a,
5084 int p0,
5085 int p1,
5086 int p2,
5087 int p3) {
5088 return ggml_pad_ext(ctx, a, 0, p0, 0, p1, 0, p2, 0, p3);
5089}
5090
5091// ggml_pad_circular
5092
5093struct ggml_tensor * ggml_pad_circular(
5094 struct ggml_context * ctx,
5095 struct ggml_tensor * a,
5096 int p0,
5097 int p1,
5098 int p2,
5099 int p3) {
5100 return ggml_pad_ext_circular(ctx, a, 0, p0, 0, p1, 0, p2, 0, p3);
5101}
5102
5103struct ggml_tensor * ggml_pad_ext(
5104 struct ggml_context * ctx,
5105 struct ggml_tensor * a,
5106 int lp0,
5107 int rp0,
5108 int lp1,
5109 int rp1,
5110 int lp2,
5111 int rp2,
5112 int lp3,
5113 int rp3
5114 ) {
5115 struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type,
5116 a->ne[0] + lp0 + rp0,
5117 a->ne[1] + lp1 + rp1,
5118 a->ne[2] + lp2 + rp2,
5119 a->ne[3] + lp3 + rp3);
5120
5121 ggml_set_op_params_i32(result, 0, lp0);
5122 ggml_set_op_params_i32(result, 1, rp0);
5123 ggml_set_op_params_i32(result, 2, lp1);
5124 ggml_set_op_params_i32(result, 3, rp1);
5125 ggml_set_op_params_i32(result, 4, lp2);
5126 ggml_set_op_params_i32(result, 5, rp2);
5127 ggml_set_op_params_i32(result, 6, lp3);
5128 ggml_set_op_params_i32(result, 7, rp3);
5129 ggml_set_op_params_i32(result, 8, 0); // not circular by default
5130
5131
5132 result->op = GGML_OP_PAD;
5133 result->src[0] = a;
5134
5135 return result;
5136}
5137
5138// ggml_pad_ext_circular
5139
5140struct ggml_tensor * ggml_pad_ext_circular(
5141 struct ggml_context * ctx,
5142 struct ggml_tensor * a,
5143 int lp0,
5144 int rp0,
5145 int lp1,
5146 int rp1,
5147 int lp2,
5148 int rp2,
5149 int lp3,
5150 int rp3
5151 ) {
5152 struct ggml_tensor * result = ggml_pad_ext(ctx, a, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3);
5153 ggml_set_op_params_i32(result, 8, 1); // circular
5154 return result;
5155}
5156
5157// ggml_pad_reflect_1d
5158
5159struct ggml_tensor * ggml_pad_reflect_1d(
5160 struct ggml_context * ctx,
5161 struct ggml_tensor * a,
5162 int p0,
5163 int p1) {
5164 GGML_ASSERT(p0 >= 0)if (!(p0 >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5164, "GGML_ASSERT(%s) failed", "p0 >= 0");
5165 GGML_ASSERT(p1 >= 0)if (!(p1 >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5165, "GGML_ASSERT(%s) failed", "p1 >= 0");
5166
5167 GGML_ASSERT(p0 < a->ne[0])if (!(p0 < a->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5167, "GGML_ASSERT(%s) failed", "p0 < a->ne[0]"); // padding length on each size must be less than the
5168 GGML_ASSERT(p1 < a->ne[0])if (!(p1 < a->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5168, "GGML_ASSERT(%s) failed", "p1 < a->ne[0]"); // existing length of the dimension being padded
5169
5170 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5170, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
5171 GGML_ASSERT(a->type == GGML_TYPE_F32)if (!(a->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5171, "GGML_ASSERT(%s) failed", "a->type == GGML_TYPE_F32"
);
5172
5173 struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type,
5174 a->ne[0] + p0 + p1,
5175 a->ne[1],
5176 a->ne[2],
5177 a->ne[3]);
5178
5179 int32_t params[] = { p0, p1 };
5180 ggml_set_op_params(result, params, sizeof(params));
5181
5182 result->op = GGML_OP_PAD_REFLECT_1D;
5183 result->src[0] = a;
5184
5185 return result;
5186}
5187
5188// ggml_roll
5189
5190struct ggml_tensor * ggml_roll(
5191 struct ggml_context * ctx,
5192 struct ggml_tensor * a,
5193 int shift0,
5194 int shift1,
5195 int shift2,
5196 int shift3) {
5197 GGML_ASSERT(a->nb[0] == ggml_type_size(a->type))if (!(a->nb[0] == ggml_type_size(a->type))) ggml_abort(
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5197, "GGML_ASSERT(%s) failed", "a->nb[0] == ggml_type_size(a->type)"
);
5198 GGML_ASSERT(abs(shift0) < a->ne[0])if (!(abs(shift0) < a->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5198, "GGML_ASSERT(%s) failed", "abs(shift0) < a->ne[0]"
);
5199 GGML_ASSERT(abs(shift1) < a->ne[1])if (!(abs(shift1) < a->ne[1])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5199, "GGML_ASSERT(%s) failed", "abs(shift1) < a->ne[1]"
);
5200 GGML_ASSERT(abs(shift2) < a->ne[2])if (!(abs(shift2) < a->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5200, "GGML_ASSERT(%s) failed", "abs(shift2) < a->ne[2]"
);
5201 GGML_ASSERT(abs(shift3) < a->ne[3])if (!(abs(shift3) < a->ne[3])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5201, "GGML_ASSERT(%s) failed", "abs(shift3) < a->ne[3]"
);
5202
5203 struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
5204
5205 ggml_set_op_params_i32(result, 0, shift0);
5206 ggml_set_op_params_i32(result, 1, shift1);
5207 ggml_set_op_params_i32(result, 2, shift2);
5208 ggml_set_op_params_i32(result, 3, shift3);
5209
5210 result->op = GGML_OP_ROLL;
5211 result->src[0] = a;
5212
5213 return result;
5214}
5215
5216// ggml_timestep_embedding
5217
5218struct ggml_tensor * ggml_timestep_embedding(
5219 struct ggml_context * ctx,
5220 struct ggml_tensor * timesteps,
5221 int dim,
5222 int max_period) {
5223
5224 struct ggml_tensor * result = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, dim, timesteps->ne[0]);
5225
5226 ggml_set_op_params_i32(result, 0, dim);
5227 ggml_set_op_params_i32(result, 1, max_period);
5228
5229 result->op = GGML_OP_TIMESTEP_EMBEDDING;
5230 result->src[0] = timesteps;
5231
5232 return result;
5233}
5234
5235// ggml_tri
5236
5237struct ggml_tensor * ggml_tri(
5238 struct ggml_context * ctx,
5239 struct ggml_tensor * a,
5240 enum ggml_tri_type type) {
5241 GGML_ASSERT(a->type == GGML_TYPE_F32)if (!(a->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5241, "GGML_ASSERT(%s) failed", "a->type == GGML_TYPE_F32"
);
5242
5243 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5243, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
5244 GGML_ASSERT(a->ne[0] == a->ne[1])if (!(a->ne[0] == a->ne[1])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5244, "GGML_ASSERT(%s) failed", "a->ne[0] == a->ne[1]"
);
5245
5246 struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
5247
5248 ggml_set_op_params_i32(result, 0, type);
5249
5250 result->op = GGML_OP_TRI;
5251 result->src[0] = a;
5252
5253 return result;
5254}
5255
5256// ggml_fill
5257
5258static struct ggml_tensor * ggml_fill_impl(
5259 struct ggml_context * ctx,
5260 struct ggml_tensor * a,
5261 float c,
5262 bool_Bool inplace) {
5263 GGML_ASSERT(a->type == GGML_TYPE_F32 || a->type == GGML_TYPE_F16)if (!(a->type == GGML_TYPE_F32 || a->type == GGML_TYPE_F16
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5263, "GGML_ASSERT(%s) failed", "a->type == GGML_TYPE_F32 || a->type == GGML_TYPE_F16"
);
5264 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5264, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
5265
5266 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
5267
5268 ggml_set_op_params_f32(result, 0, c);
5269
5270 result->op = GGML_OP_FILL;
5271 result->src[0] = a;
5272
5273 return result;
5274}
5275
5276struct ggml_tensor * ggml_fill(
5277 struct ggml_context * ctx,
5278 struct ggml_tensor * a,
5279 float c) {
5280 return ggml_fill_impl(ctx, a, c, false0);
5281}
5282
5283struct ggml_tensor * ggml_fill_inplace(
5284 struct ggml_context * ctx,
5285 struct ggml_tensor * a,
5286 float c) {
5287 return ggml_fill_impl(ctx, a, c, true1);
5288}
5289
5290// ggml_argsort
5291
5292struct ggml_tensor * ggml_argsort(
5293 struct ggml_context * ctx,
5294 struct ggml_tensor * a,
5295 enum ggml_sort_order order) {
5296 GGML_ASSERT(a->ne[0] <= INT32_MAX)if (!(a->ne[0] <= (2147483647))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5296, "GGML_ASSERT(%s) failed", "a->ne[0] <= INT32_MAX"
);
5297
5298 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_I32, GGML_MAX_DIMS4, a->ne);
5299
5300 ggml_set_op_params_i32(result, 0, (int32_t) order);
5301
5302 result->op = GGML_OP_ARGSORT;
5303 result->src[0] = a;
5304
5305 return result;
5306}
5307
5308// ggml_argsort_top_k
5309
5310struct ggml_tensor * ggml_argsort_top_k(
5311 struct ggml_context * ctx,
5312 struct ggml_tensor * a,
5313 int k) {
5314 GGML_ASSERT(a->ne[0] >= k)if (!(a->ne[0] >= k)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5314, "GGML_ASSERT(%s) failed", "a->ne[0] >= k");
5315
5316 struct ggml_tensor * result = ggml_argsort(ctx, a, GGML_SORT_ORDER_DESC);
5317
5318 result = ggml_view_4d(ctx, result,
5319 k, result->ne[1], result->ne[2], result->ne[3],
5320 result->nb[1], result->nb[2], result->nb[3],
5321 0);
5322
5323 return result;
5324}
5325
5326// ggml_top_k
5327
5328struct ggml_tensor * ggml_top_k(
5329 struct ggml_context * ctx,
5330 struct ggml_tensor * a,
5331 int k) {
5332 GGML_ASSERT(a->ne[0] >= k)if (!(a->ne[0] >= k)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5332, "GGML_ASSERT(%s) failed", "a->ne[0] >= k");
5333
5334 struct ggml_tensor * result = ggml_new_tensor_4d(ctx, GGML_TYPE_I32, k, a->ne[1], a->ne[2], a->ne[3]);
5335
5336 result->op = GGML_OP_TOP_K;
5337 result->src[0] = a;
5338
5339 return result;
5340}
5341
5342// ggml_arange
5343
5344struct ggml_tensor * ggml_arange(
5345 struct ggml_context * ctx,
5346 float start,
5347 float stop,
5348 float step) {
5349 GGML_ASSERT(stop > start)if (!(stop > start)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5349, "GGML_ASSERT(%s) failed", "stop > start");
5350
5351 const int64_t steps = (int64_t) ceilf((stop - start) / step);
5352
5353 struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, steps);
5354
5355 ggml_set_op_params_f32(result, 0, start);
5356 ggml_set_op_params_f32(result, 1, stop);
5357 ggml_set_op_params_f32(result, 2, step);
5358
5359 result->op = GGML_OP_ARANGE;
5360
5361 return result;
5362}
5363
5364// ggml_flash_attn_ext
5365
5366struct ggml_tensor * ggml_flash_attn_ext(
5367 struct ggml_context * ctx,
5368 struct ggml_tensor * q,
5369 struct ggml_tensor * k,
5370 struct ggml_tensor * v,
5371 struct ggml_tensor * mask,
5372 float scale,
5373 float max_bias,
5374 float logit_softcap) {
5375 GGML_ASSERT(ggml_can_mul_mat(k, q))if (!(ggml_can_mul_mat(k, q))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5375, "GGML_ASSERT(%s) failed", "ggml_can_mul_mat(k, q)");
5376 // TODO: check if vT can be multiplied by (k*qT)
5377
5378 GGML_ASSERT(q->ne[3] == k->ne[3])if (!(q->ne[3] == k->ne[3])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5378, "GGML_ASSERT(%s) failed", "q->ne[3] == k->ne[3]"
);
5379 GGML_ASSERT(q->ne[3] == v->ne[3])if (!(q->ne[3] == v->ne[3])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5379, "GGML_ASSERT(%s) failed", "q->ne[3] == v->ne[3]"
);
5380
5381 if (mask) {
5382 GGML_ASSERT(mask->type == GGML_TYPE_F16)if (!(mask->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5382, "GGML_ASSERT(%s) failed", "mask->type == GGML_TYPE_F16"
);
5383 GGML_ASSERT(ggml_is_contiguous(mask))if (!(ggml_is_contiguous(mask))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5383, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(mask)");
5384 //GGML_ASSERT(ggml_can_repeat_rows(mask, qk));
5385
5386 GGML_ASSERT(q->ne[2] % mask->ne[2] == 0)if (!(q->ne[2] % mask->ne[2] == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5386, "GGML_ASSERT(%s) failed", "q->ne[2] % mask->ne[2] == 0"
);
5387 GGML_ASSERT(q->ne[3] % mask->ne[3] == 0)if (!(q->ne[3] % mask->ne[3] == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5387, "GGML_ASSERT(%s) failed", "q->ne[3] % mask->ne[3] == 0"
);
5388 }
5389
5390 if (max_bias > 0.0f) {
5391 GGML_ASSERT(mask)if (!(mask)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5391, "GGML_ASSERT(%s) failed", "mask");
5392 }
5393
5394 // permute(0, 2, 1, 3)
5395 int64_t ne[4] = { v->ne[0], q->ne[2], q->ne[1], q->ne[3] };
5396 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
5397
5398 float params[] = { scale, max_bias, logit_softcap };
5399 ggml_set_op_params(result, params, sizeof(params));
5400
5401 result->op = GGML_OP_FLASH_ATTN_EXT;
5402 result->src[0] = q;
5403 result->src[1] = k;
5404 result->src[2] = v;
5405 result->src[3] = mask;
5406
5407 return result;
5408}
5409
5410void ggml_flash_attn_ext_set_prec(
5411 struct ggml_tensor * a,
5412 enum ggml_prec prec) {
5413 GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT)if (!(a->op == GGML_OP_FLASH_ATTN_EXT)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5413, "GGML_ASSERT(%s) failed", "a->op == GGML_OP_FLASH_ATTN_EXT"
);
5414
5415 const int32_t prec_i32 = (int32_t) prec;
5416
5417 ggml_set_op_params_i32(a, 3, prec_i32); // scale is on first pos, max_bias on second
5418}
5419
5420enum ggml_prec ggml_flash_attn_ext_get_prec(
5421 const struct ggml_tensor * a) {
5422 GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT)if (!(a->op == GGML_OP_FLASH_ATTN_EXT)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5422, "GGML_ASSERT(%s) failed", "a->op == GGML_OP_FLASH_ATTN_EXT"
);
5423
5424 const int32_t prec_i32 = ggml_get_op_params_i32(a, 3);
5425
5426 return (enum ggml_prec) prec_i32;
5427}
5428
5429void ggml_flash_attn_ext_add_sinks(
5430 struct ggml_tensor * a,
5431 struct ggml_tensor * sinks) {
5432 if (!sinks) {
5433 a->src[4] = NULL((void*)0);
5434 return;
5435 }
5436
5437 GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT)if (!(a->op == GGML_OP_FLASH_ATTN_EXT)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5437, "GGML_ASSERT(%s) failed", "a->op == GGML_OP_FLASH_ATTN_EXT"
);
5438 GGML_ASSERT(a->src[4] == NULL)if (!(a->src[4] == ((void*)0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5438, "GGML_ASSERT(%s) failed", "a->src[4] == NULL");
5439 GGML_ASSERT(a->src[0]->ne[2] == sinks->ne[0])if (!(a->src[0]->ne[2] == sinks->ne[0])) ggml_abort(
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5439, "GGML_ASSERT(%s) failed", "a->src[0]->ne[2] == sinks->ne[0]"
);
5440 GGML_ASSERT(sinks->type == GGML_TYPE_F32)if (!(sinks->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5440, "GGML_ASSERT(%s) failed", "sinks->type == GGML_TYPE_F32"
);
5441
5442 a->src[4] = sinks;
5443}
5444
5445// ggml_flash_attn_back
5446
5447struct ggml_tensor * ggml_flash_attn_back(
5448 struct ggml_context * ctx,
5449 struct ggml_tensor * q,
5450 struct ggml_tensor * k,
5451 struct ggml_tensor * v,
5452 struct ggml_tensor * d,
5453 bool_Bool masked) {
5454 GGML_ABORT("TODO: adapt to ggml_flash_attn_ext() changes")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5454, "TODO: adapt to ggml_flash_attn_ext() changes");
5455
5456 GGML_ASSERT(ggml_can_mul_mat(k, q))if (!(ggml_can_mul_mat(k, q))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5456, "GGML_ASSERT(%s) failed", "ggml_can_mul_mat(k, q)");
5457 // TODO: check if vT can be multiplied by (k*qT)
5458
5459 // d shape [D,N,ne2,ne3]
5460 // q shape [D,N,ne2,ne3]
5461 // k shape [D,M,kvne2,ne3]
5462 // v shape [M,D,kvne2,ne3]
5463
5464 const int64_t D = q->ne[0];
5465 const int64_t N = q->ne[1];
5466 const int64_t M = k->ne[1];
5467 const int64_t ne2 = q->ne[2];
5468 const int64_t ne3 = q->ne[3];
5469 const int64_t kvne2 = k->ne[2];
5470
5471 GGML_ASSERT(k->ne[0] == D)if (!(k->ne[0] == D)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5471, "GGML_ASSERT(%s) failed", "k->ne[0] == D");
5472 GGML_ASSERT(v->ne[0] == M)if (!(v->ne[0] == M)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5472, "GGML_ASSERT(%s) failed", "v->ne[0] == M");
5473 GGML_ASSERT(v->ne[1] == D)if (!(v->ne[1] == D)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5473, "GGML_ASSERT(%s) failed", "v->ne[1] == D");
5474 GGML_ASSERT(d->ne[0] == D)if (!(d->ne[0] == D)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5474, "GGML_ASSERT(%s) failed", "d->ne[0] == D");
5475 GGML_ASSERT(d->ne[1] == N)if (!(d->ne[1] == N)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5475, "GGML_ASSERT(%s) failed", "d->ne[1] == N");
5476 GGML_ASSERT(k->ne[2] == kvne2)if (!(k->ne[2] == kvne2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5476, "GGML_ASSERT(%s) failed", "k->ne[2] == kvne2");
5477 GGML_ASSERT(k->ne[3] == ne3)if (!(k->ne[3] == ne3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5477, "GGML_ASSERT(%s) failed", "k->ne[3] == ne3");
5478 GGML_ASSERT(v->ne[2] == kvne2)if (!(v->ne[2] == kvne2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5478, "GGML_ASSERT(%s) failed", "v->ne[2] == kvne2");
5479 GGML_ASSERT(v->ne[3] == ne3)if (!(v->ne[3] == ne3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5479, "GGML_ASSERT(%s) failed", "v->ne[3] == ne3");
5480 GGML_ASSERT(d->ne[2] == ne2)if (!(d->ne[2] == ne2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5480, "GGML_ASSERT(%s) failed", "d->ne[2] == ne2");
5481 GGML_ASSERT(d->ne[3] == ne3)if (!(d->ne[3] == ne3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5481, "GGML_ASSERT(%s) failed", "d->ne[3] == ne3");
5482
5483 GGML_ASSERT(ne2 % kvne2 == 0)if (!(ne2 % kvne2 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5483, "GGML_ASSERT(%s) failed", "ne2 % kvne2 == 0");
5484
5485 // store gradients of q, k and v as continuous tensors concatenated in result.
5486 // note: v and gradv are actually transposed, i.e. v->ne[0] != D.
5487 const int64_t elem_q = ggml_nelements(q);
5488 const int64_t elem_k = ggml_nelements(k);
5489 const int64_t elem_v = ggml_nelements(v);
5490
5491 enum ggml_type result_type = GGML_TYPE_F32;
5492 GGML_ASSERT(ggml_blck_size(result_type) == 1)if (!(ggml_blck_size(result_type) == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5492, "GGML_ASSERT(%s) failed", "ggml_blck_size(result_type) == 1"
);
5493 const size_t tsize = ggml_type_size(result_type);
5494
5495 const size_t offs_q = 0;
5496 const size_t offs_k = offs_q + GGML_PAD(elem_q * tsize, GGML_MEM_ALIGN)(((elem_q * tsize) + (16) - 1) & ~((16) - 1));
5497 const size_t offs_v = offs_k + GGML_PAD(elem_k * tsize, GGML_MEM_ALIGN)(((elem_k * tsize) + (16) - 1) & ~((16) - 1));
5498 const size_t end = offs_v + GGML_PAD(elem_v * tsize, GGML_MEM_ALIGN)(((elem_v * tsize) + (16) - 1) & ~((16) - 1));
5499
5500 const size_t nelements = (end + tsize - 1)/tsize;
5501
5502 struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, nelements);
5503
5504 int32_t masked_i = masked ? 1 : 0;
5505 ggml_set_op_params(result, &masked_i, sizeof(masked_i));
5506
5507 result->op = GGML_OP_FLASH_ATTN_BACK;
5508 result->src[0] = q;
5509 result->src[1] = k;
5510 result->src[2] = v;
5511 result->src[3] = d;
5512
5513 return result;
5514}
5515
5516// ggml_ssm_conv
5517
5518struct ggml_tensor * ggml_ssm_conv(
5519 struct ggml_context * ctx,
5520 struct ggml_tensor * sx,
5521 struct ggml_tensor * c) {
5522 GGML_ASSERT(ggml_is_3d(sx))if (!(ggml_is_3d(sx))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5522, "GGML_ASSERT(%s) failed", "ggml_is_3d(sx)");
5523 GGML_ASSERT(ggml_is_matrix(c))if (!(ggml_is_matrix(c))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5523, "GGML_ASSERT(%s) failed", "ggml_is_matrix(c)");
5524
5525 const int64_t d_conv = c->ne[0];
5526 const int64_t d_inner = c->ne[1];
5527 const int64_t n_t = sx->ne[0] - d_conv + 1; // tokens per sequence
5528 const int64_t n_s = sx->ne[2];
5529
5530 // TODO: maybe support other strides than 1?
5531 GGML_ASSERT(sx->ne[0] == d_conv - 1 + n_t)if (!(sx->ne[0] == d_conv - 1 + n_t)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5531, "GGML_ASSERT(%s) failed", "sx->ne[0] == d_conv - 1 + n_t"
);
5532 GGML_ASSERT(sx->ne[1] == d_inner)if (!(sx->ne[1] == d_inner)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5532, "GGML_ASSERT(%s) failed", "sx->ne[1] == d_inner");
5533 GGML_ASSERT(n_t >= 0)if (!(n_t >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5533, "GGML_ASSERT(%s) failed", "n_t >= 0");
5534
5535 struct ggml_tensor * result = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, d_inner, n_t, n_s);
5536
5537 result->op = GGML_OP_SSM_CONV;
5538 result->src[0] = sx;
5539 result->src[1] = c;
5540
5541 return result;
5542}
5543
5544// ggml_ssm_scan
5545
5546struct ggml_tensor * ggml_ssm_scan(
5547 struct ggml_context * ctx,
5548 struct ggml_tensor * s,
5549 struct ggml_tensor * x,
5550 struct ggml_tensor * dt,
5551 struct ggml_tensor * A,
5552 struct ggml_tensor * B,
5553 struct ggml_tensor * C,
5554 struct ggml_tensor * ids) {
5555 GGML_ASSERT(ggml_is_contiguous(s))if (!(ggml_is_contiguous(s))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5555, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(s)");
5556 GGML_ASSERT(ggml_is_contiguous(dt))if (!(ggml_is_contiguous(dt))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5556, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(dt)");
5557 GGML_ASSERT(ggml_is_contiguous(A))if (!(ggml_is_contiguous(A))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5557, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(A)");
5558 GGML_ASSERT(x->nb[0] == ggml_type_size(x->type))if (!(x->nb[0] == ggml_type_size(x->type))) ggml_abort(
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5558, "GGML_ASSERT(%s) failed", "x->nb[0] == ggml_type_size(x->type)"
);
5559 GGML_ASSERT(B->nb[0] == ggml_type_size(B->type))if (!(B->nb[0] == ggml_type_size(B->type))) ggml_abort(
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5559, "GGML_ASSERT(%s) failed", "B->nb[0] == ggml_type_size(B->type)"
);
5560 GGML_ASSERT(C->nb[0] == ggml_type_size(C->type))if (!(C->nb[0] == ggml_type_size(C->type))) ggml_abort(
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5560, "GGML_ASSERT(%s) failed", "C->nb[0] == ggml_type_size(C->type)"
);
5561 GGML_ASSERT(x->nb[1] == x->ne[0]*x->nb[0])if (!(x->nb[1] == x->ne[0]*x->nb[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5561, "GGML_ASSERT(%s) failed", "x->nb[1] == x->ne[0]*x->nb[0]"
);
5562 GGML_ASSERT(B->nb[1] == B->ne[0]*B->nb[0])if (!(B->nb[1] == B->ne[0]*B->nb[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5562, "GGML_ASSERT(%s) failed", "B->nb[1] == B->ne[0]*B->nb[0]"
);
5563 GGML_ASSERT(C->nb[1] == C->ne[0]*C->nb[0])if (!(C->nb[1] == C->ne[0]*C->nb[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5563, "GGML_ASSERT(%s) failed", "C->nb[1] == C->ne[0]*C->nb[0]"
);
5564 GGML_ASSERT(ggml_are_same_shape(B, C))if (!(ggml_are_same_shape(B, C))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5564, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(B, C)"
);
5565 GGML_ASSERT(ids->type == GGML_TYPE_I32)if (!(ids->type == GGML_TYPE_I32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5565, "GGML_ASSERT(%s) failed", "ids->type == GGML_TYPE_I32"
);
5566
5567 {
5568 const int64_t d_state = s->ne[0];
5569 const int64_t head_dim = x->ne[0];
5570 const int64_t n_head = x->ne[1];
5571 const int64_t n_seq_tokens = x->ne[2];
5572 const int64_t n_seqs = x->ne[3];
5573
5574 GGML_ASSERT(dt->ne[0] == n_head)if (!(dt->ne[0] == n_head)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5574, "GGML_ASSERT(%s) failed", "dt->ne[0] == n_head");
5575 GGML_ASSERT(dt->ne[1] == n_seq_tokens)if (!(dt->ne[1] == n_seq_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5575, "GGML_ASSERT(%s) failed", "dt->ne[1] == n_seq_tokens"
);
5576 GGML_ASSERT(dt->ne[2] == n_seqs)if (!(dt->ne[2] == n_seqs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5576, "GGML_ASSERT(%s) failed", "dt->ne[2] == n_seqs");
5577 GGML_ASSERT(ggml_is_3d(dt))if (!(ggml_is_3d(dt))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5577, "GGML_ASSERT(%s) failed", "ggml_is_3d(dt)");
5578 GGML_ASSERT(s->ne[1] == head_dim)if (!(s->ne[1] == head_dim)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5578, "GGML_ASSERT(%s) failed", "s->ne[1] == head_dim");
5579 GGML_ASSERT(s->ne[2] == n_head)if (!(s->ne[2] == n_head)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5579, "GGML_ASSERT(%s) failed", "s->ne[2] == n_head");
5580 GGML_ASSERT(B->ne[0] == d_state)if (!(B->ne[0] == d_state)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5580, "GGML_ASSERT(%s) failed", "B->ne[0] == d_state");
5581 GGML_ASSERT(B->ne[2] == n_seq_tokens)if (!(B->ne[2] == n_seq_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5581, "GGML_ASSERT(%s) failed", "B->ne[2] == n_seq_tokens"
);
5582 GGML_ASSERT(B->ne[3] == n_seqs)if (!(B->ne[3] == n_seqs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5582, "GGML_ASSERT(%s) failed", "B->ne[3] == n_seqs");
5583 GGML_ASSERT(ids->ne[0] == n_seqs)if (!(ids->ne[0] == n_seqs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5583, "GGML_ASSERT(%s) failed", "ids->ne[0] == n_seqs");
5584 GGML_ASSERT(ggml_is_vector(ids))if (!(ggml_is_vector(ids))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5584, "GGML_ASSERT(%s) failed", "ggml_is_vector(ids)");
5585 GGML_ASSERT(A->ne[1] == n_head)if (!(A->ne[1] == n_head)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5585, "GGML_ASSERT(%s) failed", "A->ne[1] == n_head");
5586 GGML_ASSERT(ggml_is_matrix(A))if (!(ggml_is_matrix(A))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5586, "GGML_ASSERT(%s) failed", "ggml_is_matrix(A)");
5587
5588 if (A->ne[0] != 1) {
5589 // Mamba-1 has more granular decay factors
5590 GGML_ASSERT(A->ne[0] == d_state)if (!(A->ne[0] == d_state)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5590, "GGML_ASSERT(%s) failed", "A->ne[0] == d_state");
5591 }
5592 }
5593
5594 // concatenated y + ssm_states
5595 struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ggml_nelements(x) + s->ne[0]*s->ne[1]*s->ne[2]*ids->ne[0]);
5596
5597 result->op = GGML_OP_SSM_SCAN;
5598 result->src[0] = s;
5599 result->src[1] = x;
5600 result->src[2] = dt;
5601 result->src[3] = A;
5602 result->src[4] = B;
5603 result->src[5] = C;
5604 result->src[6] = ids;
5605
5606 return result;
5607}
5608
5609// ggml_win_part
5610
5611struct ggml_tensor * ggml_win_part(
5612 struct ggml_context * ctx,
5613 struct ggml_tensor * a,
5614 int w) {
5615 GGML_ASSERT(a->ne[3] == 1)if (!(a->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5615, "GGML_ASSERT(%s) failed", "a->ne[3] == 1");
5616 GGML_ASSERT(a->type == GGML_TYPE_F32)if (!(a->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5616, "GGML_ASSERT(%s) failed", "a->type == GGML_TYPE_F32"
);
5617
5618 // padding
5619 const int px = (w - a->ne[1]%w)%w;
5620 const int py = (w - a->ne[2]%w)%w;
5621
5622 const int npx = (px + a->ne[1])/w;
5623 const int npy = (py + a->ne[2])/w;
5624 const int np = npx*npy;
5625
5626 const int64_t ne[4] = { a->ne[0], w, w, np, };
5627 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
5628
5629 int32_t params[] = { npx, npy, w };
5630 ggml_set_op_params(result, params, sizeof(params));
5631
5632 result->op = GGML_OP_WIN_PART;
5633 result->src[0] = a;
5634
5635 return result;
5636}
5637
5638// ggml_win_unpart
5639
5640struct ggml_tensor * ggml_win_unpart(
5641 struct ggml_context * ctx,
5642 struct ggml_tensor * a,
5643 int w0,
5644 int h0,
5645 int w) {
5646 GGML_ASSERT(a->type == GGML_TYPE_F32)if (!(a->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5646, "GGML_ASSERT(%s) failed", "a->type == GGML_TYPE_F32"
);
5647
5648 const int64_t ne[4] = { a->ne[0], w0, h0, 1, };
5649 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 3, ne);
5650
5651 int32_t params[] = { w };
5652 ggml_set_op_params(result, params, sizeof(params));
5653
5654 result->op = GGML_OP_WIN_UNPART;
5655 result->src[0] = a;
5656
5657 return result;
5658}
5659
5660// ggml_get_rel_pos
5661
5662struct ggml_tensor * ggml_get_rel_pos(
5663 struct ggml_context * ctx,
5664 struct ggml_tensor * a,
5665 int qh,
5666 int kh) {
5667 GGML_ASSERT(qh == kh)if (!(qh == kh)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5667, "GGML_ASSERT(%s) failed", "qh == kh");
5668 GGML_ASSERT(2*MAX(qh, kh) - 1 == a->ne[1])if (!(2*((qh) > (kh) ? (qh) : (kh)) - 1 == a->ne[1])) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5668, "GGML_ASSERT(%s) failed", "2*MAX(qh, kh) - 1 == a->ne[1]"
);
5669
5670 const int64_t ne[4] = { a->ne[0], kh, qh, 1, };
5671 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F16, 3, ne);
5672
5673 result->op = GGML_OP_GET_REL_POS;
5674 result->src[0] = a;
5675
5676 return result;
5677}
5678
5679// ggml_add_rel_pos
5680
5681static struct ggml_tensor * ggml_add_rel_pos_impl(
5682 struct ggml_context * ctx,
5683 struct ggml_tensor * a,
5684 struct ggml_tensor * pw,
5685 struct ggml_tensor * ph,
5686 bool_Bool inplace) {
5687 GGML_ASSERT(ggml_are_same_shape(pw, ph))if (!(ggml_are_same_shape(pw, ph))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5687, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(pw, ph)"
);
5688 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5688, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
5689 GGML_ASSERT(ggml_is_contiguous(pw))if (!(ggml_is_contiguous(pw))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5689, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(pw)");
5690 GGML_ASSERT(ggml_is_contiguous(ph))if (!(ggml_is_contiguous(ph))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5690, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(ph)");
5691 GGML_ASSERT(ph->type == GGML_TYPE_F32)if (!(ph->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5691, "GGML_ASSERT(%s) failed", "ph->type == GGML_TYPE_F32"
);
5692 GGML_ASSERT(pw->type == GGML_TYPE_F32)if (!(pw->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5692, "GGML_ASSERT(%s) failed", "pw->type == GGML_TYPE_F32"
);
5693 GGML_ASSERT(pw->ne[3] == a->ne[2])if (!(pw->ne[3] == a->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5693, "GGML_ASSERT(%s) failed", "pw->ne[3] == a->ne[2]"
);
5694 GGML_ASSERT(pw->ne[0]*pw->ne[0] == a->ne[0])if (!(pw->ne[0]*pw->ne[0] == a->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5694, "GGML_ASSERT(%s) failed", "pw->ne[0]*pw->ne[0] == a->ne[0]"
);
5695 GGML_ASSERT(pw->ne[1]*pw->ne[2] == a->ne[1])if (!(pw->ne[1]*pw->ne[2] == a->ne[1])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5695, "GGML_ASSERT(%s) failed", "pw->ne[1]*pw->ne[2] == a->ne[1]"
);
5696
5697 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
5698 ggml_set_op_params_i32(result, 0, inplace ? 1 : 0);
5699
5700 result->op = GGML_OP_ADD_REL_POS;
5701 result->src[0] = a;
5702 result->src[1] = pw;
5703 result->src[2] = ph;
5704
5705 return result;
5706}
5707
5708struct ggml_tensor * ggml_add_rel_pos(
5709 struct ggml_context * ctx,
5710 struct ggml_tensor * a,
5711 struct ggml_tensor * pw,
5712 struct ggml_tensor * ph) {
5713 return ggml_add_rel_pos_impl(ctx, a, pw, ph, false0);
5714}
5715
5716struct ggml_tensor * ggml_add_rel_pos_inplace(
5717 struct ggml_context * ctx,
5718 struct ggml_tensor * a,
5719 struct ggml_tensor * pw,
5720 struct ggml_tensor * ph) {
5721 return ggml_add_rel_pos_impl(ctx, a, pw, ph, true1);
5722}
5723
5724// ggml_rwkv_wkv6
5725
5726struct ggml_tensor * ggml_rwkv_wkv6(
5727 struct ggml_context * ctx,
5728 struct ggml_tensor * k,
5729 struct ggml_tensor * v,
5730 struct ggml_tensor * r,
5731 struct ggml_tensor * tf,
5732 struct ggml_tensor * td,
5733 struct ggml_tensor * state) {
5734 GGML_ASSERT(ggml_is_contiguous(k))if (!(ggml_is_contiguous(k))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5734, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(k)");
5735 GGML_ASSERT(ggml_is_contiguous(v))if (!(ggml_is_contiguous(v))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5735, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(v)");
5736 GGML_ASSERT(ggml_is_contiguous(r))if (!(ggml_is_contiguous(r))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5736, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(r)");
5737 GGML_ASSERT(ggml_is_contiguous(tf))if (!(ggml_is_contiguous(tf))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5737, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(tf)");
5738 GGML_ASSERT(ggml_is_contiguous(td))if (!(ggml_is_contiguous(td))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5738, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(td)");
5739 GGML_ASSERT(ggml_is_contiguous(state))if (!(ggml_is_contiguous(state))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5739, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(state)"
);
5740
5741 const int64_t S = k->ne[0];
5742 const int64_t H = k->ne[1];
5743 const int64_t n_tokens = k->ne[2];
5744 const int64_t n_seqs = state->ne[1];
5745 {
5746 GGML_ASSERT(v->ne[0] == S && v->ne[1] == H && v->ne[2] == n_tokens)if (!(v->ne[0] == S && v->ne[1] == H &&
v->ne[2] == n_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5746, "GGML_ASSERT(%s) failed", "v->ne[0] == S && v->ne[1] == H && v->ne[2] == n_tokens"
);
5747 GGML_ASSERT(r->ne[0] == S && r->ne[1] == H && r->ne[2] == n_tokens)if (!(r->ne[0] == S && r->ne[1] == H &&
r->ne[2] == n_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5747, "GGML_ASSERT(%s) failed", "r->ne[0] == S && r->ne[1] == H && r->ne[2] == n_tokens"
);
5748 GGML_ASSERT(td->ne[0] == S && td->ne[1] == H && td->ne[2] == n_tokens)if (!(td->ne[0] == S && td->ne[1] == H &&
td->ne[2] == n_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5748, "GGML_ASSERT(%s) failed", "td->ne[0] == S && td->ne[1] == H && td->ne[2] == n_tokens"
);
5749 GGML_ASSERT(ggml_nelements(state) == S * S * H * n_seqs)if (!(ggml_nelements(state) == S * S * H * n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5749, "GGML_ASSERT(%s) failed", "ggml_nelements(state) == S * S * H * n_seqs"
);
5750 }
5751
5752 // concat output and new_state
5753 const int64_t ne[4] = { S * H, n_tokens + S * n_seqs, 1, 1 };
5754 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
5755
5756 result->op = GGML_OP_RWKV_WKV6;
5757 result->src[0] = k;
5758 result->src[1] = v;
5759 result->src[2] = r;
5760 result->src[3] = tf;
5761 result->src[4] = td;
5762 result->src[5] = state;
5763
5764 return result;
5765}
5766
5767// ggml_gated_linear_attn
5768
5769struct ggml_tensor * ggml_gated_linear_attn(
5770 struct ggml_context * ctx,
5771 struct ggml_tensor * k,
5772 struct ggml_tensor * v,
5773 struct ggml_tensor * q,
5774 struct ggml_tensor * g,
5775 struct ggml_tensor * state,
5776 float scale) {
5777 GGML_ASSERT(ggml_is_contiguous(k))if (!(ggml_is_contiguous(k))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5777, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(k)");
5778 GGML_ASSERT(ggml_is_contiguous(v))if (!(ggml_is_contiguous(v))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5778, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(v)");
5779 GGML_ASSERT(ggml_is_contiguous(q))if (!(ggml_is_contiguous(q))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5779, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(q)");
5780 GGML_ASSERT(ggml_is_contiguous(g))if (!(ggml_is_contiguous(g))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5780, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(g)");
5781 GGML_ASSERT(ggml_is_contiguous(state))if (!(ggml_is_contiguous(state))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5781, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(state)"
);
5782
5783 const int64_t S = k->ne[0];
5784 const int64_t H = k->ne[1];
5785 const int64_t n_tokens = k->ne[2];
5786 const int64_t n_seqs = state->ne[1];
5787 {
5788 GGML_ASSERT(v->ne[0] == S && v->ne[1] == H && v->ne[2] == n_tokens)if (!(v->ne[0] == S && v->ne[1] == H &&
v->ne[2] == n_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5788, "GGML_ASSERT(%s) failed", "v->ne[0] == S && v->ne[1] == H && v->ne[2] == n_tokens"
);
5789 GGML_ASSERT(q->ne[0] == S && q->ne[1] == H && q->ne[2] == n_tokens)if (!(q->ne[0] == S && q->ne[1] == H &&
q->ne[2] == n_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5789, "GGML_ASSERT(%s) failed", "q->ne[0] == S && q->ne[1] == H && q->ne[2] == n_tokens"
);
5790 GGML_ASSERT(g->ne[0] == S && g->ne[1] == H && g->ne[2] == n_tokens)if (!(g->ne[0] == S && g->ne[1] == H &&
g->ne[2] == n_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5790, "GGML_ASSERT(%s) failed", "g->ne[0] == S && g->ne[1] == H && g->ne[2] == n_tokens"
);
5791 GGML_ASSERT(ggml_nelements(state) == S * S * H * n_seqs)if (!(ggml_nelements(state) == S * S * H * n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5791, "GGML_ASSERT(%s) failed", "ggml_nelements(state) == S * S * H * n_seqs"
);
5792 }
5793
5794 // concat output and new_state
5795 const int64_t ne[4] = { S * H, n_tokens + S * n_seqs, 1, 1 };
5796 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
5797
5798 ggml_set_op_params_f32(result, 0, scale);
5799
5800 result->op = GGML_OP_GATED_LINEAR_ATTN;
5801 result->src[0] = k;
5802 result->src[1] = v;
5803 result->src[2] = q;
5804 result->src[3] = g;
5805 result->src[4] = state;
5806
5807 return result;
5808}
5809
5810// ggml_rwkv_wkv7
5811
5812struct ggml_tensor * ggml_rwkv_wkv7(
5813 struct ggml_context * ctx,
5814 struct ggml_tensor * r,
5815 struct ggml_tensor * w,
5816 struct ggml_tensor * k,
5817 struct ggml_tensor * v,
5818 struct ggml_tensor * a,
5819 struct ggml_tensor * b,
5820 struct ggml_tensor * state) {
5821 GGML_ASSERT(ggml_is_contiguous(r))if (!(ggml_is_contiguous(r))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5821, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(r)");
5822 GGML_ASSERT(ggml_is_contiguous(w))if (!(ggml_is_contiguous(w))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5822, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(w)");
5823 GGML_ASSERT(ggml_is_contiguous(k))if (!(ggml_is_contiguous(k))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5823, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(k)");
5824 GGML_ASSERT(ggml_is_contiguous(v))if (!(ggml_is_contiguous(v))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5824, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(v)");
5825 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5825, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
5826 GGML_ASSERT(ggml_is_contiguous(b))if (!(ggml_is_contiguous(b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5826, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(b)");
5827 GGML_ASSERT(ggml_is_contiguous(state))if (!(ggml_is_contiguous(state))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5827, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(state)"
);
5828
5829 const int64_t S = k->ne[0];
5830 const int64_t H = k->ne[1];
5831 const int64_t n_tokens = k->ne[2];
5832 const int64_t n_seqs = state->ne[1];
5833 {
5834 GGML_ASSERT(w->ne[0] == S && w->ne[1] == H && w->ne[2] == n_tokens)if (!(w->ne[0] == S && w->ne[1] == H &&
w->ne[2] == n_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5834, "GGML_ASSERT(%s) failed", "w->ne[0] == S && w->ne[1] == H && w->ne[2] == n_tokens"
);
5835 GGML_ASSERT(k->ne[0] == S && k->ne[1] == H && k->ne[2] == n_tokens)if (!(k->ne[0] == S && k->ne[1] == H &&
k->ne[2] == n_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5835, "GGML_ASSERT(%s) failed", "k->ne[0] == S && k->ne[1] == H && k->ne[2] == n_tokens"
);
5836 GGML_ASSERT(v->ne[0] == S && v->ne[1] == H && v->ne[2] == n_tokens)if (!(v->ne[0] == S && v->ne[1] == H &&
v->ne[2] == n_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5836, "GGML_ASSERT(%s) failed", "v->ne[0] == S && v->ne[1] == H && v->ne[2] == n_tokens"
);
5837 GGML_ASSERT(a->ne[0] == S && a->ne[1] == H && a->ne[2] == n_tokens)if (!(a->ne[0] == S && a->ne[1] == H &&
a->ne[2] == n_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5837, "GGML_ASSERT(%s) failed", "a->ne[0] == S && a->ne[1] == H && a->ne[2] == n_tokens"
);
5838 GGML_ASSERT(b->ne[0] == S && b->ne[1] == H && b->ne[2] == n_tokens)if (!(b->ne[0] == S && b->ne[1] == H &&
b->ne[2] == n_tokens)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5838, "GGML_ASSERT(%s) failed", "b->ne[0] == S && b->ne[1] == H && b->ne[2] == n_tokens"
);
5839 GGML_ASSERT(ggml_nelements(state) == S * S * H * n_seqs)if (!(ggml_nelements(state) == S * S * H * n_seqs)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5839, "GGML_ASSERT(%s) failed", "ggml_nelements(state) == S * S * H * n_seqs"
);
5840 }
5841
5842 // concat output and new_state
5843 const int64_t ne[4] = { S * H, n_tokens + S * n_seqs, 1, 1 };
5844 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
5845
5846 result->op = GGML_OP_RWKV_WKV7;
5847 result->src[0] = r;
5848 result->src[1] = w;
5849 result->src[2] = k;
5850 result->src[3] = v;
5851 result->src[4] = a;
5852 result->src[5] = b;
5853 result->src[6] = state;
5854
5855 return result;
5856}
5857
5858// ggml_unary
5859
5860static struct ggml_tensor * ggml_unary_impl(
5861 struct ggml_context * ctx,
5862 struct ggml_tensor * a,
5863 enum ggml_unary_op op,
5864 bool_Bool inplace) {
5865 GGML_ASSERT(ggml_is_contiguous_rows(a))if (!(ggml_is_contiguous_rows(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5865, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_rows(a)"
);
5866
5867 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
5868
5869 ggml_set_op_params_i32(result, 0, (int32_t) op);
5870
5871 result->op = GGML_OP_UNARY;
5872 result->src[0] = a;
5873
5874 return result;
5875}
5876
5877struct ggml_tensor * ggml_unary(
5878 struct ggml_context * ctx,
5879 struct ggml_tensor * a,
5880 enum ggml_unary_op op) {
5881 return ggml_unary_impl(ctx, a, op, false0);
5882}
5883
5884struct ggml_tensor * ggml_unary_inplace(
5885 struct ggml_context * ctx,
5886 struct ggml_tensor * a,
5887 enum ggml_unary_op op) {
5888 return ggml_unary_impl(ctx, a, op, true1);
5889}
5890
5891// ggml_map_custom1
5892
5893static struct ggml_tensor * ggml_map_custom1_impl(
5894 struct ggml_context * ctx,
5895 struct ggml_tensor * a,
5896 const ggml_custom1_op_t fun,
5897 int n_tasks,
5898 void * userdata,
5899 bool_Bool inplace) {
5900 GGML_ASSERT(n_tasks == GGML_N_TASKS_MAX || n_tasks > 0)if (!(n_tasks == (-1) || n_tasks > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5900, "GGML_ASSERT(%s) failed", "n_tasks == GGML_N_TASKS_MAX || n_tasks > 0"
);
5901
5902 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
5903
5904 struct ggml_map_custom1_op_params params = {
5905 /*.fun =*/ fun,
5906 /*.n_tasks =*/ n_tasks,
5907 /*.userdata =*/ userdata
5908 };
5909 ggml_set_op_params(result, &params, sizeof(params));
5910
5911 result->op = GGML_OP_MAP_CUSTOM1;
5912 result->src[0] = a;
5913
5914 return result;
5915}
5916
5917struct ggml_tensor * ggml_map_custom1(
5918 struct ggml_context * ctx,
5919 struct ggml_tensor * a,
5920 const ggml_custom1_op_t fun,
5921 int n_tasks,
5922 void * userdata) {
5923 return ggml_map_custom1_impl(ctx, a, fun, n_tasks, userdata, false0);
5924}
5925
5926struct ggml_tensor * ggml_map_custom1_inplace(
5927 struct ggml_context * ctx,
5928 struct ggml_tensor * a,
5929 const ggml_custom1_op_t fun,
5930 int n_tasks,
5931 void * userdata) {
5932 return ggml_map_custom1_impl(ctx, a, fun, n_tasks, userdata, true1);
5933}
5934
5935// ggml_map_custom2
5936
5937static struct ggml_tensor * ggml_map_custom2_impl(
5938 struct ggml_context * ctx,
5939 struct ggml_tensor * a,
5940 struct ggml_tensor * b,
5941 const ggml_custom2_op_t fun,
5942 int n_tasks,
5943 void * userdata,
5944 bool_Bool inplace) {
5945 GGML_ASSERT(n_tasks == GGML_N_TASKS_MAX || n_tasks > 0)if (!(n_tasks == (-1) || n_tasks > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5945, "GGML_ASSERT(%s) failed", "n_tasks == GGML_N_TASKS_MAX || n_tasks > 0"
);
5946
5947 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
5948
5949 struct ggml_map_custom2_op_params params = {
5950 /*.fun =*/ fun,
5951 /*.n_tasks =*/ n_tasks,
5952 /*.userdata =*/ userdata
5953 };
5954 ggml_set_op_params(result, &params, sizeof(params));
5955
5956 result->op = GGML_OP_MAP_CUSTOM2;
5957 result->src[0] = a;
5958 result->src[1] = b;
5959
5960 return result;
5961}
5962
5963struct ggml_tensor * ggml_map_custom2(
5964 struct ggml_context * ctx,
5965 struct ggml_tensor * a,
5966 struct ggml_tensor * b,
5967 const ggml_custom2_op_t fun,
5968 int n_tasks,
5969 void * userdata) {
5970 return ggml_map_custom2_impl(ctx, a, b, fun, n_tasks, userdata, false0);
5971}
5972
5973struct ggml_tensor * ggml_map_custom2_inplace(
5974 struct ggml_context * ctx,
5975 struct ggml_tensor * a,
5976 struct ggml_tensor * b,
5977 const ggml_custom2_op_t fun,
5978 int n_tasks,
5979 void * userdata) {
5980 return ggml_map_custom2_impl(ctx, a, b, fun, n_tasks, userdata, true1);
5981}
5982
5983// ggml_map_custom3
5984
5985static struct ggml_tensor * ggml_map_custom3_impl(
5986 struct ggml_context * ctx,
5987 struct ggml_tensor * a,
5988 struct ggml_tensor * b,
5989 struct ggml_tensor * c,
5990 const ggml_custom3_op_t fun,
5991 int n_tasks,
5992 void * userdata,
5993 bool_Bool inplace) {
5994 GGML_ASSERT(n_tasks == GGML_N_TASKS_MAX || n_tasks > 0)if (!(n_tasks == (-1) || n_tasks > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 5994, "GGML_ASSERT(%s) failed", "n_tasks == GGML_N_TASKS_MAX || n_tasks > 0"
);
5995
5996 struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
5997
5998 struct ggml_map_custom3_op_params params = {
5999 /*.fun =*/ fun,
6000 /*.n_tasks =*/ n_tasks,
6001 /*.userdata =*/ userdata
6002 };
6003 ggml_set_op_params(result, &params, sizeof(params));
6004
6005 result->op = GGML_OP_MAP_CUSTOM3;
6006 result->src[0] = a;
6007 result->src[1] = b;
6008 result->src[2] = c;
6009
6010 return result;
6011}
6012
6013struct ggml_tensor * ggml_map_custom3(
6014 struct ggml_context * ctx,
6015 struct ggml_tensor * a,
6016 struct ggml_tensor * b,
6017 struct ggml_tensor * c,
6018 const ggml_custom3_op_t fun,
6019 int n_tasks,
6020 void * userdata) {
6021 return ggml_map_custom3_impl(ctx, a, b, c, fun, n_tasks, userdata, false0);
6022}
6023
6024struct ggml_tensor * ggml_map_custom3_inplace(
6025 struct ggml_context * ctx,
6026 struct ggml_tensor * a,
6027 struct ggml_tensor * b,
6028 struct ggml_tensor * c,
6029 const ggml_custom3_op_t fun,
6030 int n_tasks,
6031 void * userdata) {
6032 return ggml_map_custom3_impl(ctx, a, b, c, fun, n_tasks, userdata, true1);
6033}
6034
6035struct ggml_tensor * ggml_custom_4d(
6036 struct ggml_context * ctx,
6037 enum ggml_type type,
6038 int64_t ne0,
6039 int64_t ne1,
6040 int64_t ne2,
6041 int64_t ne3,
6042 struct ggml_tensor ** args,
6043 int n_args,
6044 ggml_custom_op_t fun,
6045 int n_tasks,
6046 void * userdata) {
6047
6048 GGML_ASSERT(n_args < GGML_MAX_SRC)if (!(n_args < 10)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6048, "GGML_ASSERT(%s) failed", "n_args < GGML_MAX_SRC");
6049
6050 struct ggml_tensor * result = ggml_new_tensor_4d(ctx, type, ne0, ne1, ne2, ne3);
6051
6052 struct ggml_custom_op_params params = {
6053 /*.fun =*/ fun,
6054 /*.n_tasks =*/ n_tasks,
6055 /*.userdata =*/ userdata
6056 };
6057 ggml_set_op_params(result, &params, sizeof(params));
6058
6059 result->op = GGML_OP_CUSTOM;
6060 for (int i = 0; i < n_args; i++) {
6061 result->src[i] = args[i];
6062 }
6063
6064 return result;
6065}
6066
6067struct ggml_tensor * ggml_custom_inplace(
6068 struct ggml_context * ctx,
6069 struct ggml_tensor * a,
6070 struct ggml_tensor ** args,
6071 int n_args,
6072 ggml_custom_op_t fun,
6073 int n_tasks,
6074 void * userdata) {
6075
6076 GGML_ASSERT(n_args < GGML_MAX_SRC - 1)if (!(n_args < 10 - 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6076, "GGML_ASSERT(%s) failed", "n_args < GGML_MAX_SRC - 1"
);
6077
6078 struct ggml_tensor * result = ggml_view_tensor(ctx, a);
6079
6080 struct ggml_custom_op_params params = {
6081 /*.fun =*/ fun,
6082 /*.n_tasks =*/ n_tasks,
6083 /*.userdata =*/ userdata
6084 };
6085 ggml_set_op_params(result, &params, sizeof(params));
6086
6087 result->op = GGML_OP_CUSTOM;
6088 result->src[0] = a;
6089 for (int i = 0; i < n_args; i++) {
6090 result->src[i + 1] = args[i];
6091 }
6092
6093 return result;
6094}
6095// ggml_cross_entropy_loss
6096
6097struct ggml_tensor * ggml_cross_entropy_loss(
6098 struct ggml_context * ctx,
6099 struct ggml_tensor * a,
6100 struct ggml_tensor * b) {
6101 GGML_ASSERT(ggml_are_same_shape(a, b))if (!(ggml_are_same_shape(a, b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6101, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(a, b)"
);
6102
6103 struct ggml_tensor * result = ggml_new_tensor_1d(ctx, a->type, 1);
6104
6105 result->op = GGML_OP_CROSS_ENTROPY_LOSS;
6106 result->src[0] = a;
6107 result->src[1] = b;
6108
6109 return result;
6110}
6111
6112// ggml_cross_entropy_loss_back
6113
6114struct ggml_tensor * ggml_cross_entropy_loss_back(
6115 struct ggml_context * ctx,
6116 struct ggml_tensor * a,
6117 struct ggml_tensor * b,
6118 struct ggml_tensor * c) {
6119 GGML_ASSERT(ggml_is_scalar(a))if (!(ggml_is_scalar(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6119, "GGML_ASSERT(%s) failed", "ggml_is_scalar(a)");
6120 GGML_ASSERT(ggml_are_same_shape(b, c))if (!(ggml_are_same_shape(b, c))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6120, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(b, c)"
);
6121
6122 struct ggml_tensor * result = ggml_dup_tensor(ctx, b);
6123
6124 result->op = GGML_OP_CROSS_ENTROPY_LOSS_BACK;
6125 result->src[0] = a;
6126 result->src[1] = b;
6127 result->src[2] = c;
6128
6129 return result;
6130}
6131
6132// opt_step_adamw
6133
6134struct ggml_tensor * ggml_opt_step_adamw(
6135 struct ggml_context * ctx,
6136 struct ggml_tensor * a,
6137 struct ggml_tensor * grad,
6138 struct ggml_tensor * m,
6139 struct ggml_tensor * v,
6140 struct ggml_tensor * adamw_params) {
6141 GGML_ASSERT(a->flags & GGML_TENSOR_FLAG_PARAM)if (!(a->flags & GGML_TENSOR_FLAG_PARAM)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6141, "GGML_ASSERT(%s) failed", "a->flags & GGML_TENSOR_FLAG_PARAM"
);
6142 GGML_ASSERT(ggml_are_same_shape(a, grad))if (!(ggml_are_same_shape(a, grad))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6142, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(a, grad)"
);
6143 GGML_ASSERT(ggml_are_same_shape(a, m))if (!(ggml_are_same_shape(a, m))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6143, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(a, m)"
);
6144 GGML_ASSERT(ggml_are_same_shape(a, v))if (!(ggml_are_same_shape(a, v))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6144, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(a, v)"
);
6145 GGML_ASSERT(adamw_params->type == GGML_TYPE_F32)if (!(adamw_params->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6145, "GGML_ASSERT(%s) failed", "adamw_params->type == GGML_TYPE_F32"
);
6146 GGML_ASSERT(ggml_nelements(adamw_params) == 7)if (!(ggml_nelements(adamw_params) == 7)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6146, "GGML_ASSERT(%s) failed", "ggml_nelements(adamw_params) == 7"
);
6147
6148 struct ggml_tensor * result = ggml_view_tensor(ctx, a);
6149
6150 result->op = GGML_OP_OPT_STEP_ADAMW;
6151 result->src[0] = a;
6152 result->src[1] = grad;
6153 result->src[2] = m;
6154 result->src[3] = v;
6155 result->src[4] = adamw_params;
6156
6157 return result;
6158}
6159
6160// opt_step_sgd
6161
6162struct ggml_tensor * ggml_opt_step_sgd(
6163 struct ggml_context * ctx,
6164 struct ggml_tensor * a,
6165 struct ggml_tensor * grad,
6166 struct ggml_tensor * params) {
6167 GGML_ASSERT(a->flags & GGML_TENSOR_FLAG_PARAM)if (!(a->flags & GGML_TENSOR_FLAG_PARAM)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6167, "GGML_ASSERT(%s) failed", "a->flags & GGML_TENSOR_FLAG_PARAM"
);
6168 GGML_ASSERT(ggml_are_same_shape(a, grad))if (!(ggml_are_same_shape(a, grad))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6168, "GGML_ASSERT(%s) failed", "ggml_are_same_shape(a, grad)"
);
6169 GGML_ASSERT(params->type == GGML_TYPE_F32)if (!(params->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6169, "GGML_ASSERT(%s) failed", "params->type == GGML_TYPE_F32"
);
6170 GGML_ASSERT(ggml_nelements(params) == 2)if (!(ggml_nelements(params) == 2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6170, "GGML_ASSERT(%s) failed", "ggml_nelements(params) == 2"
);
6171
6172 struct ggml_tensor * result = ggml_view_tensor(ctx, a);
6173
6174 result->op = GGML_OP_OPT_STEP_SGD;
6175 result->src[0] = a;
6176 result->src[1] = grad;
6177 result->src[2] = params;
6178
6179 return result;
6180}
6181
6182// solve_tri
6183
6184struct ggml_tensor * ggml_solve_tri(
6185 struct ggml_context * ctx,
6186 struct ggml_tensor * a,
6187 struct ggml_tensor * b,
6188 bool_Bool left,
6189 bool_Bool lower,
6190 bool_Bool uni) {
6191 GGML_ASSERT(a->type == GGML_TYPE_F32)if (!(a->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6191, "GGML_ASSERT(%s) failed", "a->type == GGML_TYPE_F32"
);
6192 GGML_ASSERT(b->type == GGML_TYPE_F32)if (!(b->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6192, "GGML_ASSERT(%s) failed", "b->type == GGML_TYPE_F32"
);
6193
6194 // A must be square and lower diagonal
6195 GGML_ASSERT(a->ne[0] == a->ne[1])if (!(a->ne[0] == a->ne[1])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6195, "GGML_ASSERT(%s) failed", "a->ne[0] == a->ne[1]"
);
6196 // B must have same outer dimension as A
6197 GGML_ASSERT(a->ne[1] == b->ne[1])if (!(a->ne[1] == b->ne[1])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6197, "GGML_ASSERT(%s) failed", "a->ne[1] == b->ne[1]"
);
6198
6199 // batch dimensions must be equal
6200 GGML_ASSERT(a->ne[2] == b->ne[2])if (!(a->ne[2] == b->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6200, "GGML_ASSERT(%s) failed", "a->ne[2] == b->ne[2]"
);
6201 GGML_ASSERT(a->ne[3] == b->ne[3])if (!(a->ne[3] == b->ne[3])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6201, "GGML_ASSERT(%s) failed", "a->ne[3] == b->ne[3]"
);
6202
6203 GGML_ASSERT(ggml_is_contiguous(a))if (!(ggml_is_contiguous(a))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6203, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(a)");
6204 GGML_ASSERT(ggml_is_contiguous(b))if (!(ggml_is_contiguous(b))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6204, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(b)");
6205
6206 GGML_ASSERT(lower && left && !uni)if (!(lower && left && !uni)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6206, "GGML_ASSERT(%s) failed", "lower && left && !uni"
); // TODO: support other variants
6207
6208 struct ggml_tensor * result = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, b->ne[0], b->ne[1], b->ne[2], b->ne[3]);
6209
6210 result->op = GGML_OP_SOLVE_TRI;
6211 result->src[0] = a;
6212 result->src[1] = b;
6213
6214 return result;
6215}
6216
6217// ggml_gated_delta_net
6218
6219struct ggml_tensor * ggml_gated_delta_net(
6220 struct ggml_context * ctx,
6221 struct ggml_tensor * q,
6222 struct ggml_tensor * k,
6223 struct ggml_tensor * v,
6224 struct ggml_tensor * g,
6225 struct ggml_tensor * beta,
6226 struct ggml_tensor * state,
6227 int64_t K) {
6228 GGML_ASSERT(ggml_is_contiguous_rows(q))if (!(ggml_is_contiguous_rows(q))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6228, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_rows(q)"
);
6229 GGML_ASSERT(ggml_is_contiguous_rows(k))if (!(ggml_is_contiguous_rows(k))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6229, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_rows(k)"
);
6230 GGML_ASSERT(ggml_is_contiguous_rows(v))if (!(ggml_is_contiguous_rows(v))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6230, "GGML_ASSERT(%s) failed", "ggml_is_contiguous_rows(v)"
);
6231 GGML_ASSERT(ggml_is_contiguous(g))if (!(ggml_is_contiguous(g))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6231, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(g)");
6232 GGML_ASSERT(ggml_is_contiguous(beta))if (!(ggml_is_contiguous(beta))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6232, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(beta)");
6233 GGML_ASSERT(ggml_is_contiguous(state))if (!(ggml_is_contiguous(state))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6233, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(state)"
);
6234
6235 GGML_ASSERT(q->type == GGML_TYPE_F32)if (!(q->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6235, "GGML_ASSERT(%s) failed", "q->type == GGML_TYPE_F32"
);
6236 GGML_ASSERT(k->type == GGML_TYPE_F32)if (!(k->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6236, "GGML_ASSERT(%s) failed", "k->type == GGML_TYPE_F32"
);
6237 GGML_ASSERT(v->type == GGML_TYPE_F32)if (!(v->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6237, "GGML_ASSERT(%s) failed", "v->type == GGML_TYPE_F32"
);
6238 GGML_ASSERT(g->type == GGML_TYPE_F32)if (!(g->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6238, "GGML_ASSERT(%s) failed", "g->type == GGML_TYPE_F32"
);
6239 GGML_ASSERT(beta->type == GGML_TYPE_F32)if (!(beta->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6239, "GGML_ASSERT(%s) failed", "beta->type == GGML_TYPE_F32"
);
6240 GGML_ASSERT(state->type == GGML_TYPE_F32)if (!(state->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6240, "GGML_ASSERT(%s) failed", "state->type == GGML_TYPE_F32"
);
6241
6242 const int64_t S_v = v->ne[0];
6243 const int64_t H = v->ne[1];
6244 const int64_t n_tokens = v->ne[2];
6245 const int64_t n_seqs = v->ne[3];
6246
6247 // gate: scalar [1, H, T, B] or vector [S_v, H, T, B] (KDA)
6248 GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v)if (!(g->ne[0] == 1 || g->ne[0] == S_v)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6248, "GGML_ASSERT(%s) failed", "g->ne[0] == 1 || g->ne[0] == S_v"
);
6249 GGML_ASSERT(beta->ne[0] == 1)if (!(beta->ne[0] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6249, "GGML_ASSERT(%s) failed", "beta->ne[0] == 1");
6250
6251 // state holds the initial state s0 only: [S_v, S_v, H, n_seqs]. K (snapshot slot count) is an op param.
6252 GGML_ASSERT(state->ne[0] == S_v)if (!(state->ne[0] == S_v)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6252, "GGML_ASSERT(%s) failed", "state->ne[0] == S_v");
6253 GGML_ASSERT(state->ne[1] == S_v)if (!(state->ne[1] == S_v)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6253, "GGML_ASSERT(%s) failed", "state->ne[1] == S_v");
6254 GGML_ASSERT(state->ne[2] == H)if (!(state->ne[2] == H)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6254, "GGML_ASSERT(%s) failed", "state->ne[2] == H");
6255 GGML_ASSERT(state->ne[3] == n_seqs)if (!(state->ne[3] == n_seqs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6255, "GGML_ASSERT(%s) failed", "state->ne[3] == n_seqs"
);
6256 GGML_ASSERT(K >= 1)if (!(K >= 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6256, "GGML_ASSERT(%s) failed", "K >= 1");
6257 const int64_t state_rows = K * S_v * n_seqs;
6258 const int64_t ne[4] = { S_v * H, n_tokens * n_seqs + state_rows, 1, 1 };
6259 struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
6260
6261 ggml_set_op_params_i32(result, 0, (int32_t) K);
6262
6263 result->op = GGML_OP_GATED_DELTA_NET;
6264 result->src[0] = q;
6265 result->src[1] = k;
6266 result->src[2] = v;
6267 result->src[3] = g;
6268 result->src[4] = beta;
6269 result->src[5] = state;
6270
6271 return result;
6272}
6273
6274////////////////////////////////////////////////////////////////////////////////
6275
6276struct ggml_hash_set ggml_hash_set_new(size_t size) {
6277 size = ggml_hash_size(size);
6278 struct ggml_hash_set result;
6279 result.size = size;
6280 result.keys = GGML_MALLOC(sizeof(struct ggml_tensor *) * size)ggml_malloc(sizeof(struct ggml_tensor *) * size);
6281 result.used = GGML_CALLOC(ggml_bitset_size(size), sizeof(ggml_bitset_t))ggml_calloc(ggml_bitset_size(size), sizeof(ggml_bitset_t));
6282 return result;
6283}
6284
6285void ggml_hash_set_reset(struct ggml_hash_set * hash_set) {
6286 memset(hash_set->used, 0, sizeof(ggml_bitset_t) * ggml_bitset_size(hash_set->size));
6287}
6288
6289void ggml_hash_set_free(struct ggml_hash_set * hash_set) {
6290 GGML_FREE(hash_set->used)free(hash_set->used);
6291 GGML_FREE(hash_set->keys)free(hash_set->keys);
6292}
6293
6294size_t ggml_hash_size(size_t min_sz) {
6295 // next primes after powers of two
6296 static const size_t primes[] = {
6297 2, 3, 5, 11, 17, 37, 67, 131, 257, 521, 1031,
6298 2053, 4099, 8209, 16411, 32771, 65537, 131101,
6299 262147, 524309, 1048583, 2097169, 4194319, 8388617,
6300 16777259, 33554467, 67108879, 134217757, 268435459,
6301 536870923, 1073741827, 2147483659
6302 };
6303 static const size_t n_primes = sizeof(primes)/sizeof(primes[0]);
6304
6305 // find the smallest prime that is larger or equal than min_sz
6306 size_t l = 0;
6307 size_t r = n_primes;
6308 while (l < r) {
6309 size_t m = (l + r)/2;
6310 if (primes[m] < min_sz) {
6311 l = m + 1;
6312 } else {
6313 r = m;
6314 }
6315 }
6316 size_t sz = l < n_primes ? primes[l] : min_sz | 1;
6317 return sz;
6318}
6319
6320struct hash_map {
6321 struct ggml_hash_set set;
6322 struct ggml_tensor ** vals;
6323};
6324
6325static struct hash_map * ggml_new_hash_map(size_t size) {
6326 struct hash_map * result = GGML_MALLOC(sizeof(struct hash_map))ggml_malloc(sizeof(struct hash_map));
6327 result->set = ggml_hash_set_new(size);
6328 result->vals = GGML_CALLOC(result->set.size, sizeof(struct ggml_tensor *))ggml_calloc(result->set.size, sizeof(struct ggml_tensor *)
);
6329 return result;
6330}
6331
6332static void ggml_hash_map_free(struct hash_map * map) {
6333 ggml_hash_set_free(&map->set);
6334 GGML_FREE(map->vals)free(map->vals);
6335 GGML_FREE(map)free(map);
6336}
6337
6338// utility functions to change gradients
6339// isrc is the index of tensor in cgraph->visited_has_set.keys
6340// the corresponding gradient (accumulators) are also at position isrc
6341// if tensor has a gradient accumulator, modify that accumulator in-place
6342// else if there is no gradient for tensor, set the corresponding value
6343// else, just add/subtract/etc. the gradients
6344
6345static void ggml_add_or_set(
6346 struct ggml_context * ctx,
6347 struct ggml_cgraph * cgraph,
6348 size_t isrc,
6349 struct ggml_tensor * tensor) {
6350 struct ggml_tensor * src = cgraph->visited_hash_set.keys[isrc];
6351 GGML_ASSERT(src)if (!(src)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6351, "GGML_ASSERT(%s) failed", "src");
6352 if (cgraph->grads[isrc]) {
6353 cgraph->grads[isrc] = ggml_add_impl(ctx, cgraph->grads[isrc], tensor, /*inplace =*/ cgraph->grad_accs[isrc]);
6354 } else {
6355 cgraph->grads[isrc] = tensor;
6356 }
6357 ggml_format_name(cgraph->grads[isrc], "grad for %s", src->name);
6358 ggml_build_forward_expand(cgraph, cgraph->grads[isrc]);
6359}
6360
6361static void ggml_acc_or_set(
6362 struct ggml_context * ctx,
6363 struct ggml_cgraph * cgraph,
6364 size_t isrc,
6365 struct ggml_tensor * tensor,
6366 const size_t nb1,
6367 const size_t nb2,
6368 const size_t nb3,
6369 const size_t offset) {
6370 struct ggml_tensor * src = cgraph->visited_hash_set.keys[isrc];
6371 GGML_ASSERT(src)if (!(src)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6371, "GGML_ASSERT(%s) failed", "src");
6372 if (cgraph->grads[isrc]) {
6373 cgraph->grads[isrc] = ggml_acc_impl(ctx, cgraph->grads[isrc], tensor, nb1, nb2, nb3, offset, cgraph->grad_accs[isrc]);
6374 } else {
6375 struct ggml_tensor * a_zero = ggml_scale(ctx, src, 0.0f); // FIXME this is going to produce NaN if a contains inf/NaN
6376 cgraph->grads[isrc] = ggml_acc_impl(ctx, a_zero, tensor, nb1, nb2, nb3, offset, false0);
6377 }
6378 ggml_format_name(cgraph->grads[isrc], "grad for %s", cgraph->visited_hash_set.keys[isrc]->name);
6379 ggml_build_forward_expand(cgraph, cgraph->grads[isrc]);
6380}
6381
6382static void ggml_add1_or_set(
6383 struct ggml_context * ctx,
6384 struct ggml_cgraph * cgraph,
6385 size_t isrc,
6386 struct ggml_tensor * tensor) {
6387 struct ggml_tensor * src = cgraph->visited_hash_set.keys[isrc];
6388 GGML_ASSERT(src)if (!(src)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6388, "GGML_ASSERT(%s) failed", "src");
6389 if (cgraph->grads[isrc]) {
6390 cgraph->grads[isrc] = ggml_add1_impl(ctx, cgraph->grads[isrc], tensor, cgraph->grad_accs[isrc]);
6391 } else {
6392 cgraph->grads[isrc] = ggml_repeat(ctx, tensor, src);
6393 }
6394 ggml_format_name(cgraph->grads[isrc], "grad for %s", src->name);
6395 ggml_build_forward_expand(cgraph, cgraph->grads[isrc]);
6396}
6397
6398static void ggml_sub_or_set(
6399 struct ggml_context * ctx,
6400 struct ggml_cgraph * cgraph,
6401 size_t isrc,
6402 struct ggml_tensor * tensor) {
6403 struct ggml_tensor * src = cgraph->visited_hash_set.keys[isrc];
6404 GGML_ASSERT(src)if (!(src)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6404, "GGML_ASSERT(%s) failed", "src");
6405 if (cgraph->grads[isrc]) {
6406 cgraph->grads[isrc] = ggml_sub_impl(ctx, cgraph->grads[isrc], tensor, cgraph->grad_accs[isrc]);
6407 } else {
6408 cgraph->grads[isrc] = ggml_neg(ctx, tensor);
6409 }
6410 ggml_format_name(cgraph->grads[isrc], "grad for %s", src->name);
6411 ggml_build_forward_expand(cgraph, cgraph->grads[isrc]);
6412}
6413
6414static void ggml_compute_backward(
6415 struct ggml_context * ctx, struct ggml_cgraph * cgraph, int i, const bool_Bool * grads_needed) {
6416 struct ggml_tensor * tensor = cgraph->nodes[i];
6417 struct ggml_tensor * grad = ggml_graph_get_grad(cgraph, tensor);
6418
6419 if (!grad) {
1
Assuming 'grad' is non-null
→
2
←
Taking false branch
→
6420 return;
6421 }
6422
6423 struct ggml_tensor * src0 = tensor->src[0];
3
←
'src0' initialized here
→
6424 struct ggml_tensor * src1 = tensor->src[1];
6425 struct ggml_tensor * src2 = tensor->src[2];
6426 struct ggml_hash_set * hash_set = &cgraph->visited_hash_set;
6427 const size_t isrc0 = src0 ? ggml_hash_find(hash_set, src0) : (size_t) -1;
4
←
Assuming 'src0' is null
→
5
←
'?' condition is false
→
6428 const size_t isrc1 = src1 ? ggml_hash_find(hash_set, src1) : (size_t) -1;
6
←
Assuming 'src1' is non-null
→
7
←
'?' condition is true
→
6429 const size_t isrc2 = src2 ? ggml_hash_find(hash_set, src2) : (size_t) -1;
8
←
Assuming 'src2' is null
→
9
←
'?' condition is false
→
6430 const bool_Bool src0_needs_grads = src0
9.1
'src0' is null
 && isrc0 != GGML_HASHSET_FULL((size_t)-1) && ggml_bitset_get(hash_set->used, isrc0) && grads_needed[isrc0];
6431 const bool_Bool src1_needs_grads = src1
9.2
'src1' is non-null
 && isrc1 != GGML_HASHSET_FULL((size_t)-1) && ggml_bitset_get(hash_set->used, isrc1) && grads_needed[isrc1];
10
←
Assuming the condition is true
→
11
←
Assuming the condition is true
→
6432 const bool_Bool src2_needs_grads = src2
11.1
'src2' is null
 && isrc2 != GGML_HASHSET_FULL((size_t)-1) && ggml_bitset_get(hash_set->used, isrc2) && grads_needed[isrc2];
6433
6434 switch (tensor->op) {
12
←
Control jumps to 'case GGML_OP_MUL_MAT:' at line 6562
→
6435 case GGML_OP_DUP: {
6436 if (src0_needs_grads) {
6437 ggml_add_or_set(ctx, cgraph, isrc0, grad);
6438 }
6439 } break;
6440 case GGML_OP_ADD: {
6441 if (src0_needs_grads) {
6442 ggml_add_or_set(ctx, cgraph, isrc0, grad);
6443 }
6444 if (src1_needs_grads) {
6445 struct ggml_tensor * tmp = grad;
6446 if (!ggml_are_same_shape(src0, src1)) {
6447 tmp = ggml_repeat_back(ctx, tmp, src1);
6448 }
6449 ggml_add_or_set(ctx, cgraph, isrc1, tmp);
6450 }
6451 } break;
6452 case GGML_OP_ADD1: {
6453 if (src0_needs_grads) {
6454 ggml_add_or_set(ctx, cgraph, isrc0, grad);
6455 }
6456 if (src1_needs_grads) {
6457 ggml_add_or_set(ctx, cgraph, isrc1, ggml_mean(ctx, grad)); // TODO: should probably be sum instead of mean
6458 }
6459 } break;
6460 case GGML_OP_ACC: {
6461 if (src0_needs_grads) {
6462 ggml_add_or_set(ctx, cgraph, isrc0, grad);
6463 }
6464 if (src1_needs_grads) {
6465 const size_t nb1 = ((int32_t *) tensor->op_params)[0];
6466 const size_t nb2 = ((int32_t *) tensor->op_params)[1];
6467 const size_t nb3 = ((int32_t *) tensor->op_params)[2];
6468 const size_t offset = ((int32_t *) tensor->op_params)[3];
6469
6470 struct ggml_tensor * tensor_grad_view = ggml_view_4d(ctx,
6471 grad, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],
6472 nb1, nb2, nb3, offset);
6473
6474 ggml_add_or_set(ctx, cgraph, isrc1, ggml_reshape(ctx, ggml_cont(ctx, tensor_grad_view), src1));
6475 }
6476 } break;
6477 case GGML_OP_SUB: {
6478 if (src0_needs_grads) {
6479 ggml_add_or_set(ctx, cgraph, isrc0, grad);
6480 }
6481 if (src1_needs_grads) {
6482 ggml_sub_or_set(ctx, cgraph, isrc1, grad);
6483 }
6484 } break;
6485 case GGML_OP_MUL: {
6486 if (src0_needs_grads) {
6487 ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, grad, src1));
6488 }
6489 if (src1_needs_grads) {
6490 struct ggml_tensor * tmp = ggml_mul(ctx, src0, grad);
6491 if (!ggml_are_same_shape(src0, src1)) {
6492 tmp = ggml_repeat_back(ctx, tmp, src1);
6493 }
6494 ggml_add_or_set(ctx, cgraph, isrc1, tmp);
6495 }
6496 } break;
6497 case GGML_OP_DIV: {
6498 if (src0_needs_grads) {
6499 ggml_add_or_set(ctx, cgraph, isrc0, ggml_div(ctx, grad, src1));
6500 }
6501 if (src1_needs_grads) {
6502 ggml_sub_or_set(ctx, cgraph, isrc1, ggml_mul(ctx, grad, ggml_div(ctx, tensor, src1)));
6503 }
6504 } break;
6505 case GGML_OP_SQR: {
6506 if (src0_needs_grads) {
6507 ggml_add_or_set(ctx, cgraph, isrc0, ggml_scale(ctx, ggml_mul(ctx, src0, grad), 2.0f));
6508 }
6509 } break;
6510 case GGML_OP_SQRT: {
6511 if (src0_needs_grads) {
6512 ggml_add_or_set(ctx, cgraph, isrc0, ggml_scale(ctx, ggml_div(ctx, grad, tensor), 0.5f));
6513 }
6514 } break;
6515 case GGML_OP_LOG: {
6516 if (src0_needs_grads) {
6517 ggml_add_or_set(ctx, cgraph, isrc0, ggml_div(ctx, grad, src0));
6518 }
6519 } break;
6520 case GGML_OP_SIN: {
6521 if (src0_needs_grads) {
6522 ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, grad, ggml_cos(ctx, src0)));
6523 }
6524 } break;
6525 case GGML_OP_COS: {
6526 if (src0_needs_grads) {
6527 ggml_sub_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, grad, ggml_sin(ctx, src0)));
6528 }
6529 } break;
6530 case GGML_OP_SUM: {
6531 if (src0_needs_grads) {
6532 ggml_add1_or_set(ctx, cgraph, isrc0, grad);
6533 }
6534 } break;
6535 case GGML_OP_SUM_ROWS: {
6536 if (src0_needs_grads) {
6537 ggml_add_or_set(ctx, cgraph, isrc0, ggml_repeat(ctx, grad, src0));
6538 }
6539 } break;
6540 case GGML_OP_MEAN: {
6541 if (src0_needs_grads) {
6542 ggml_add1_or_set(ctx, cgraph, isrc0, ggml_scale_impl(ctx, grad, 1.0f/src0->ne[0], 0.0, false0));
6543 }
6544 } break;
6545 case GGML_OP_REPEAT: {
6546 if (src0_needs_grads) {
6547 ggml_add_or_set(ctx, cgraph, isrc0, ggml_repeat_back(ctx, grad, src0));
6548 }
6549 } break;
6550 case GGML_OP_REPEAT_BACK: {
6551 if (src0_needs_grads) {
6552 ggml_add_or_set(ctx, cgraph, isrc0, ggml_repeat(ctx, grad, src0));
6553 }
6554 } break;
6555 case GGML_OP_RMS_NORM: {
6556 if (src0_needs_grads) {
6557 float eps;
6558 memcpy(&eps, tensor->op_params, sizeof(float));
6559 ggml_add_or_set(ctx, cgraph, isrc0, ggml_rms_norm_back(ctx, grad, src0, eps));
6560 }
6561 } break;
6562 case GGML_OP_MUL_MAT: {
6563 // https://cs231n.github.io/optimization-2/#staged
6564 // # forward pass
6565 // s0 = np.random.randn(5, 10)
6566 // s1 = np.random.randn(10, 3)
6567 // t = s0.dot(s1)
6568
6569 // # now suppose we had the gradient on t from above in the circuit
6570 // dt = np.random.randn(*t.shape) # same shape as t
6571 // ds0 = dt.dot(s1.T) #.T gives the transpose of the matrix
6572 // ds1 = t.T.dot(dt)
6573
6574 // tensor.shape [m,p,qq,rr]
6575 // src0.shape [n,m,q1,r1]
6576 // src1.shape [n,p,qq,rr]
6577
6578 if (src0_needs_grads
12.1
'src0_needs_grads' is false
) {
13
←
Taking false branch
→
6579 GGML_ASSERT(grad->ne[2] == src1->ne[2])if (!(grad->ne[2] == src1->ne[2])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6579, "GGML_ASSERT(%s) failed", "grad->ne[2] == src1->ne[2]"
);
6580 GGML_ASSERT(grad->ne[3] == src1->ne[3])if (!(grad->ne[3] == src1->ne[3])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6580, "GGML_ASSERT(%s) failed", "grad->ne[3] == src1->ne[3]"
);
6581 struct ggml_tensor * tmp =
6582 ggml_out_prod(ctx, // [n,m,qq,rr]
6583 src1, // [n,p,qq,rr]
6584 grad); // [m,p,qq,rr]
6585 if (!ggml_are_same_shape(tmp, src0)) {
6586 GGML_ASSERT(tmp->ne[0] == src0->ne[0])if (!(tmp->ne[0] == src0->ne[0])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6586, "GGML_ASSERT(%s) failed", "tmp->ne[0] == src0->ne[0]"
);
6587 GGML_ASSERT(tmp->ne[1] == src0->ne[1])if (!(tmp->ne[1] == src0->ne[1])) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6587, "GGML_ASSERT(%s) failed", "tmp->ne[1] == src0->ne[1]"
);
6588 GGML_ASSERT(tmp->ne[3] == 1)if (!(tmp->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6588, "GGML_ASSERT(%s) failed", "tmp->ne[3] == 1");
6589
6590 const int64_t nr2 = tmp->ne[2] / src0->ne[2];
6591 const size_t nb2 = tmp->nb[2] * nr2;
6592 const size_t nb3 = tmp->nb[2];
6593
6594 tmp = ggml_view_4d(ctx, tmp, src0->ne[0], src0->ne[1], src0->ne[2], nr2, tmp->nb[1], nb2, nb3, 0);
6595 tmp = ggml_repeat_back(ctx, tmp, src0);
6596 }
6597 ggml_add_or_set(ctx, cgraph, isrc0, tmp);
6598 }
6599 if (src1_needs_grads) {
14
←
Assuming 'src1_needs_grads' is true
→
15
←
Taking true branch
→
6600 ggml_add_or_set(ctx, cgraph, isrc1,
6601 // ggml_mul_mat(ctx, // [n,p,qq,rr]
6602 // ggml_cont(ctx, // [m,n,q1,r1]
6603 // ggml_transpose(ctx, src0)), // [m,n,q1,r1]
6604 // grad), // [m,p,qq,rr]
6605
6606 // when src0 is bigger than tensor->grad (this is mostly the case in llama),
6607 // avoid transpose of src0, rather transpose smaller tensor->grad
6608 // and then use ggml_out_prod
6609 ggml_out_prod(ctx, // [n,p,qq,rr]
17
←
Calling 'ggml_out_prod'
→
6610 src0, // [n,m,q1,r1]
16
←
Passing null pointer value via 2nd parameter 'a'
→
6611 ggml_transpose(ctx, // [p,m,qq,rr]
6612 grad))); // [m,p,qq,rr]
6613 }
6614 } break;
6615 case GGML_OP_SCALE: {
6616 if (src0_needs_grads) {
6617 float s;
6618 memcpy(&s, tensor->op_params, sizeof(float));
6619 ggml_add_or_set(ctx, cgraph, isrc0, ggml_scale_impl(ctx, grad, s, 0.0, false0));
6620 }
6621 } break;
6622 case GGML_OP_SET: {
6623 const size_t nb1 = ((const int32_t *) tensor->op_params)[0];
6624 const size_t nb2 = ((const int32_t *) tensor->op_params)[1];
6625 const size_t nb3 = ((const int32_t *) tensor->op_params)[2];
6626 const size_t offset = ((const int32_t *) tensor->op_params)[3];
6627
6628 struct ggml_tensor * tensor_grad_view = NULL((void*)0);
6629
6630 if (src0_needs_grads || src1_needs_grads) {
6631 GGML_ASSERT(src0->type == tensor->type)if (!(src0->type == tensor->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6631, "GGML_ASSERT(%s) failed", "src0->type == tensor->type"
);
6632 GGML_ASSERT(!cgraph->grads[isrc0] || cgraph->grads[isrc0]->type == grad->type)if (!(!cgraph->grads[isrc0] || cgraph->grads[isrc0]->
type == grad->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6632, "GGML_ASSERT(%s) failed", "!cgraph->grads[isrc0] || cgraph->grads[isrc0]->type == grad->type"
);
6633 GGML_ASSERT(!cgraph->grads[isrc1] || !src1_needs_grads || cgraph->grads[isrc1]->type == grad->type)if (!(!cgraph->grads[isrc1] || !src1_needs_grads || cgraph
->grads[isrc1]->type == grad->type)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6633, "GGML_ASSERT(%s) failed", "!cgraph->grads[isrc1] || !src1_needs_grads || cgraph->grads[isrc1]->type == grad->type"
);
6634
6635 tensor_grad_view = ggml_view_4d(ctx,
6636 grad, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],
6637 nb1, nb2, nb3, offset);
6638 }
6639
6640 if (src0_needs_grads) {
6641 struct ggml_tensor * tmp = ggml_neg(ctx, tensor_grad_view);
6642 ggml_add_or_set(ctx, cgraph, isrc0, ggml_acc_impl(ctx, grad, tmp, nb1, nb2, nb3, offset, false0));
6643 }
6644
6645 if (src1_needs_grads) {
6646 ggml_add_or_set(ctx, cgraph, isrc1, ggml_reshape(ctx, ggml_cont(ctx, tensor_grad_view), src1));
6647 }
6648 } break;
6649 case GGML_OP_CPY: {
6650 // cpy overwrites value of src1 by src0 and returns view(src1)
6651 // the overwriting is mathematically equivalent to:
6652 // tensor = src0 * 1 + src1 * 0
6653 if (src0_needs_grads) {
6654 // dsrc0 = dtensor * 1
6655 ggml_add_or_set(ctx, cgraph, isrc0, ggml_reshape(ctx, grad, src0));
6656 }
6657 if (src1_needs_grads) {
6658 // dsrc1 = dtensor * 0 -> noop
6659 }
6660 } break;
6661 case GGML_OP_CONT: {
6662 // same as cpy
6663 if (src0_needs_grads) {
6664 GGML_ASSERT(!cgraph->grads[isrc0] || ggml_is_contiguous(cgraph->grads[isrc0]))if (!(!cgraph->grads[isrc0] || ggml_is_contiguous(cgraph->
grads[isrc0]))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6664, "GGML_ASSERT(%s) failed", "!cgraph->grads[isrc0] || ggml_is_contiguous(cgraph->grads[isrc0])"
);
6665 GGML_ASSERT(ggml_is_contiguous(grad))if (!(ggml_is_contiguous(grad))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6665, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(grad)");
6666 GGML_ASSERT(ggml_nelements(tensor) == ggml_nelements(src0))if (!(ggml_nelements(tensor) == ggml_nelements(src0))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6666, "GGML_ASSERT(%s) failed", "ggml_nelements(tensor) == ggml_nelements(src0)"
);
6667 ggml_add_or_set(ctx, cgraph, isrc0,
6668 ggml_are_same_shape(tensor, src0) ? grad : ggml_reshape(ctx, grad, src0));
6669 }
6670 } break;
6671 case GGML_OP_RESHAPE: {
6672 if (src0_needs_grads) {
6673 struct ggml_tensor * grad_cont = ggml_is_contiguous(grad) ? grad : ggml_cont(ctx, grad);
6674 ggml_add_or_set(ctx, cgraph, isrc0, ggml_reshape(ctx, grad_cont, src0));
6675 }
6676 } break;
6677 case GGML_OP_VIEW: {
6678 if (src0_needs_grads) {
6679 size_t offset;
6680
6681 memcpy(&offset, tensor->op_params, sizeof(offset));
6682
6683 size_t nb1 = tensor->nb[1];
6684 size_t nb2 = tensor->nb[2];
6685 size_t nb3 = tensor->nb[3];
6686
6687 if (cgraph->grads[isrc0] && src0->type != cgraph->grads[isrc0]->type) {
6688 // gradient is typically F32, but src0 could be other type
6689 size_t ng = ggml_element_size(cgraph->grads[isrc0]);
6690 size_t n0 = ggml_element_size(src0);
6691 GGML_ASSERT(offset % n0 == 0)if (!(offset % n0 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6691, "GGML_ASSERT(%s) failed", "offset % n0 == 0");
6692 GGML_ASSERT(nb1 % n0 == 0)if (!(nb1 % n0 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6692, "GGML_ASSERT(%s) failed", "nb1 % n0 == 0");
6693 GGML_ASSERT(nb2 % n0 == 0)if (!(nb2 % n0 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6693, "GGML_ASSERT(%s) failed", "nb2 % n0 == 0");
6694 GGML_ASSERT(nb3 % n0 == 0)if (!(nb3 % n0 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6694, "GGML_ASSERT(%s) failed", "nb3 % n0 == 0");
6695 offset = (offset / n0) * ng;
6696 nb1 = (nb1 / n0) * ng;
6697 nb2 = (nb2 / n0) * ng;
6698 nb3 = (nb3 / n0) * ng;
6699 }
6700
6701 ggml_acc_or_set(ctx, cgraph, isrc0, grad, nb1, nb2, nb3, offset);
6702 }
6703 } break;
6704 case GGML_OP_PERMUTE: {
6705 if (src0_needs_grads) {
6706 const int32_t * axes = (const int32_t *) tensor->op_params;
6707 const int axis0 = axes[0] & 0x3;
6708 const int axis1 = axes[1] & 0x3;
6709 const int axis2 = axes[2] & 0x3;
6710 const int axis3 = axes[3] & 0x3;
6711 int axb[4] = {0,0,0,0}; // axes backward
6712 axb[axis0] = 0;
6713 axb[axis1] = 1;
6714 axb[axis2] = 2;
6715 axb[axis3] = 3;
6716 ggml_add_or_set(ctx, cgraph, isrc0, ggml_permute(ctx, grad, axb[0], axb[1], axb[2], axb[3]));
6717 }
6718 } break;
6719 case GGML_OP_TRANSPOSE: {
6720 if (src0_needs_grads) {
6721 ggml_add_or_set(ctx, cgraph, isrc0, ggml_transpose(ctx, grad));
6722 }
6723 } break;
6724 case GGML_OP_GET_ROWS: {
6725 if (src0_needs_grads) {
6726 ggml_add_or_set(ctx, cgraph, isrc0, ggml_get_rows_back(ctx, grad, src1, src0));
6727 }
6728 if (src1_needs_grads) {
6729 // noop
6730 }
6731 } break;
6732 case GGML_OP_DIAG_MASK_INF: {
6733 if (src0_needs_grads) {
6734 /* ggml_diag_mask_inf_impl() shouldn't be here */
6735 /* ref: https://github.com/ggml-org/llama.cpp/pull/4203#discussion_r1412377992 */
6736 const int n_past = ((const int32_t *) tensor->op_params)[0];
6737 ggml_add_or_set(ctx, cgraph, isrc0, ggml_diag_mask_zero_impl(ctx, grad, n_past, false0));
6738 }
6739 } break;
6740 case GGML_OP_DIAG_MASK_ZERO: {
6741 if (src0_needs_grads) {
6742 const int n_past = ((const int32_t *) tensor->op_params)[0];
6743 ggml_add_or_set(ctx, cgraph, isrc0, ggml_diag_mask_zero_impl(ctx, grad, n_past, false0));
6744 }
6745 } break;
6746 case GGML_OP_SOFT_MAX: {
6747 if (src0_needs_grads) {
6748 float scale = 1.0f;
6749 float max_bias = 0.0f;
6750
6751 memcpy(&scale, (const float *) tensor->op_params + 0, sizeof(float));
6752 memcpy(&max_bias, (const float *) tensor->op_params + 1, sizeof(float));
6753
6754 ggml_add_or_set(ctx, cgraph, isrc0, ggml_soft_max_ext_back(ctx, grad, tensor, scale, max_bias));
6755 }
6756 GGML_ASSERT((!src1 || !src1_needs_grads) && "backward pass for softmax mask not implemented")if (!((!src1 || !src1_needs_grads) && "backward pass for softmax mask not implemented"
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6756, "GGML_ASSERT(%s) failed", "(!src1 || !src1_needs_grads) && \"backward pass for softmax mask not implemented\""
);
6757 } break;
6758 case GGML_OP_ROPE: {
6759 if (src0_needs_grads) {
6760 //const int n_past = ((int32_t *) tensor->op_params)[0];
6761 const int n_dims = ((const int32_t *) tensor->op_params)[1];
6762 const int mode = ((const int32_t *) tensor->op_params)[2];
6763 //const int n_ctx = ((int32_t *) tensor->op_params)[3];
6764 const int n_ctx_orig = ((const int32_t *) tensor->op_params)[4];
6765 float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
6766 int sections[4] = {0, 0, 0, 0};
6767
6768 memcpy(&freq_base, (const float *) tensor->op_params + 5, sizeof(float));
6769 memcpy(&freq_scale, (const float *) tensor->op_params + 6, sizeof(float));
6770 memcpy(&ext_factor, (const float *) tensor->op_params + 7, sizeof(float));
6771 memcpy(&attn_factor, (const float *) tensor->op_params + 8, sizeof(float));
6772 memcpy(&beta_fast, (const float *) tensor->op_params + 9, sizeof(float));
6773 memcpy(&beta_slow, (const float *) tensor->op_params + 10, sizeof(float));
6774 memcpy(&sections, tensor->op_params + 11, sizeof(sections));
6775
6776 struct ggml_tensor * rope_back = grad->ne[2] == src1->ne[0] ?
6777 ggml_rope_ext_back(ctx, grad, src1, src2, n_dims,
6778 mode, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow) :
6779 ggml_rope_multi_back(ctx, grad, src1, src2, n_dims, sections,
6780 mode, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
6781 ggml_add_or_set(ctx, cgraph, isrc0, rope_back);
6782 }
6783 GGML_ASSERT((!src2 || !src2_needs_grads) && "gradients for freq factors not implemented")if (!((!src2 || !src2_needs_grads) && "gradients for freq factors not implemented"
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6783, "GGML_ASSERT(%s) failed", "(!src2 || !src2_needs_grads) && \"gradients for freq factors not implemented\""
);
6784 } break;
6785 case GGML_OP_IM2COL: {
6786 if (src1_needs_grads) {
6787 const int32_t s0 = ggml_get_op_params_i32(tensor, 0);
6788 const int32_t s1 = ggml_get_op_params_i32(tensor, 1);
6789 const int32_t p0 = ggml_get_op_params_i32(tensor, 2);
6790 const int32_t p1 = ggml_get_op_params_i32(tensor, 3);
6791 const int32_t d0 = ggml_get_op_params_i32(tensor, 4);
6792 const int32_t d1 = ggml_get_op_params_i32(tensor, 5);
6793 const bool_Bool is_2D = ggml_get_op_params_i32(tensor, 6) == 1;
6794
6795 ggml_add_or_set(ctx, cgraph, isrc1, ggml_im2col_back(ctx, grad, src0, src1->ne, s0, s1, p0, p1, d0, d1, is_2D));
6796 }
6797 } break;
6798 case GGML_OP_POOL_2D: {
6799 if (src0_needs_grads) {
6800 const enum ggml_op_pool op = ggml_get_op_params_i32(tensor, 0);
6801 const int32_t k0 = ggml_get_op_params_i32(tensor, 1);
6802 const int32_t k1 = ggml_get_op_params_i32(tensor, 2);
6803 const int32_t s0 = ggml_get_op_params_i32(tensor, 3);
6804 const int32_t s1 = ggml_get_op_params_i32(tensor, 4);
6805 const int32_t p0 = ggml_get_op_params_i32(tensor, 5);
6806 const int32_t p1 = ggml_get_op_params_i32(tensor, 6);
6807
6808 ggml_add_or_set(ctx, cgraph, isrc0, ggml_pool_2d_back(ctx, grad, src0, op, k0, k1, s0, s1, p0, p1));
6809 }
6810 } break;
6811 case GGML_OP_WIN_PART:
6812 case GGML_OP_WIN_UNPART:
6813 case GGML_OP_UNARY: {
6814 switch (ggml_get_unary_op(tensor)) {
6815 case GGML_UNARY_OP_ABS: {
6816 if (src0_needs_grads) {
6817 ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, ggml_sgn(ctx, src0), grad));
6818 }
6819 } break;
6820 case GGML_UNARY_OP_SGN: {
6821 // noop
6822 } break;
6823 case GGML_UNARY_OP_NEG: {
6824 if (src0_needs_grads) {
6825 ggml_sub_or_set(ctx, cgraph, isrc0, grad);
6826 }
6827 } break;
6828 case GGML_UNARY_OP_STEP: {
6829 // noop
6830 } break;
6831 case GGML_UNARY_OP_RELU: {
6832 if (src0_needs_grads) {
6833 ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, ggml_step(ctx, src0), grad));
6834 }
6835 } break;
6836 case GGML_UNARY_OP_SILU: {
6837 if (src0_needs_grads) {
6838 ggml_add_or_set(ctx, cgraph, isrc0, ggml_silu_back(ctx, grad, src0));
6839 }
6840 } break;
6841 case GGML_UNARY_OP_EXP: {
6842 if (src0_needs_grads) {
6843 ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, tensor, grad));
6844 }
6845 } break;
6846 case GGML_UNARY_OP_EXPM1: {
6847 if (src0_needs_grads) {
6848 ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, grad, ggml_exp(ctx, src0)));
6849 }
6850 } break;
6851 case GGML_UNARY_OP_SOFTPLUS: {
6852 if (src0_needs_grads) {
6853 ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, grad, ggml_sigmoid(ctx, src0)));
6854 }
6855 } break;
6856 default: {
6857 fprintf(stderrstderr, "%s: unsupported unary op for backward pass: %s\n",
6858 __func__, ggml_unary_op_name(ggml_get_unary_op(tensor)));
6859 GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6859, "fatal error");
6860 } //break;
6861 }
6862 } break;
6863 case GGML_OP_CROSS_ENTROPY_LOSS: {
6864 if (src0_needs_grads) {
6865 ggml_add_or_set(ctx, cgraph, isrc0, ggml_cross_entropy_loss_back(ctx, grad, src0, src1));
6866 }
6867 GGML_ASSERT(!src1_needs_grads && "backward pass for labels not implemented")if (!(!src1_needs_grads && "backward pass for labels not implemented"
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6867, "GGML_ASSERT(%s) failed", "!src1_needs_grads && \"backward pass for labels not implemented\""
);
6868 } break;
6869 case GGML_OP_GLU: {
6870 switch (ggml_get_glu_op(tensor)) {
6871 case GGML_GLU_OP_SWIGLU: {
6872 if (src0_needs_grads) {
6873 GGML_ASSERT(src1 && "backward pass only implemented for split swiglu")if (!(src1 && "backward pass only implemented for split swiglu"
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6873, "GGML_ASSERT(%s) failed", "src1 && \"backward pass only implemented for split swiglu\""
);
6874 ggml_add_or_set(ctx, cgraph, isrc0, ggml_silu_back(ctx, ggml_mul(ctx, grad, src1), src0));
6875 }
6876 if (src1_needs_grads) {
6877 ggml_add_or_set(ctx, cgraph, isrc1, ggml_mul(ctx, ggml_silu(ctx, src0), grad));
6878 }
6879 } break;
6880 default: {
6881 GGML_ABORT("unsupported glu op for backward pass: %s", ggml_glu_op_name(ggml_get_glu_op(tensor)))ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6881, "unsupported glu op for backward pass: %s", ggml_glu_op_name
(ggml_get_glu_op(tensor)));
6882 } //break;
6883 }
6884 } break;
6885 case GGML_OP_NONE: {
6886 // noop
6887 } break;
6888 case GGML_OP_COUNT:
6889 default: {
6890 GGML_ABORT("%s: unsupported ggml op for backward pass: %s\n", __func__, ggml_op_name(tensor->op))ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6890, "%s: unsupported ggml op for backward pass: %s\n", __func__
, ggml_op_name(tensor->op));
6891 } //break;
6892 }
6893
6894 GGML_ASSERT(!src0_needs_grads || ggml_are_same_shape(src0, cgraph->grads[isrc0]))if (!(!src0_needs_grads || ggml_are_same_shape(src0, cgraph->
grads[isrc0]))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6894, "GGML_ASSERT(%s) failed", "!src0_needs_grads || ggml_are_same_shape(src0, cgraph->grads[isrc0])"
);
6895 GGML_ASSERT(!src1_needs_grads || ggml_are_same_shape(src1, cgraph->grads[isrc1]))if (!(!src1_needs_grads || ggml_are_same_shape(src1, cgraph->
grads[isrc1]))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6895, "GGML_ASSERT(%s) failed", "!src1_needs_grads || ggml_are_same_shape(src1, cgraph->grads[isrc1])"
);
6896 GGML_ASSERT(!src2_needs_grads || ggml_are_same_shape(src2, cgraph->grads[isrc2]))if (!(!src2_needs_grads || ggml_are_same_shape(src2, cgraph->
grads[isrc2]))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6896, "GGML_ASSERT(%s) failed", "!src2_needs_grads || ggml_are_same_shape(src2, cgraph->grads[isrc2])"
);
6897}
6898
6899static size_t ggml_visit_parents_graph(struct ggml_cgraph * cgraph, struct ggml_tensor * node, bool_Bool compute) {
6900 if (node->op != GGML_OP_NONE && compute) {
6901 node->flags |= GGML_TENSOR_FLAG_COMPUTE;
6902 }
6903
6904 const size_t node_hash_pos = ggml_hash_find(&cgraph->visited_hash_set, node);
6905 GGML_ASSERT(node_hash_pos != GGML_HASHSET_FULL)if (!(node_hash_pos != ((size_t)-1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6905, "GGML_ASSERT(%s) failed", "node_hash_pos != GGML_HASHSET_FULL"
);
6906
6907 if (ggml_bitset_get(cgraph->visited_hash_set.used, node_hash_pos)) {
6908 // already visited
6909
6910 if (compute) {
6911 // update the compute flag regardless
6912 for (int i = 0; i < GGML_MAX_SRC10; ++i) {
6913 struct ggml_tensor * src = node->src[i];
6914 if (src && ((src->flags & GGML_TENSOR_FLAG_COMPUTE) == 0)) {
6915 ggml_visit_parents_graph(cgraph, src, true1);
6916 }
6917 }
6918 }
6919
6920 return node_hash_pos;
6921 }
6922
6923 // This is the first time we see this node in the current graph.
6924 cgraph->visited_hash_set.keys[node_hash_pos] = node;
6925 ggml_bitset_set(cgraph->visited_hash_set.used, node_hash_pos);
6926 cgraph->use_counts[node_hash_pos] = 0;
6927
6928 for (int i = 0; i < GGML_MAX_SRC10; ++i) {
6929 const int k =
6930 (cgraph->order == GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT) ? i :
6931 (cgraph->order == GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT) ? (GGML_MAX_SRC10-1-i) :
6932 /* unknown order, just fall back to using i */ i;
6933
6934 struct ggml_tensor * src = node->src[k];
6935 if (src) {
6936 const size_t src_hash_pos = ggml_visit_parents_graph(cgraph, src, compute);
6937
6938 // Update the use count for this operand.
6939 cgraph->use_counts[src_hash_pos]++;
6940 }
6941 }
6942
6943 if (node->op == GGML_OP_NONE && !(node->flags & GGML_TENSOR_FLAG_PARAM)) {
6944 // reached a leaf node, not part of the gradient graph (e.g. a constant)
6945 GGML_ASSERT(cgraph->n_leafs < cgraph->size)if (!(cgraph->n_leafs < cgraph->size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6945, "GGML_ASSERT(%s) failed", "cgraph->n_leafs < cgraph->size"
);
6946
6947 if (strlen(node->name) == 0) {
6948 ggml_format_name(node, "leaf_%d", cgraph->n_leafs);
6949 }
6950
6951 cgraph->leafs[cgraph->n_leafs] = node;
6952 cgraph->n_leafs++;
6953 } else {
6954 GGML_ASSERT(cgraph->n_nodes < cgraph->size)if (!(cgraph->n_nodes < cgraph->size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6954, "GGML_ASSERT(%s) failed", "cgraph->n_nodes < cgraph->size"
);
6955
6956 if (strlen(node->name) == 0) {
6957 ggml_format_name(node, "node_%d", cgraph->n_nodes);
6958 }
6959
6960 cgraph->nodes[cgraph->n_nodes] = node;
6961 cgraph->n_nodes++;
6962 }
6963
6964 return node_hash_pos;
6965}
6966
6967static void ggml_build_forward_impl(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor, bool_Bool expand, bool_Bool compute) {
6968 if (!expand) {
6969 // TODO: this branch isn't accessible anymore, maybe move this to ggml_build_forward_expand
6970 ggml_graph_clear(cgraph);
6971 }
6972
6973 const int n_old = cgraph->n_nodes;
6974
6975 ggml_visit_parents_graph(cgraph, tensor, compute);
6976
6977 const int n_new = cgraph->n_nodes - n_old;
6978 GGML_PRINT_DEBUG("%s: visited %d new nodes\n", __func__, n_new);
6979
6980 if (n_new > 0) {
6981 // the last added node should always be starting point
6982 GGML_ASSERT(cgraph->nodes[cgraph->n_nodes - 1] == tensor)if (!(cgraph->nodes[cgraph->n_nodes - 1] == tensor)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6982, "GGML_ASSERT(%s) failed", "cgraph->nodes[cgraph->n_nodes - 1] == tensor"
);
6983 }
6984}
6985
6986struct ggml_tensor * ggml_build_forward_select(
6987 struct ggml_cgraph * cgraph,
6988 struct ggml_tensor ** tensors,
6989 int n_tensors,
6990 int idx) {
6991 GGML_ASSERT(idx >= 0 && idx < n_tensors)if (!(idx >= 0 && idx < n_tensors)) ggml_abort(
"/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 6991, "GGML_ASSERT(%s) failed", "idx >= 0 && idx < n_tensors"
);
6992
6993 for (int i = 0; i < n_tensors; i++) {
6994 ggml_build_forward_impl(cgraph, tensors[i], true1, i == idx ? true1 : false0);
6995 }
6996
6997 return tensors[idx];
6998}
6999
7000void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor) {
7001 ggml_build_forward_impl(cgraph, tensor, true1, true1);
7002}
7003
7004void ggml_build_backward_expand(
7005 struct ggml_context * ctx,
7006 struct ggml_cgraph * cgraph,
7007 struct ggml_tensor ** grad_accs) {
7008 GGML_ASSERT(cgraph->n_nodes > 0)if (!(cgraph->n_nodes > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7008, "GGML_ASSERT(%s) failed", "cgraph->n_nodes > 0"
);
7009 GGML_ASSERT(cgraph->grads)if (!(cgraph->grads)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7009, "GGML_ASSERT(%s) failed", "cgraph->grads");
7010 GGML_ASSERT(cgraph->grad_accs)if (!(cgraph->grad_accs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7010, "GGML_ASSERT(%s) failed", "cgraph->grad_accs");
7011
7012 const int n_nodes_f = cgraph->n_nodes;
7013
7014 memset(cgraph->grads, 0, cgraph->visited_hash_set.size*sizeof(struct ggml_tensor *));
7015 memset(cgraph->grad_accs, 0, cgraph->visited_hash_set.size*sizeof(struct ggml_tensor *));
7016 bool_Bool * grads_needed = calloc(cgraph->visited_hash_set.size, sizeof(bool_Bool));
7017
7018 {
7019 bool_Bool any_params = false0;
7020 bool_Bool any_loss = false0;
7021 for (int i = 0; i < n_nodes_f; ++i) {
7022 struct ggml_tensor * node = cgraph->nodes[i];
7023 any_params = any_params || (node->flags & GGML_TENSOR_FLAG_PARAM);
7024 any_loss = any_loss || (node->flags & GGML_TENSOR_FLAG_LOSS);
7025 }
7026 GGML_ASSERT(any_params && "no trainable parameters found, did you forget to call ggml_set_param?")if (!(any_params && "no trainable parameters found, did you forget to call ggml_set_param?"
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7026, "GGML_ASSERT(%s) failed", "any_params && \"no trainable parameters found, did you forget to call ggml_set_param?\""
);
7027 GGML_ASSERT(any_loss && "no training loss found, did you forget to call ggml_set_loss?")if (!(any_loss && "no training loss found, did you forget to call ggml_set_loss?"
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7027, "GGML_ASSERT(%s) failed", "any_loss && \"no training loss found, did you forget to call ggml_set_loss?\""
);
7028 }
7029
7030 for (int i = 0; i < n_nodes_f; ++i) {
7031 struct ggml_tensor * node = cgraph->nodes[i];
7032
7033 if (node->type == GGML_TYPE_I32) {
7034 continue;
7035 }
7036
7037 bool_Bool node_needs_grad = (node->flags & GGML_TENSOR_FLAG_PARAM) || (node->flags & GGML_TENSOR_FLAG_LOSS);
7038 bool_Bool ignore_src[GGML_MAX_SRC10] = {false0};
7039 switch (node->op) {
7040 // gradients in node->src[0] for one reason or another have no effect on output gradients
7041 case GGML_OP_IM2COL: // only used for its shape
7042 case GGML_OP_IM2COL_BACK: // same as IM2COL
7043 ignore_src[0] = true1;
7044 break;
7045 case GGML_OP_UNARY: {
7046 const enum ggml_unary_op uop = ggml_get_unary_op(node);
7047 // SGN and STEP unary ops are piecewise constant
7048 if (uop == GGML_UNARY_OP_SGN || uop == GGML_UNARY_OP_STEP) {
7049 ignore_src[0] = true1;
7050 }
7051 } break;
7052
7053 // gradients in node->src[1] for one reason or another have no effect on output gradients
7054 case GGML_OP_CPY: // gradients in CPY target are irrelevant
7055 case GGML_OP_GET_ROWS: // row indices not differentiable
7056 case GGML_OP_GET_ROWS_BACK: // same as for GET_ROWS
7057 case GGML_OP_ROPE: // positions not differentiable
7058 ignore_src[1] = true1;
7059 break;
7060
7061 default:
7062 break;
7063 }
7064 for (int j = 0; j < GGML_MAX_SRC10; ++j) {
7065 if (!node->src[j] || ignore_src[j] || !grads_needed[ggml_hash_find(&cgraph->visited_hash_set, node->src[j])]) {
7066 continue;
7067 }
7068 GGML_ASSERT(node->src[j]->type == GGML_TYPE_F32 || node->src[j]->type == GGML_TYPE_F16)if (!(node->src[j]->type == GGML_TYPE_F32 || node->src
[j]->type == GGML_TYPE_F16)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7068, "GGML_ASSERT(%s) failed", "node->src[j]->type == GGML_TYPE_F32 || node->src[j]->type == GGML_TYPE_F16"
);
7069 node_needs_grad = true1;
7070 break;
7071 }
7072 if (!node_needs_grad) {
7073 continue;
7074 }
7075
7076 // inplace operations are currently not supported
7077 GGML_ASSERT(!node->view_src || node->op == GGML_OP_CPY || node->op == GGML_OP_VIEW ||if (!(!node->view_src || node->op == GGML_OP_CPY || node
->op == GGML_OP_VIEW || node->op == GGML_OP_RESHAPE || node
->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7078, "GGML_ASSERT(%s) failed", "!node->view_src || node->op == GGML_OP_CPY || node->op == GGML_OP_VIEW || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE"
)
7078 node->op == GGML_OP_RESHAPE || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE)if (!(!node->view_src || node->op == GGML_OP_CPY || node
->op == GGML_OP_VIEW || node->op == GGML_OP_RESHAPE || node
->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE
)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7078, "GGML_ASSERT(%s) failed", "!node->view_src || node->op == GGML_OP_CPY || node->op == GGML_OP_VIEW || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE"
);
7079
7080 const size_t ihash = ggml_hash_find(&cgraph->visited_hash_set, node);
7081 GGML_ASSERT(ihash != GGML_HASHSET_FULL)if (!(ihash != ((size_t)-1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7081, "GGML_ASSERT(%s) failed", "ihash != GGML_HASHSET_FULL"
);
7082 GGML_ASSERT(ggml_bitset_get(cgraph->visited_hash_set.used, ihash))if (!(ggml_bitset_get(cgraph->visited_hash_set.used, ihash
))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7082, "GGML_ASSERT(%s) failed", "ggml_bitset_get(cgraph->visited_hash_set.used, ihash)"
);
7083 if (grad_accs && grad_accs[i]) {
7084 cgraph->grad_accs[ihash] = grad_accs[i];
7085 cgraph->grads[ihash] = cgraph->grad_accs[ihash];
7086 } else if (node->flags & GGML_TENSOR_FLAG_LOSS) {
7087 // loss tensors always need a gradient accumulator
7088 cgraph->grad_accs[ihash] = ggml_new_tensor(ctx, GGML_TYPE_F32, GGML_MAX_DIMS4, node->ne);
7089 cgraph->grads[ihash] = cgraph->grad_accs[ihash];
7090 }
7091 grads_needed[ihash] = true1;
7092 }
7093
7094 for (int i = n_nodes_f - 1; i >= 0; --i) {
7095 // inplace operations to add gradients are not created by ggml_compute_backward except for gradient accumulation
7096 // use allocator to automatically make inplace operations
7097 ggml_compute_backward(ctx, cgraph, i, grads_needed);
7098 }
7099
7100 free(grads_needed);
7101}
7102
7103static void * incr_ptr_aligned(void ** p, size_t size, size_t align) {
7104 void * ptr = *p;
7105 ptr = (void *) GGML_PAD((uintptr_t) ptr, align)((((uintptr_t) ptr) + (align) - 1) & ~((align) - 1));
7106 *p = (void *) ((char *) ptr + size);
7107 return ptr;
7108}
7109
7110static size_t ggml_graph_nbytes(size_t size, bool_Bool grads) {
7111 size_t hash_size = ggml_hash_size(size * 2);
7112 void * p = 0;
7113 incr_ptr_aligned(&p, sizeof(struct ggml_cgraph), 1);
7114 incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)); // nodes
7115 incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)); // leafs
7116 incr_ptr_aligned(&p, hash_size * sizeof(int32_t), sizeof(int32_t)); // use_counts
7117 incr_ptr_aligned(&p, hash_size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)); // hash keys
7118 if (grads) {
7119 incr_ptr_aligned(&p, hash_size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)); // grads
7120 incr_ptr_aligned(&p, hash_size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)); // grad_accs
7121 }
7122 incr_ptr_aligned(&p, ggml_bitset_size(hash_size) * sizeof(ggml_bitset_t), sizeof(ggml_bitset_t));
7123
7124 size_t nbytes = (size_t) p;
7125 return nbytes;
7126}
7127
7128size_t ggml_graph_overhead_custom(size_t size, bool_Bool grads) {
7129 return GGML_OBJECT_SIZE + GGML_PAD(ggml_graph_nbytes(size, grads), GGML_MEM_ALIGN)(((ggml_graph_nbytes(size, grads)) + (16) - 1) & ~((16) -
1));
7130}
7131
7132size_t ggml_graph_overhead(void) {
7133 return ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE2048, false0);
7134}
7135
7136struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool_Bool grads) {
7137 const size_t obj_size = ggml_graph_nbytes(size, grads);
7138 struct ggml_object * obj = ggml_new_object(ctx, GGML_OBJECT_TYPE_GRAPH, obj_size);
7139 struct ggml_cgraph * cgraph = (struct ggml_cgraph *) ((char *) ctx->mem_buffer + obj->offs);
7140
7141 // the size of the hash table is doubled since it needs to hold both nodes and leafs
7142 size_t hash_size = ggml_hash_size(size * 2);
7143
7144 void * p = cgraph + 1;
7145
7146 struct ggml_tensor ** nodes_ptr = incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *));
7147 struct ggml_tensor ** leafs_ptr = incr_ptr_aligned(&p, size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *));
7148 int32_t * use_counts_ptr = incr_ptr_aligned(&p, hash_size * sizeof(int32_t), sizeof(int32_t));
7149 struct ggml_tensor ** hash_keys_ptr = incr_ptr_aligned(&p, hash_size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *));
7150 struct ggml_tensor ** grads_ptr = grads ? incr_ptr_aligned(&p, hash_size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)) : NULL((void*)0);
7151 struct ggml_tensor ** grad_accs_ptr = grads ? incr_ptr_aligned(&p, hash_size * sizeof(struct ggml_tensor *), sizeof(struct ggml_tensor *)) : NULL((void*)0);
7152
7153 ggml_bitset_t * hash_used = incr_ptr_aligned(&p, ggml_bitset_size(hash_size) * sizeof(ggml_bitset_t), sizeof(ggml_bitset_t));
7154
7155 // check that we allocated the correct amount of memory
7156 assert(obj_size == (size_t)((char *)p - (char *)cgraph))((void) sizeof (__assert_single_arg (obj_size == (size_t)((char
*)p - (char *)cgraph))), __extension__ ({ if (obj_size == (size_t
)((char *)p - (char *)cgraph)) ; else __assert_fail ("obj_size == (size_t)((char *)p - (char *)cgraph)"
, "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7156, __extension__ __PRETTY_FUNCTION__); }));
7157
7158 *cgraph = (struct ggml_cgraph) {
7159 /*.size =*/ size,
7160 /*.n_nodes =*/ 0,
7161 /*.n_leafs =*/ 0,
7162 /*.nodes =*/ nodes_ptr,
7163 /*.grads =*/ grads_ptr,
7164 /*.grad_accs =*/ grad_accs_ptr,
7165 /*.leafs =*/ leafs_ptr,
7166 /*.use_counts =*/ use_counts_ptr,
7167 /*.hash_table =*/ { hash_size, hash_used, hash_keys_ptr },
7168 /*.order =*/ GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT,
7169 /*.uid =*/ 0,
7170 };
7171
7172 ggml_hash_set_reset(&cgraph->visited_hash_set);
7173 if (grads) {
7174 memset(cgraph->grads, 0, hash_size*sizeof(struct ggml_tensor *));
7175 memset(cgraph->grad_accs, 0, hash_size*sizeof(struct ggml_tensor *));
7176 }
7177
7178 return cgraph;
7179}
7180
7181struct ggml_cgraph * ggml_new_graph(struct ggml_context * ctx) {
7182 return ggml_new_graph_custom(ctx, GGML_DEFAULT_GRAPH_SIZE2048, false0);
7183}
7184
7185struct ggml_cgraph ggml_graph_view(struct ggml_cgraph * cgraph0, int i0, int i1) {
7186 struct ggml_cgraph cgraph = {
7187 /*.size =*/ 0,
7188 /*.n_nodes =*/ i1 - i0,
7189 /*.n_leafs =*/ 0,
7190 /*.nodes =*/ cgraph0->nodes + i0,
7191 /*.grads =*/ NULL((void*)0), // gradients would need visited_hash_set
7192 /*.grad_accs =*/ NULL((void*)0),
7193 /*.leafs =*/ NULL((void*)0),
7194 /*.use_counts =*/ cgraph0->use_counts,
7195 /*.visited_hash_set =*/ cgraph0->visited_hash_set,
7196 /*.order =*/ cgraph0->order,
7197 /*.uid =*/ 0
7198 };
7199
7200 return cgraph;
7201}
7202
7203void ggml_graph_cpy(struct ggml_cgraph * src, struct ggml_cgraph * dst) {
7204 GGML_ASSERT(dst->size >= src->n_leafs)if (!(dst->size >= src->n_leafs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7204, "GGML_ASSERT(%s) failed", "dst->size >= src->n_leafs"
);
7205 GGML_ASSERT(dst->size >= src->n_nodes)if (!(dst->size >= src->n_nodes)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7205, "GGML_ASSERT(%s) failed", "dst->size >= src->n_nodes"
);
7206 GGML_ASSERT(dst->visited_hash_set.size >= src->visited_hash_set.size)if (!(dst->visited_hash_set.size >= src->visited_hash_set
.size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7206, "GGML_ASSERT(%s) failed", "dst->visited_hash_set.size >= src->visited_hash_set.size"
);
7207
7208 dst->n_leafs = src->n_leafs;
7209 dst->n_nodes = src->n_nodes;
7210 dst->order = src->order;
7211
7212 for (int i = 0; i < src->n_leafs; ++i) {
7213 dst->leafs[i] = src->leafs[i];
7214 }
7215
7216 for (int i = 0; i < src->n_nodes; ++i) {
7217 dst->nodes[i] = src->nodes[i];
7218 }
7219
7220 for (size_t i = 0; i < src->visited_hash_set.size; ++i) {
7221 // copy all hashset keys (tensors) that are in use
7222 if (ggml_bitset_get(src->visited_hash_set.used, i)) {
7223 size_t new_hash_pos = ggml_hash_insert(&dst->visited_hash_set, src->visited_hash_set.keys[i]);
7224 dst->use_counts[new_hash_pos] = src->use_counts[i];
7225 }
7226 }
7227
7228 if (dst->grads) {
7229 memset(dst->grads, 0, dst->visited_hash_set.size*sizeof(struct ggml_tensor *));
7230 memset(dst->grad_accs, 0, dst->visited_hash_set.size*sizeof(struct ggml_tensor *));
7231 }
7232 if (src->grads) {
7233 GGML_ASSERT(dst->grads != NULL)if (!(dst->grads != ((void*)0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7233, "GGML_ASSERT(%s) failed", "dst->grads != NULL");
7234 GGML_ASSERT(dst->grad_accs != NULL)if (!(dst->grad_accs != ((void*)0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7234, "GGML_ASSERT(%s) failed", "dst->grad_accs != NULL"
);
7235 for (int i = 0; i < src->n_nodes; ++i) {
7236 const size_t igrad_src = ggml_hash_find(&src->visited_hash_set, src->nodes[i]);
7237 const size_t igrad_dst = ggml_hash_find(&dst->visited_hash_set, dst->nodes[i]);
7238
7239 GGML_ASSERT(igrad_src != GGML_HASHSET_FULL)if (!(igrad_src != ((size_t)-1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7239, "GGML_ASSERT(%s) failed", "igrad_src != GGML_HASHSET_FULL"
);
7240 GGML_ASSERT(ggml_bitset_get(src->visited_hash_set.used, igrad_src))if (!(ggml_bitset_get(src->visited_hash_set.used, igrad_src
))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7240, "GGML_ASSERT(%s) failed", "ggml_bitset_get(src->visited_hash_set.used, igrad_src)"
);
7241 GGML_ASSERT(igrad_dst != GGML_HASHSET_FULL)if (!(igrad_dst != ((size_t)-1))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7241, "GGML_ASSERT(%s) failed", "igrad_dst != GGML_HASHSET_FULL"
);
7242 GGML_ASSERT(ggml_bitset_get(dst->visited_hash_set.used, igrad_dst))if (!(ggml_bitset_get(dst->visited_hash_set.used, igrad_dst
))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7242, "GGML_ASSERT(%s) failed", "ggml_bitset_get(dst->visited_hash_set.used, igrad_dst)"
);
7243
7244 dst->grads[igrad_dst] = src->grads[igrad_src];
7245 dst->grad_accs[igrad_dst] = src->grad_accs[igrad_src];
7246 }
7247 }
7248}
7249
7250struct ggml_cgraph * ggml_graph_dup(struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool_Bool force_grads) {
7251 struct ggml_cgraph * result = ggml_new_graph_custom(ctx, cgraph->size, cgraph->grads || force_grads);
7252 ggml_graph_cpy(cgraph, result);
7253 return result;
7254}
7255
7256struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor) {
7257 if (ggml_is_empty(tensor)) {
7258 return tensor;
7259 }
7260 if (tensor->buffer) {
7261 ggml_backend_tensor_memset(tensor, 0, 0, ggml_nbytes(tensor));
7262 } else {
7263 GGML_ASSERT(tensor->data)if (!(tensor->data)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7263, "GGML_ASSERT(%s) failed", "tensor->data");
7264 memset(tensor->data, 0, ggml_nbytes(tensor));
7265 }
7266 return tensor;
7267}
7268
7269void ggml_graph_reset(struct ggml_cgraph * cgraph) {
7270 if (!cgraph) {
7271 return;
7272 }
7273 GGML_ASSERT(cgraph->grads != NULL)if (!(cgraph->grads != ((void*)0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7273, "GGML_ASSERT(%s) failed", "cgraph->grads != NULL");
7274
7275 for (int i = 0; i < cgraph->n_nodes; i++) {
7276 struct ggml_tensor * node = cgraph->nodes[i];
7277 struct ggml_tensor * grad_acc = ggml_graph_get_grad_acc(cgraph, node);
7278
7279 if (node->op == GGML_OP_OPT_STEP_ADAMW) {
7280 // clear momenta
7281 ggml_set_zero(node->src[2]);
7282 ggml_set_zero(node->src[3]);
7283 }
7284
7285 // initial gradients of loss should be 1, 0 otherwise
7286 if (grad_acc) {
7287 if (node->flags & GGML_TENSOR_FLAG_LOSS) {
7288 GGML_ASSERT(grad_acc->type == GGML_TYPE_F32)if (!(grad_acc->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7288, "GGML_ASSERT(%s) failed", "grad_acc->type == GGML_TYPE_F32"
);
7289 GGML_ASSERT(ggml_is_scalar(grad_acc))if (!(ggml_is_scalar(grad_acc))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7289, "GGML_ASSERT(%s) failed", "ggml_is_scalar(grad_acc)");
7290
7291 const float onef = 1.0f;
7292 if (grad_acc->buffer) {
7293 ggml_backend_tensor_set(grad_acc, &onef, 0, sizeof(float));
7294 } else {
7295 GGML_ASSERT(grad_acc->data)if (!(grad_acc->data)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7295, "GGML_ASSERT(%s) failed", "grad_acc->data");
7296 *((float *) grad_acc->data) = onef;
7297 }
7298 } else {
7299 ggml_set_zero(grad_acc);
7300 }
7301 }
7302 }
7303}
7304
7305void ggml_graph_clear(struct ggml_cgraph * cgraph) {
7306 cgraph->n_leafs = 0;
7307 cgraph->n_nodes = 0;
7308 ggml_hash_set_reset(&cgraph->visited_hash_set);
7309}
7310
7311int ggml_graph_size(struct ggml_cgraph * cgraph) {
7312 return cgraph->size;
7313}
7314
7315struct ggml_tensor * ggml_graph_node(struct ggml_cgraph * cgraph, int i) {
7316 if (i < 0) {
7317 GGML_ASSERT(cgraph->n_nodes + i >= 0)if (!(cgraph->n_nodes + i >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7317, "GGML_ASSERT(%s) failed", "cgraph->n_nodes + i >= 0"
);
7318 return cgraph->nodes[cgraph->n_nodes + i];
7319 }
7320
7321 GGML_ASSERT(i < cgraph->n_nodes)if (!(i < cgraph->n_nodes)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7321, "GGML_ASSERT(%s) failed", "i < cgraph->n_nodes"
);
7322 return cgraph->nodes[i];
7323}
7324
7325struct ggml_tensor ** ggml_graph_nodes(struct ggml_cgraph * cgraph) {
7326 return cgraph->nodes;
7327}
7328
7329int ggml_graph_n_nodes(struct ggml_cgraph * cgraph) {
7330 return cgraph->n_nodes;
7331}
7332
7333void ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor) {
7334 GGML_ASSERT(cgraph->size > cgraph->n_nodes)if (!(cgraph->size > cgraph->n_nodes)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7334, "GGML_ASSERT(%s) failed", "cgraph->size > cgraph->n_nodes"
);
7335 cgraph->nodes[cgraph->n_nodes] = tensor;
7336 cgraph->n_nodes++;
7337}
7338
7339struct ggml_tensor * ggml_graph_get_tensor(const struct ggml_cgraph * cgraph, const char * name) {
7340 for (int i = 0; i < cgraph->n_leafs; i++) {
7341 struct ggml_tensor * leaf = cgraph->leafs[i];
7342
7343 if (strcmp(leaf->name, name) == 0) {
7344 return leaf;
7345 }
7346 }
7347
7348 for (int i = 0; i < cgraph->n_nodes; i++) {
7349 struct ggml_tensor * node = cgraph->nodes[i];
7350
7351 if (strcmp(node->name, name) == 0) {
7352 return node;
7353 }
7354 }
7355
7356 return NULL((void*)0);
7357}
7358
7359struct ggml_tensor * ggml_graph_get_grad(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node) {
7360 const size_t igrad = ggml_hash_find(&cgraph->visited_hash_set, node);
7361 return igrad != GGML_HASHSET_FULL((size_t)-1) && ggml_bitset_get(cgraph->visited_hash_set.used, igrad) && cgraph->grads ? cgraph->grads[igrad] : NULL((void*)0);
7362}
7363
7364struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node) {
7365 const size_t igrad = ggml_hash_find(&cgraph->visited_hash_set, node);
7366 return igrad != GGML_HASHSET_FULL((size_t)-1) && ggml_bitset_get(cgraph->visited_hash_set.used, igrad) && cgraph->grad_accs ? cgraph->grad_accs[igrad] : NULL((void*)0);
7367}
7368
7369void ggml_graph_print(const struct ggml_cgraph * cgraph) {
7370 GGML_LOG_INFO("=== GRAPH ===\n")ggml_log_internal(GGML_LOG_LEVEL_INFO , "=== GRAPH ===\n");
7371
7372 GGML_LOG_INFO("n_nodes = %d\n", cgraph->n_nodes)ggml_log_internal(GGML_LOG_LEVEL_INFO , "n_nodes = %d\n", cgraph
->n_nodes);
7373 for (int i = 0; i < cgraph->n_nodes; i++) {
7374 struct ggml_tensor * node = cgraph->nodes[i];
7375
7376 GGML_LOG_INFO(" - %3d: [ %5" PRId64 ", %5" PRId64 ", %5" PRId64 "] %16s %s\n",ggml_log_internal(GGML_LOG_LEVEL_INFO , " - %3d: [ %5" "l" "d"
", %5" "l" "d" ", %5" "l" "d" "] %16s %s\n", i, node->ne[
0], node->ne[1], node->ne[2], ggml_op_name(node->op)
, (node->flags & GGML_TENSOR_FLAG_PARAM) ? "x" : ggml_graph_get_grad
(cgraph, node) ? "g" : " ")
7377 i,ggml_log_internal(GGML_LOG_LEVEL_INFO , " - %3d: [ %5" "l" "d"
", %5" "l" "d" ", %5" "l" "d" "] %16s %s\n", i, node->ne[
0], node->ne[1], node->ne[2], ggml_op_name(node->op)
, (node->flags & GGML_TENSOR_FLAG_PARAM) ? "x" : ggml_graph_get_grad
(cgraph, node) ? "g" : " ")
7378 node->ne[0], node->ne[1], node->ne[2],ggml_log_internal(GGML_LOG_LEVEL_INFO , " - %3d: [ %5" "l" "d"
", %5" "l" "d" ", %5" "l" "d" "] %16s %s\n", i, node->ne[
0], node->ne[1], node->ne[2], ggml_op_name(node->op)
, (node->flags & GGML_TENSOR_FLAG_PARAM) ? "x" : ggml_graph_get_grad
(cgraph, node) ? "g" : " ")
7379 ggml_op_name(node->op), (node->flags & GGML_TENSOR_FLAG_PARAM) ? "x" :ggml_log_internal(GGML_LOG_LEVEL_INFO , " - %3d: [ %5" "l" "d"
", %5" "l" "d" ", %5" "l" "d" "] %16s %s\n", i, node->ne[
0], node->ne[1], node->ne[2], ggml_op_name(node->op)
, (node->flags & GGML_TENSOR_FLAG_PARAM) ? "x" : ggml_graph_get_grad
(cgraph, node) ? "g" : " ")
7380 ggml_graph_get_grad(cgraph, node) ? "g" : " ")ggml_log_internal(GGML_LOG_LEVEL_INFO , " - %3d: [ %5" "l" "d"
", %5" "l" "d" ", %5" "l" "d" "] %16s %s\n", i, node->ne[
0], node->ne[1], node->ne[2], ggml_op_name(node->op)
, (node->flags & GGML_TENSOR_FLAG_PARAM) ? "x" : ggml_graph_get_grad
(cgraph, node) ? "g" : " ");
7381 }
7382
7383 GGML_LOG_INFO("n_leafs = %d\n", cgraph->n_leafs)ggml_log_internal(GGML_LOG_LEVEL_INFO , "n_leafs = %d\n", cgraph
->n_leafs);
7384 for (int i = 0; i < cgraph->n_leafs; i++) {
7385 struct ggml_tensor * node = cgraph->leafs[i];
7386
7387 GGML_LOG_INFO(" - %3d: [ %5" PRId64 ", %5" PRId64 "] %8s %16s\n",ggml_log_internal(GGML_LOG_LEVEL_INFO , " - %3d: [ %5" "l" "d"
", %5" "l" "d" "] %8s %16s\n", i, node->ne[0], node->ne
[1], ggml_op_name(node->op), ggml_get_name(node))
7388 i,ggml_log_internal(GGML_LOG_LEVEL_INFO , " - %3d: [ %5" "l" "d"
", %5" "l" "d" "] %8s %16s\n", i, node->ne[0], node->ne
[1], ggml_op_name(node->op), ggml_get_name(node))
7389 node->ne[0], node->ne[1],ggml_log_internal(GGML_LOG_LEVEL_INFO , " - %3d: [ %5" "l" "d"
", %5" "l" "d" "] %8s %16s\n", i, node->ne[0], node->ne
[1], ggml_op_name(node->op), ggml_get_name(node))
7390 ggml_op_name(node->op),ggml_log_internal(GGML_LOG_LEVEL_INFO , " - %3d: [ %5" "l" "d"
", %5" "l" "d" "] %8s %16s\n", i, node->ne[0], node->ne
[1], ggml_op_name(node->op), ggml_get_name(node))
7391 ggml_get_name(node))ggml_log_internal(GGML_LOG_LEVEL_INFO , " - %3d: [ %5" "l" "d"
", %5" "l" "d" "] %8s %16s\n", i, node->ne[0], node->ne
[1], ggml_op_name(node->op), ggml_get_name(node));
7392 }
7393
7394 GGML_LOG_INFO("========================================\n")ggml_log_internal(GGML_LOG_LEVEL_INFO , "========================================\n"
);
7395}
7396
7397static int ggml_node_list_find_tensor(const struct ggml_cgraph * cgraph,
7398 const int * idxs,
7399 int count,
7400 const struct ggml_tensor * tensor) {
7401 GGML_ASSERT(cgraph && idxs)if (!(cgraph && idxs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7401, "GGML_ASSERT(%s) failed", "cgraph && idxs");
7402 for (int i = 0; i < count; ++i) {
7403 const int node_idx = idxs[i];
7404
7405 if (node_idx >= cgraph->n_nodes) {
7406 return -1;
7407 }
7408 if (cgraph->nodes[node_idx] == tensor) {
7409 return i;
7410 }
7411 }
7412 return -1;
7413}
7414
7415bool_Bool ggml_can_fuse_subgraph_ext(const struct ggml_cgraph * cgraph,
7416 const int * node_idxs,
7417 int count,
7418 const enum ggml_op * ops,
7419 const int * outputs,
7420 int num_outputs) {
7421 GGML_ASSERT(outputs && num_outputs > 0)if (!(outputs && num_outputs > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7421, "GGML_ASSERT(%s) failed", "outputs && num_outputs > 0"
);
7422
7423 for (int i = 0; i < count; ++i) {
7424 if (node_idxs[i] >= cgraph->n_nodes) {
7425 return false0;
7426 }
7427
7428 const struct ggml_tensor * node = cgraph->nodes[node_idxs[i]];
7429
7430 if (node->op != ops[i]) {
7431 return false0;
7432 }
7433
7434 if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {
7435 return false0;
7436 }
7437
7438 if (ggml_node_list_find_tensor(cgraph, outputs, num_outputs, node) != -1) {
7439 continue;
7440 }
7441
7442 if (node->flags & GGML_TENSOR_FLAG_OUTPUT) {
7443 return false0;
7444 }
7445
7446 int subgraph_uses = 0;
7447 for (int j = i + 1; j < count; ++j) {
7448 const struct ggml_tensor * other_node = cgraph->nodes[node_idxs[j]];
7449 for (int src_idx = 0; src_idx < GGML_MAX_SRC10; src_idx++) {
7450 if (other_node->src[src_idx] == node) {
7451 subgraph_uses++;
7452 }
7453 }
7454 }
7455
7456 if (subgraph_uses != ggml_node_get_use_count(cgraph, node_idxs[i])) {
7457 return false0;
7458 }
7459
7460 // if node is a view, check if the view_src and all it's parent view_srcs are within the subgraph
7461 struct ggml_tensor * view_src = node->view_src;
7462 while (view_src) {
7463 if (ggml_node_list_find_tensor(cgraph, node_idxs, count, view_src) == -1) {
7464 return false0;
7465 }
7466 view_src = view_src->view_src;
7467 }
7468 }
7469
7470 return true1;
7471}
7472
7473// check if node is part of the graph
7474static bool_Bool ggml_graph_find(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node) {
7475 if (cgraph == NULL((void*)0)) {
7476 return true1;
7477 }
7478
7479 for (int i = 0; i < cgraph->n_nodes; i++) {
7480 if (cgraph->nodes[i] == node) {
7481 return true1;
7482 }
7483 }
7484
7485 return false0;
7486}
7487
7488static struct ggml_tensor * ggml_graph_get_parent(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node) {
7489 for (int i = 0; i < cgraph->n_nodes; i++) {
7490 struct ggml_tensor * parent = cgraph->nodes[i];
7491 struct ggml_tensor * grad = ggml_graph_get_grad(cgraph, parent);
7492
7493 if (grad == node) {
7494 return parent;
7495 }
7496 }
7497
7498 return NULL((void*)0);
7499}
7500
7501static void ggml_graph_dump_dot_node_edge(FILE * fp, const struct ggml_cgraph * gb, struct ggml_tensor * node, struct ggml_tensor * parent, const char * label) {
7502 struct ggml_tensor * gparent = ggml_graph_get_parent(gb, node);
7503 struct ggml_tensor * gparent0 = ggml_graph_get_parent(gb, parent);
7504 fprintf(fp, " \"%p\" -> \"%p\" [ arrowhead = %s; style = %s; label = \"%s\"; ]\n",
7505 gparent0 ? (void *) gparent0 : (void *) parent,
7506 gparent ? (void *) gparent : (void *) node,
7507 gparent ? "empty" : "vee",
7508 gparent ? "dashed" : "solid",
7509 label);
7510}
7511
7512static void ggml_graph_dump_dot_leaf_edge(FILE * fp, struct ggml_tensor * node, struct ggml_tensor * parent, const char * label) {
7513 fprintf(fp, " \"%p\" -> \"%p\" [ label = \"%s\"; ]\n",
7514 (void *) parent,
7515 (void *) node,
7516 label);
7517}
7518
7519void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * cgraph, const char * filename) {
7520 char color[16];
7521
7522 FILE * fp = ggml_fopen(filename, "w");
7523 GGML_ASSERT(fp)if (!(fp)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7523, "GGML_ASSERT(%s) failed", "fp");
7524
7525 fprintf(fp, "digraph G {\n");
7526 fprintf(fp, " newrank = true;\n");
7527 fprintf(fp, " rankdir = TB;\n");
7528
7529 for (int i = 0; i < gb->n_nodes; i++) {
7530 struct ggml_tensor * node = gb->nodes[i];
7531 struct ggml_tensor * grad = ggml_graph_get_grad(gb, node);
7532
7533 if (ggml_graph_get_parent(gb, node) != NULL((void*)0)) {
7534 continue;
7535 }
7536
7537 if (node->flags & GGML_TENSOR_FLAG_PARAM) {
7538 snprintf(color, sizeof(color), "yellow");
7539 } else if (grad) {
7540 if (ggml_graph_find(cgraph, node)) {
7541 snprintf(color, sizeof(color), "green");
7542 } else {
7543 snprintf(color, sizeof(color), "lightblue");
7544 }
7545 } else {
7546 snprintf(color, sizeof(color), "white");
7547 }
7548
7549 fprintf(fp, " \"%p\" [ "
7550 "style = filled; fillcolor = %s; shape = record; "
7551 "label=\"",
7552 (void *) node, color);
7553
7554 if (strlen(node->name) > 0) {
7555 fprintf(fp, "%s (%s)|", node->name, ggml_type_name(node->type));
7556 } else {
7557 fprintf(fp, "(%s)|", ggml_type_name(node->type));
7558 }
7559
7560 if (ggml_is_matrix(node)) {
7561 fprintf(fp, "%d [%" PRId64"l" "d" ", %" PRId64"l" "d" "] | <x>%s", i, node->ne[0], node->ne[1], ggml_op_symbol(node->op));
7562 } else {
7563 fprintf(fp, "%d [%" PRId64"l" "d" ", %" PRId64"l" "d" ", %" PRId64"l" "d" "] | <x>%s", i, node->ne[0], node->ne[1], node->ne[2], ggml_op_symbol(node->op));
7564 }
7565
7566 if (grad) {
7567 fprintf(fp, " | <g>%s\"; ]\n", ggml_op_symbol(grad->op));
7568 } else {
7569 fprintf(fp, "\"; ]\n");
7570 }
7571 }
7572
7573 for (int i = 0; i < gb->n_leafs; i++) {
7574 struct ggml_tensor * node = gb->leafs[i];
7575
7576 snprintf(color, sizeof(color), "pink");
7577
7578 fprintf(fp, " \"%p\" [ "
7579 "style = filled; fillcolor = %s; shape = record; "
7580 "label=\"<x>",
7581 (void *) node, color);
7582
7583 if (strlen(node->name) > 0) {
7584 fprintf(fp, "%s (%s)|", node->name, ggml_type_name(node->type));
7585 } else {
7586 fprintf(fp, "(%s)|", ggml_type_name(node->type));
7587 }
7588
7589 fprintf(fp, "CONST %d [%" PRId64"l" "d" ", %" PRId64"l" "d" "]", i, node->ne[0], node->ne[1]);
7590 if (ggml_nelements(node) < 5 && node->data != NULL((void*)0)) {
7591 fprintf(fp, " | (");
7592 for (int j = 0; j < ggml_nelements(node); j++) {
7593 // FIXME: use ggml-backend to obtain the tensor data
7594 //if (node->type == GGML_TYPE_I8 || node->type == GGML_TYPE_I16 || node->type == GGML_TYPE_I32) {
7595 // fprintf(fp, "%d", ggml_get_i32_1d(node, j));
7596 //}
7597 //else if (node->type == GGML_TYPE_F32 ||
7598 // node->type == GGML_TYPE_F16 ||
7599 // node->type == GGML_TYPE_BF16) {
7600 // fprintf(fp, "%.1e", (double)ggml_get_f32_1d(node, j));
7601 //}
7602 //else
7603 {
7604 fprintf(fp, "#");
7605 }
7606 if (j < ggml_nelements(node) - 1) {
7607 fprintf(fp, ", ");
7608 }
7609 }
7610 fprintf(fp, ")");
7611 }
7612 fprintf(fp, "\"; ]\n");
7613 }
7614
7615 for (int i = 0; i < gb->n_nodes; i++) {
7616 struct ggml_tensor * node = gb->nodes[i];
7617
7618 for (int j = 0; j < GGML_MAX_SRC10; j++) {
7619 if (node->src[j]) {
7620 char label[16];
7621 snprintf(label, sizeof(label), "src %d", j);
7622 ggml_graph_dump_dot_node_edge(fp, gb, node, node->src[j], label);
7623 }
7624 }
7625 }
7626
7627 for (int i = 0; i < gb->n_leafs; i++) {
7628 struct ggml_tensor * node = gb->leafs[i];
7629
7630 for (int j = 0; j < GGML_MAX_SRC10; j++) {
7631 if (node->src[j]) {
7632 char label[16];
7633 snprintf(label, sizeof(label), "src %d", j);
7634 ggml_graph_dump_dot_leaf_edge(fp, node, node->src[j], label);
7635 }
7636 }
7637 }
7638
7639 fprintf(fp, "}\n");
7640
7641 fclose(fp);
7642
7643 GGML_LOG_INFO("%s: dot -Tpng %s -o %s.png && open %s.png\n", __func__, filename, filename, filename)ggml_log_internal(GGML_LOG_LEVEL_INFO , "%s: dot -Tpng %s -o %s.png && open %s.png\n"
, __func__, filename, filename, filename);
7644}
7645
7646////////////////////////////////////////////////////////////////////////////////
7647
7648void ggml_set_input(struct ggml_tensor * tensor) {
7649 tensor->flags |= GGML_TENSOR_FLAG_INPUT;
7650}
7651
7652void ggml_set_output(struct ggml_tensor * tensor) {
7653 tensor->flags |= GGML_TENSOR_FLAG_OUTPUT;
7654}
7655
7656void ggml_set_param(struct ggml_tensor * tensor) {
7657 GGML_ASSERT(tensor->op == GGML_OP_NONE)if (!(tensor->op == GGML_OP_NONE)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7657, "GGML_ASSERT(%s) failed", "tensor->op == GGML_OP_NONE"
);
7658 tensor->flags |= GGML_TENSOR_FLAG_PARAM;
7659}
7660
7661void ggml_set_loss(struct ggml_tensor * tensor) {
7662 GGML_ASSERT(ggml_is_scalar(tensor))if (!(ggml_is_scalar(tensor))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7662, "GGML_ASSERT(%s) failed", "ggml_is_scalar(tensor)");
7663 GGML_ASSERT(tensor->type == GGML_TYPE_F32)if (!(tensor->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7663, "GGML_ASSERT(%s) failed", "tensor->type == GGML_TYPE_F32"
);
7664 tensor->flags |= GGML_TENSOR_FLAG_LOSS;
7665}
7666
7667////////////////////////////////////////////////////////////////////////////////
7668
7669void ggml_quantize_init(enum ggml_type type) {
7670 ggml_critical_section_start();
7671
7672 switch (type) {
7673 case GGML_TYPE_IQ2_XXS:
7674 case GGML_TYPE_IQ2_XS:
7675 case GGML_TYPE_IQ2_S:
7676 case GGML_TYPE_IQ1_S:
7677 case GGML_TYPE_IQ1_M: iq2xs_init_impl(type); break;
7678 case GGML_TYPE_IQ3_XXS: iq3xs_init_impl(256); break;
7679 case GGML_TYPE_IQ3_S: iq3xs_init_impl(512); break;
7680 default: // nothing
7681 break;
7682 }
7683
7684 ggml_critical_section_end();
7685}
7686
7687void ggml_quantize_free(void) {
7688 ggml_critical_section_start();
7689
7690 iq2xs_free_impl(GGML_TYPE_IQ2_XXS);
7691 iq2xs_free_impl(GGML_TYPE_IQ2_XS);
7692 iq2xs_free_impl(GGML_TYPE_IQ2_S);
7693 iq2xs_free_impl(GGML_TYPE_IQ1_S);
7694 iq2xs_free_impl(GGML_TYPE_IQ1_M);
7695 iq3xs_free_impl(256);
7696 iq3xs_free_impl(512);
7697
7698 ggml_critical_section_end();
7699}
7700
7701bool_Bool ggml_quantize_requires_imatrix(enum ggml_type type) {
7702 return
7703 type == GGML_TYPE_IQ2_XXS ||
7704 type == GGML_TYPE_IQ2_XS ||
7705 type == GGML_TYPE_IQ1_S;// ||
7706 //type == GGML_TYPE_IQ1_M;
7707}
7708
7709size_t ggml_quantize_chunk(
7710 enum ggml_type type,
7711 const float * src,
7712 void * dst,
7713 int64_t start,
7714 int64_t nrows,
7715 int64_t n_per_row,
7716 const float * imatrix) {
7717 const int64_t n = nrows * n_per_row;
7718
7719 if (ggml_quantize_requires_imatrix(type)) {
7720 GGML_ASSERT(imatrix != NULL)if (!(imatrix != ((void*)0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7720, "GGML_ASSERT(%s) failed", "imatrix != NULL");
7721 }
7722
7723 GGML_ASSERT(start % type_traits[type].blck_size == 0)if (!(start % type_traits[type].blck_size == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7723, "GGML_ASSERT(%s) failed", "start % type_traits[type].blck_size == 0"
);
7724 GGML_ASSERT(start % n_per_row == 0)if (!(start % n_per_row == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7724, "GGML_ASSERT(%s) failed", "start % n_per_row == 0");
7725
7726 ggml_quantize_init(type); // this is noop if already initialized
7727
7728 const size_t start_row = start / n_per_row;
7729 const size_t row_size = ggml_row_size(type, n_per_row);
7730
7731 size_t result = 0;
7732
7733 switch (type) {
7734 case GGML_TYPE_Q1_0: result = quantize_q1_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7735 case GGML_TYPE_Q4_0: result = quantize_q4_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7736 case GGML_TYPE_Q4_1: result = quantize_q4_1 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7737 case GGML_TYPE_Q5_0: result = quantize_q5_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7738 case GGML_TYPE_Q5_1: result = quantize_q5_1 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7739 case GGML_TYPE_Q8_0: result = quantize_q8_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7740 case GGML_TYPE_MXFP4: result = quantize_mxfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7741 case GGML_TYPE_NVFP4: result = quantize_nvfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7742 case GGML_TYPE_Q2_K: result = quantize_q2_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7743 case GGML_TYPE_Q3_K: result = quantize_q3_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7744 case GGML_TYPE_Q4_K: result = quantize_q4_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7745 case GGML_TYPE_Q5_K: result = quantize_q5_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7746 case GGML_TYPE_Q6_K: result = quantize_q6_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7747 case GGML_TYPE_TQ1_0: result = quantize_tq1_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7748 case GGML_TYPE_TQ2_0: result = quantize_tq2_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7749 case GGML_TYPE_IQ2_XXS: result = quantize_iq2_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7750 case GGML_TYPE_IQ2_XS: result = quantize_iq2_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7751 case GGML_TYPE_IQ3_XXS: result = quantize_iq3_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7752 case GGML_TYPE_IQ3_S: result = quantize_iq3_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7753 case GGML_TYPE_IQ2_S: result = quantize_iq2_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7754 case GGML_TYPE_IQ1_S: result = quantize_iq1_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7755 case GGML_TYPE_IQ1_M: result = quantize_iq1_m (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7756 case GGML_TYPE_IQ4_NL: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7757 case GGML_TYPE_IQ4_XS: result = quantize_iq4_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
7758 case GGML_TYPE_F16:
7759 {
7760 size_t elemsize = sizeof(ggml_fp16_t);
7761 ggml_fp32_to_fp16_row(src + start, (ggml_fp16_t *)dst + start, n);
7762 result = n * elemsize;
7763 } break;
7764 case GGML_TYPE_BF16:
7765 {
7766 size_t elemsize = sizeof(ggml_bf16_t);
7767 ggml_fp32_to_bf16_row_ref(src + start, (ggml_bf16_t *)dst + start, n);
7768 result = n * elemsize;
7769 } break;
7770 case GGML_TYPE_F32:
7771 {
7772 size_t elemsize = sizeof(float);
7773 result = n * elemsize;
7774 memcpy((uint8_t *)dst + start * elemsize, src + start, result);
7775 } break;
7776 default:
7777 assert(false)((void) sizeof (__assert_single_arg (0)), __extension__ ({ if
(0) ; else __assert_fail ("false", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7777, __extension__ __PRETTY_FUNCTION__); }));
7778 }
7779
7780 GGML_ASSERT(result == nrows * row_size)if (!(result == nrows * row_size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-c.c"
, 7780, "GGML_ASSERT(%s) failed", "result == nrows * row_size"
);
7781
7782 return result;
7783}
7784
7785////////////////////////////////////////////////////////////////////////////////
7786
7787void ggml_log_get(ggml_log_callback * log_callback, void ** user_data) {
7788 *log_callback = g_logger_state.log_callback;
7789 *user_data = g_logger_state.log_callback_user_data;
7790}
7791
7792void ggml_log_set(ggml_log_callback log_callback, void * user_data) {
7793 g_logger_state.log_callback = log_callback ? log_callback : ggml_log_callback_default;
7794 g_logger_state.log_callback_user_data = user_data;
7795}
7796
7797void ggml_threadpool_params_init(struct ggml_threadpool_params * p, int n_threads) {
7798 p->n_threads = n_threads;
7799 p->prio = 0; // default priority (usually means normal or inherited)
7800 p->poll = 50; // hybrid-polling enabled
7801 p->strict_cpu = false0; // no strict placement (all threads share same cpumask)
7802 p->paused = false0; // threads are ready to go
7803 p->thread_create_callback = 0;
7804 p->thread_destroy_callback = 0;
7805 memset(p->cpumask, 0, GGML_MAX_N_THREADS512); // all-zero means use the default affinity (usually inherited)
7806}
7807
7808struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads) {
7809 struct ggml_threadpool_params p;
7810 ggml_threadpool_params_init(&p, n_threads);
7811 return p;
7812}
7813
7814bool_Bool ggml_threadpool_params_match(const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1) {
7815 if (p0->n_threads != p1->n_threads ) return false0;
7816 if (p0->prio != p1->prio ) return false0;
7817 if (p0->poll != p1->poll ) return false0;
7818 if (p0->strict_cpu != p1->strict_cpu ) return false0;
7819 return memcmp(p0->cpumask, p1->cpumask, GGML_MAX_N_THREADS512) == 0;
7820}